feat(api): add the Glue Data Catalog to the cloud explorer

Create and delete Glue databases, and describe a folder of CSV, JSON-lines or
Parquet files in S3 as a table: its columns, and for CSV the delimiter and
whether the first line is a header. Tables are written with the storage
settings Athena expects, so TheCloud's Athena can query one as soon as it
exists. Glue jobs and crawlers are not offered: the runtime stores them but
never runs them.

packages/frontend/src/api/api.ts

@@ -155,6 +155,10 @@ export const apiEndpointKeys = {
       export: "clouds.services.certificates.export",
       revoke: "clouds.services.certificates.revoke",
     },
+    datacatalog: {
+      tables: "clouds.services.datacatalog.tables",
+      table: "clouds.services.datacatalog.table",
+    },
     filesystems: {
       mountTargets: "clouds.services.filesystems.mountTargets",
       mountTarget: "clouds.services.filesystems.mountTarget",
@@ -964,6 +968,20 @@ export const endpointRegistry: EndpointRegistry = new Map([
       method: "POST",
     },
   ],
+  [
+    apiEndpointKeys.clouds.datacatalog.tables,
+    {
+      path: "/clouds/:cloud/services/datacatalog/resources/:id/tables",
+      method: "POST",
+    },
+  ],
+  [
+    apiEndpointKeys.clouds.datacatalog.table,
+    {
+      path: "/clouds/:cloud/services/datacatalog/resources/:id/tables/:tableName",
+      method: "DELETE",
+    },
+  ],
   [
     apiEndpointKeys.clouds.filesystems.mountTargets,
     {

packages/frontend/src/api/cloudProxyClient.ts

@@ -35,6 +35,7 @@ import type {
   DistributionInvalidation,
   AutoScalingCapacityInput,
   AccessPointInput,
+  CatalogTableInput,
   FileSystemAttachmentInput,
   CertificatePem,
   CertificateRevocationReason,
@@ -1723,6 +1724,32 @@ export async function revokeCertificate(
   return res.data;
 }
 
+export async function createCatalogTable(
+  cloud: CloudProvider,
+  databaseName: string,
+  input: CatalogTableInput,
+): Promise<CloudResource> {
+  const res = await apiClient.call<CloudResource, CatalogTableInput>(
+    apiEndpointKeys.clouds.datacatalog.tables,
+    requestOptions(cloud, "datacatalog", { body: input }),
+    { cloud, id: databaseName },
+  );
+  return res.data;
+}
+
+export async function deleteCatalogTable(
+  cloud: CloudProvider,
+  databaseName: string,
+  tableName: string,
+): Promise<CloudResource> {
+  const res = await apiClient.call<CloudResource>(
+    apiEndpointKeys.clouds.datacatalog.table,
+    requestOptions(cloud, "datacatalog"),
+    { cloud, id: databaseName, tableName },
+  );
+  return res.data;
+}
+
 export async function createMountTarget(
   cloud: CloudProvider,
   fileSystemId: string,

packages/frontend/src/components/DataCatalogPanel.test.tsx

@@ -0,0 +1,90 @@
+import {QueryClient, QueryClientProvider} from '@tanstack/react-query'
+import {render, screen, within} from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import {beforeEach, expect, test, vi} from 'vitest'
+import {DataCatalogPanel} from './DataCatalogPanel'
+import type {CloudResource} from '@/types/resource'
+
+const api = vi.hoisted(() => ({
+    getCloudResource: vi.fn(),
+    createCatalogTable: vi.fn(),
+    deleteCatalogTable: vi.fn(),
+}))
+
+vi.mock('@/api/cloudProxyClient', () => api)
+vi.mock('@/lib/accountStore', () => ({useAccountId: () => '000000000000'}))
+
+function database(tables: unknown[] = [{
+    name: 'orders', description: null, format: 'csv', location: 's3://data/orders/', delimiter: ';', headerRow: true,
+    columns: [{name: 'id', type: 'int'}, {name: 'amount', type: 'double'}], partitionKeys: [], createdAt: null, updatedAt: null,
+}]): CloudResource {
+    return {id: 'shop', name: 'shop', cloud: 'aws', service: 'datacatalog', type: 'glueDatabase', region: null, createdAt: null, metadata: {description: 'Orders', tableCount: tables.length, tables}}
+}
+
+function renderPanel(resource: CloudResource) {
+    api.getCloudResource.mockResolvedValue(resource)
+    const client = new QueryClient({defaultOptions: {queries: {retry: false}}})
+    return render(<QueryClientProvider client={client}><DataCatalogPanel resource={resource} runtimeReachable/></QueryClientProvider>)
+}
+
+beforeEach(() => {
+    vi.clearAllMocks()
+    api.createCatalogTable.mockResolvedValue(database())
+    api.deleteCatalogTable.mockResolvedValue(database([]))
+})
+
+test('lists tables with their format, location and columns', async () => {
+    renderPanel(database())
+
+    const row = (await screen.findByText('orders')).closest('tr') as HTMLElement
+    expect(within(row).getByText('s3://data/orders/')).toBeInTheDocument()
+    expect(within(row).getByText('semicolon-separated, header row')).toBeInTheDocument()
+    expect(within(row).getByText('amount')).toBeInTheDocument()
+})
+
+test('creates a CSV table from the columns entered', async () => {
+    const user = userEvent.setup()
+    renderPanel(database([]))
+
+    const form = await screen.findByRole('form', {name: 'Create table'})
+    await user.type(within(form).getByLabelText('Name'), 'events')
+    const location = within(form).getByLabelText('S3 location')
+    await user.clear(location)
+    await user.type(location, 's3://data/events/')
+    await user.type(within(form).getByLabelText('Column 1 name'), 'id')
+    await user.selectOptions(within(form).getByLabelText('Column 1 type'), 'int')
+    await user.click(within(form).getByRole('button', {name: 'Add column'}))
+    await user.type(within(form).getByLabelText('Column 2 name'), 'kind')
+    await user.click(within(form).getByRole('button', {name: 'Create table'}))
+
+    expect(api.createCatalogTable).toHaveBeenCalledWith('aws', 'shop', {
+        name: 'events', location: 's3://data/events/', format: 'csv', delimiter: ',', headerRow: true,
+        columns: [{name: 'id', type: 'int'}, {name: 'kind', type: 'string'}],
+    })
+    expect(await screen.findByText('Created table events. Athena can query it now.')).toBeInTheDocument()
+})
+
+test('JSON tables need no delimiter or header', async () => {
+    const user = userEvent.setup()
+    renderPanel(database([]))
+
+    const form = await screen.findByRole('form', {name: 'Create table'})
+    await user.selectOptions(within(form).getByLabelText('Format'), 'json')
+    expect(within(form).queryByLabelText('Delimiter')).not.toBeInTheDocument()
+    await user.type(within(form).getByLabelText('Name'), 'events')
+    await user.type(within(form).getByLabelText('S3 location'), 'data/events/')
+    await user.type(within(form).getByLabelText('Column 1 name'), 'kind')
+    await user.click(within(form).getByRole('button', {name: 'Create table'}))
+
+    expect(api.createCatalogTable).toHaveBeenCalledWith('aws', 'shop', {name: 'events', location: 's3://data/events/', format: 'json', columns: [{name: 'kind', type: 'string'}]})
+})
+
+test('deletes a table after confirming', async () => {
+    const user = userEvent.setup()
+    vi.spyOn(window, 'confirm').mockReturnValue(true)
+    renderPanel(database())
+
+    await user.click(await screen.findByRole('button', {name: 'Delete table orders'}))
+    expect(api.deleteCatalogTable).toHaveBeenCalledWith('aws', 'shop', 'orders')
+    expect(await screen.findByText('Deleted table orders. Its files stay in S3.')).toBeInTheDocument()
+})

packages/frontend/src/components/DataCatalogPanel.tsx

@@ -0,0 +1,145 @@
+import {Plus, RefreshCw, X} from 'lucide-react'
+import {useMutation, useQuery, useQueryClient} from '@tanstack/react-query'
+import {useEffect, useState} from 'react'
+import {createCatalogTable, deleteCatalogTable, getCloudResource} from '@/api/cloudProxyClient'
+import {cloudQueryKeys} from '@/api/queries/cloudQueries'
+import {useAccountId} from '@/lib/accountStore'
+import {formatDateTime} from '@/lib/format'
+import type {CatalogTableInput, CloudResource} from '@/types/resource'
+
+type Props = {
+    resource?: CloudResource
+    runtimeReachable: boolean
+}
+
+type Column = {name: string; type: string}
+type CatalogTable = {
+    name: string
+    description: string | null
+    format: string
+    location: string | null
+    delimiter: string | null
+    headerRow: boolean
+    columns: Column[]
+    partitionKeys: Column[]
+    updatedAt: string | null
+    createdAt: string | null
+}
+type TableForm = {name: string; location: string; format: CatalogTableInput['format']; delimiter: string; headerRow: boolean; columns: Column[]}
+
+/** Matches the types the API accepts; decimal(p,s) can also be typed in by hand through the API. */
+const COLUMN_TYPES = ['string', 'int', 'bigint', 'double', 'float', 'boolean', 'date', 'timestamp', 'decimal(10,2)']
+const FORMAT_LABELS: Record<string, string> = {csv: 'CSV', json: 'JSON lines', parquet: 'Parquet', iceberg: 'Iceberg'}
+const DELIMITER_NAMES: Record<string, string> = {',': 'comma-separated', ';': 'semicolon-separated', '\t': 'tab-separated', '|': 'pipe-separated'}
+const delimiterLabel = (delimiter: string | null) => DELIMITER_NAMES[delimiter ?? ','] ?? `separated by "${delimiter}"`
+const emptyForm = (): TableForm => ({name: '', location: 's3://', format: 'csv', delimiter: ',', headerRow: true, columns: [{name: '', type: 'string'}]})
+
+export function DataCatalogPanel({resource, runtimeReachable}: Props) {
+    const accountId = useAccountId()
+    const queryClient = useQueryClient()
+    const [form, setForm] = useState<TableForm>(emptyForm)
+    const [notice, setNotice] = useState('')
+    const detailKey = ['glue-database', accountId, resource?.id]
+
+    const detailQuery = useQuery({
+        queryKey: detailKey,
+        queryFn: ({signal}) => getCloudResource('aws', 'datacatalog', resource?.id ?? '', signal),
+        enabled: Boolean(resource) && runtimeReachable,
+    })
+
+    useEffect(() => {
+        setNotice('')
+        setForm(emptyForm())
+    }, [resource?.id])
+
+    const settle = (updated: CloudResource, message: string) => {
+        queryClient.setQueryData(detailKey, updated)
+        setNotice(message)
+        return queryClient.invalidateQueries({queryKey: cloudQueryKeys.resources('aws', 'datacatalog')})
+    }
+    const createTable = useMutation({
+        mutationFn: () => createCatalogTable('aws', resource?.id ?? '', {
+            name: form.name.trim(),
+            location: form.location.trim(),
+            format: form.format,
+            ...(form.format === 'csv' ? {delimiter: form.delimiter, headerRow: form.headerRow} : {}),
+            columns: form.columns.map((column) => ({name: column.name.trim(), type: column.type})),
+        }),
+        onSuccess: (updated) => {
+            const name = form.name.trim()
+            setForm(emptyForm())
+            return settle(updated, `Created table ${name}. Athena can query it now.`)
+        },
+    })
+    const deleteTable = useMutation({
+        mutationFn: (tableName: string) => deleteCatalogTable('aws', resource?.id ?? '', tableName),
+        onSuccess: (updated, tableName) => settle(updated, `Deleted table ${tableName}. Its files stay in S3.`),
+    })
+
+    if (!resource) {
+        return <section className="table-panel"><div className="empty compact"><h3>Select a database</h3><p>Choose a database to see its tables, or describe files in S3 as a new table.</p></div></section>
+    }
+
+    const database = detailQuery.data ?? resource
+    const tables = Array.isArray(database.metadata.tables) ? database.metadata.tables as CatalogTable[] : []
+    const disabled = !runtimeReachable || createTable.isPending || deleteTable.isPending
+    const error = detailQuery.error ?? createTable.error ?? deleteTable.error
+    const setColumn = (index: number, patch: Partial<Column>) => setForm({...form, columns: form.columns.map((column, i) => i === index ? {...column, ...patch} : column)})
+
+    return <section className="table-panel" style={{marginTop: 16}} aria-label={`Database ${database.name}`}>
+        <div className="input-row resource-table-bar">
+            <div><p className="eyebrow">Glue database</p><span className="muted">{database.metadata.description ? String(database.metadata.description) : 'Tables here describe files in S3 so Athena can query them.'}</span></div>
+            <div className="cdn-actions">
+                <button className="button" type="button" disabled={!runtimeReachable || detailQuery.isFetching} onClick={() => { void detailQuery.refetch() }}><RefreshCw size={14}/>{detailQuery.isFetching ? 'Loading' : 'Refresh'}</button>
+            </div>
+        </div>
+
+        <div className="input-row resource-table-bar"><div><p className="eyebrow">Tables</p><span className="muted">{tables.length === 1 ? '1 table' : `${tables.length} tables`}</span></div></div>
+        {tables.length === 0 ? <p className="muted compact-text catalog-note">No tables yet. Describe a folder of CSV, JSON or Parquet files below.</p>
+            : <div className="cognito-table"><table className="table"><thead><tr><th>Table</th><th>Format</th><th>Location</th><th>Columns</th><th>Updated</th><th aria-label="Actions"/></tr></thead><tbody>{tables.map((table) => (
+                <tr key={table.name}>
+                    <td className="mono">{table.name}{table.description && <><br/><small className="muted">{table.description}</small></>}</td>
+                    <td>{FORMAT_LABELS[table.format] ?? table.format}{table.format === 'csv' && <><br/><small className="muted">{delimiterLabel(table.delimiter)}{table.headerRow ? ', header row' : ''}</small></>}</td>
+                    <td className="mono catalog-location">{table.location ?? '—'}</td>
+                    <td className="catalog-columns">{table.columns.map((column) => <span key={column.name}><code>{column.name}</code> <small className="muted">{column.type}</small></span>)}
+                        {table.partitionKeys.length > 0 && <small className="muted">partitioned by {table.partitionKeys.map((key) => key.name).join(', ')}</small>}</td>
+                    <td>{formatDateTime(table.updatedAt ?? table.createdAt) ?? '—'}</td>
+                    <td className="efs-row-actions"><button className="button compact danger" type="button" disabled={disabled} aria-label={`Delete table ${table.name}`} onClick={() => {
+                        setNotice('')
+                        if (window.confirm(`Delete table ${table.name}? Its files stay in S3.`)) deleteTable.mutate(table.name)
+                    }}>Delete</button></td>
+                </tr>
+            ))}</tbody></table></div>}
+
+        <form className="catalog-form" aria-label="Create table" onSubmit={(event) => { event.preventDefault(); setNotice(''); createTable.mutate() }}>
+            <p className="eyebrow">New table</p>
+            <div className="dynamic-form catalog-form-fields">
+                <label className="dynamic-field"><span>Name</span><input className="input mono" value={form.name} onChange={(event) => setForm({...form, name: event.target.value})} disabled={disabled} required pattern="[a-z_][a-z0-9_]*" title="Lowercase letters, numbers and underscores"/></label>
+                <label className="dynamic-field catalog-span"><span>S3 location</span><input className="input mono" value={form.location} onChange={(event) => setForm({...form, location: event.target.value})} disabled={disabled} required pattern="s3://[a-z0-9.\-]{3,63}(/.*)?" title="A folder such as s3://my-bucket/orders/"/></label>
+                <label className="dynamic-field"><span>Format</span><select className="input" value={form.format} onChange={(event) => setForm({...form, format: event.target.value as TableForm['format']})} disabled={disabled}>
+                    <option value="csv">CSV</option><option value="json">JSON lines</option><option value="parquet">Parquet</option>
+                </select></label>
+                {form.format === 'csv' && <>
+                    <label className="dynamic-field"><span>Delimiter</span><input className="input mono" value={form.delimiter} maxLength={1} onChange={(event) => setForm({...form, delimiter: event.target.value})} disabled={disabled} required/></label>
+                    <label className="dynamic-field catalog-check"><input type="checkbox" checked={form.headerRow} onChange={(event) => setForm({...form, headerRow: event.target.checked})} disabled={disabled}/><span>First line is a header</span></label>
+                </>}
+            </div>
+            <fieldset className="catalog-columns-editor" disabled={disabled}>
+                <legend>Columns</legend>
+                {form.columns.map((column, index) => <div className="catalog-column-row" key={index}>
+                    <input className="input mono" aria-label={`Column ${index + 1} name`} value={column.name} onChange={(event) => setColumn(index, {name: event.target.value})} required pattern="[a-z_][a-z0-9_]*" title="Lowercase letters, numbers and underscores"/>
+                    <select className="input" aria-label={`Column ${index + 1} type`} value={column.type} onChange={(event) => setColumn(index, {type: event.target.value})}>
+                        {COLUMN_TYPES.map((type) => <option key={type} value={type}>{type}</option>)}
+                    </select>
+                    <button className="button compact" type="button" aria-label={`Remove column ${index + 1}`} disabled={form.columns.length === 1} onClick={() => setForm({...form, columns: form.columns.filter((_, i) => i !== index)})}><X size={14}/></button>
+                </div>)}
+                <button className="button compact" type="button" onClick={() => setForm({...form, columns: [...form.columns, {name: '', type: 'string'}]})}><Plus size={14}/>Add column</button>
+            </fieldset>
+            <p className="muted compact-text">List the columns in the order they appear in the files. For JSON, use the field names.</p>
+            <div><button className="button primary" type="submit" disabled={disabled}>{createTable.isPending ? 'Creating' : 'Create table'}</button></div>
+        </form>
+
+        {notice && <p className="cognito-outcome cognito-outcome--ok" role="status">{notice}</p>}
+        {error instanceof Error && <p className="error-text compact-text" role="alert">{error.message}</p>}
+    </section>
+}

packages/frontend/src/components/DynamicResourceView.tsx

@@ -70,6 +70,7 @@ import { Route53RecordsPanel } from "@/components/Route53RecordsPanel";
 import { CloudFrontPanel } from "@/components/CloudFrontPanel";
 import { AutoScalingPanel } from "@/components/AutoScalingPanel";
 import { FileSystemPanel } from "@/components/FileSystemPanel";
+import { DataCatalogPanel } from "@/components/DataCatalogPanel";
 import { CertificatePanel } from "@/components/CertificatePanel";
 import { PipePanel } from "@/components/PipePanel";
 import { CloudWatchAlarmPanel } from "@/components/CloudWatchAlarmPanel";
@@ -1003,6 +1004,9 @@ export function DynamicResourceView({
       {service === "filesystems" && cloud === "aws" && (
         <FileSystemPanel resource={activeSelected} runtimeReachable={canUseRuntime} />
       )}
+      {service === "datacatalog" && cloud === "aws" && (
+        <DataCatalogPanel resource={activeSelected} runtimeReachable={canUseRuntime} />
+      )}
       {service === "cdn" && cloud === "aws" && (
         <CloudFrontPanel resource={activeSelected} runtimeReachable={canUseRuntime} />
       )}

packages/frontend/src/components/serviceIcons.tsx

@@ -41,6 +41,7 @@ import route53Icon from '../../../../aws-icons-svg/Networking-Content-Delivery/R
 import cloudFrontIcon from '../../../../aws-icons-svg/Networking-Content-Delivery/CloudFront.svg'
 import autoScalingIcon from '../../../../aws-icons-svg/Compute/EC2-Auto-Scaling.svg'
 import efsIcon from '../../../../aws-icons-svg/Storage/EFS.svg'
+import glueIcon from '../../../../aws-icons-svg/Analytics/Glue.svg'
 import certificateManagerIcon from '../../../../aws-icons-svg/Security-Identity-Compliance/Certificate-Manager.svg'
 import cloudWatchIcon from '../../../../aws-icons-svg/Management-Governance/CloudWatch.svg'
 import cloudFormationIcon from '@/assets/aws-services/cloudformation.svg'
@@ -104,6 +105,7 @@ const SERVICE_ICONS: Record<string, LucideIcon> = {
     cdn: Earth,
     autoscaling: Scaling,
     filesystems: FolderTree,
+    datacatalog: Table2,
     certificates: FileBadge,
     monitoring: Activity,
     loadbalancing: Scale,
@@ -155,6 +157,7 @@ const AWS_SERVICE_ICON_ASSETS: Record<string, string> = {
     cdn: cloudFrontIcon,
     autoscaling: autoScalingIcon,
     filesystems: efsIcon,
+    datacatalog: glueIcon,
     certificates: certificateManagerIcon,
     monitoring: cloudWatchIcon,
     workflows: stepFunctionsIcon,

packages/frontend/src/index.css

@@ -6636,3 +6636,21 @@ tr[aria-current="true"] td { background: var(--hover-md); }
 .efs-inline-form { grid-template-columns: minmax(180px, 320px) auto; align-items: end; padding: 4px 16px 12px; }
 .efs-access-point-form { grid-template-columns: repeat(auto-fit, minmax(130px, 1fr)); align-items: end; padding: 4px 16px 12px; }
 .efs-row-actions { text-align: right; }
+
+/* Glue Data Catalog: column chips in the tables list, a compact column editor. */
+.catalog-note { padding: 0 16px 12px; margin: 0; }
+.catalog-location { overflow-wrap: anywhere; }
+.catalog-columns { display: flex; flex-wrap: wrap; gap: 4px 10px; }
+.catalog-columns > small { flex-basis: 100%; }
+.catalog-form { display: grid; gap: 10px; padding: 12px 16px; border-top: 1px solid var(--border); }
+.catalog-form p { margin: 0; }
+.catalog-form-fields { grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); align-items: end; }
+.catalog-span { grid-column: span 2; }
+.catalog-check { display: flex; align-items: center; gap: 8px; min-height: 30px; }
+.catalog-check input { width: auto; }
+.catalog-columns-editor { display: grid; gap: 6px; border: 0; padding: 0; margin: 0; min-width: 0; }
+.catalog-columns-editor legend { color: var(--text-2); font-size: 11px; font-weight: 600; text-transform: uppercase; margin-bottom: 4px; }
+.catalog-column-row { display: grid; grid-template-columns: minmax(0, 2fr) minmax(0, 1fr) auto; gap: 6px; max-width: 560px; }
+.catalog-column-row .input { width: 100%; min-width: 0; }
+.catalog-columns-editor > .button { justify-self: start; }
+@media (max-width: 640px) { .catalog-span { grid-column: auto; } }

packages/frontend/src/types/resource.ts

@@ -362,6 +362,21 @@ export interface AutoScalingCapacityInput {
     maxSize?: number
 }
 
+export interface CatalogColumnInput {
+    name: string
+    type: string
+}
+
+export interface CatalogTableInput {
+    name: string
+    description?: string
+    location: string
+    format: 'csv' | 'json' | 'parquet'
+    delimiter?: string
+    headerRow?: boolean
+    columns: CatalogColumnInput[]
+}
+
 export interface AccessPointInput {
     name?: string
     path: string

packages/frontend/src/types/schema.ts

@@ -30,6 +30,7 @@ export type ItemActionName = 'list' | 'get' | 'put' | 'delete' | 'query'
 export type DistributionActionName = 'setEnabled' | 'createInvalidation'
 export type AutoScalingActionName = 'setCapacity' | 'terminateInstance'
 export type CertificateActionName = 'download' | 'export' | 'revoke'
+export type DataCatalogActionName = 'createTable' | 'deleteTable'
 export type FileSystemActionName = 'createMountTarget' | 'deleteMountTarget' | 'createAccessPoint' | 'deleteAccessPoint' | 'attachFunction' | 'detachFunction'
 export type MonitoringActionName = 'listMetrics' | 'getMetricData' | 'alarmHistory' | 'setAlarmState' | 'setAlarmActions'
 export type UserPoolActionName = 'updateUser' | 'setUserGroups' | 'listGroups' | 'createGroup' | 'deleteGroup' | 'testSignIn' | 'describeClient' | 'updateClientOAuth'
@@ -97,6 +98,7 @@ export interface ServiceSchema {
         autoScalingActions?: Array<CapabilitySchema<AutoScalingActionName>>
         certificateActions?: Array<CapabilitySchema<CertificateActionName>>
         fileSystemActions?: Array<CapabilitySchema<FileSystemActionName>>
+        dataCatalogActions?: Array<CapabilitySchema<DataCatalogActionName>>
         monitoringActions?: Array<CapabilitySchema<MonitoringActionName>>
     }
     filters: FieldSchema[]