feat(api): add the Glue Data Catalog to the cloud explorer

Create and delete Glue databases, and describe a folder of CSV, JSON-lines or
Parquet files in S3 as a table: its columns, and for CSV the delimiter and
whether the first line is a header. Tables are written with the storage
settings Athena expects, so TheCloud's Athena can query one as soon as it
exists. Glue jobs and crawlers are not offered: the runtime stores them but
never runs them.

packages/api/src/adapter-aws/AwsDataCatalogAdapter.test.ts

@@ -0,0 +1,155 @@
+import {describe, expect, test} from 'bun:test'
+import {
+    CreateDatabaseCommand,
+    CreateTableCommand,
+    DeleteDatabaseCommand,
+    DeleteTableCommand,
+    GetDatabaseCommand,
+    GetDatabasesCommand,
+    GetTablesCommand,
+    type GlueClient,
+} from '@aws-sdk/client-glue'
+import {AwsDataCatalogAdapter, describeTable} from './AwsDataCatalogAdapter'
+import {NotFoundError, ValidationError} from '../cloud-spi/errors'
+
+type SendResult = Record<string, unknown>
+
+const database = {Name: 'shop', Description: 'Orders and customers', LocationUri: 's3://data/shop/', CatalogId: '000000000000', CreateTime: new Date('2026-10-25T18:00:00Z')}
+const ordersTable = {
+    Name: 'orders',
+    TableType: 'EXTERNAL_TABLE',
+    Parameters: {classification: 'csv', 'skip.header.line.count': '1'},
+    StorageDescriptor: {
+        Columns: [{Name: 'id', Type: 'int'}, {Name: 'amount', Type: 'double'}],
+        Location: 's3://data/orders/',
+        InputFormat: 'org.apache.hadoop.mapred.TextInputFormat',
+        SerdeInfo: {SerializationLibrary: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', Parameters: {'field.delim': ';'}},
+    },
+    PartitionKeys: [{Name: 'day', Type: 'string'}],
+}
+const eventsTable = {
+    Name: 'events',
+    StorageDescriptor: {Columns: [{Name: 'kind', Type: 'string'}], Location: 's3://data/events/', SerdeInfo: {SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe'}},
+}
+
+function adapterWith(handler: (command: object) => SendResult | undefined = () => undefined) {
+    const sent: object[] = []
+    const client = {
+        async send(command: object) {
+            sent.push(command)
+            const result = handler(command)
+            if (result) return result
+            if (command instanceof GetDatabasesCommand) return {DatabaseList: [database]}
+            if (command instanceof GetDatabaseCommand) return {Database: database}
+            if (command instanceof GetTablesCommand) return {TableList: [ordersTable, eventsTable]}
+            return {}
+        },
+    }
+    return {sent, adapter: new AwsDataCatalogAdapter(client as unknown as GlueClient)}
+}
+
+describe('AwsDataCatalogAdapter', () => {
+    test('lists databases across pages with their table counts', async () => {
+        const {adapter} = adapterWith((command) => {
+            if (command instanceof GetDatabasesCommand) {
+                return command.input.NextToken ? {DatabaseList: [{Name: 'logs'}]} : {DatabaseList: [database], NextToken: 'n'}
+            }
+            if (command instanceof GetTablesCommand && command.input.DatabaseName === 'logs') return {TableList: []}
+            return undefined
+        })
+
+        const listed = await adapter.list()
+        expect(listed.map((db) => [db.name, db.metadata.tableCount, db.metadata.description])).toEqual([['shop', 2, 'Orders and customers'], ['logs', 0, null]])
+        expect((await adapter.list({search: 'customers'})).map((db) => db.name)).toEqual(['shop'])
+    })
+
+    test('describes each table with its format, location and columns, sorted by name', async () => {
+        const {adapter} = adapterWith()
+        const detail = await adapter.get('shop')
+        const tables = detail?.metadata.tables as Array<Record<string, unknown>>
+
+        expect(tables.map((table) => table.name)).toEqual(['events', 'orders'])
+        expect(tables[1]).toMatchObject({format: 'csv', delimiter: ';', headerRow: true, location: 's3://data/orders/', columns: [{name: 'id', type: 'int'}, {name: 'amount', type: 'double'}], partitionKeys: [{name: 'day', type: 'string'}]})
+        expect(tables[0]).toMatchObject({format: 'json', delimiter: null, headerRow: false})
+    })
+
+    test('reads the format the way the query engine does', () => {
+        expect(describeTable({Name: 'p', StorageDescriptor: {InputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'}}).format).toBe('parquet')
+        expect(describeTable({Name: 'i', Parameters: {table_type: 'ICEBERG'}}).format).toBe('iceberg')
+        expect(describeTable({Name: 'c'}).format).toBe('csv')
+    })
+
+    test('returns null for a database that does not exist', async () => {
+        const missing = Object.assign(new Error('missing'), {name: 'EntityNotFoundException'})
+        const {adapter} = adapterWith((command) => { if (command instanceof GetDatabaseCommand) throw missing; return undefined })
+        expect(await adapter.get('nope')).toBeNull()
+    })
+
+    test('creates a database with an optional description and location', async () => {
+        const {adapter, sent} = adapterWith()
+        await adapter.create({values: {name: 'shop', description: 'Orders', locationUri: 's3://data/shop/'}})
+        expect((sent.find((command) => command instanceof CreateDatabaseCommand) as CreateDatabaseCommand).input).toEqual({DatabaseInput: {Name: 'shop', Description: 'Orders', LocationUri: 's3://data/shop/'}})
+
+        await expect(adapter.create({values: {name: 'Shop'}})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.create({values: {name: 'shop', locationUri: 'https://example.com'}})).rejects.toBeInstanceOf(ValidationError)
+    })
+
+    test('creates a CSV table Athena can read, with its delimiter and header row', async () => {
+        const {adapter, sent} = adapterWith()
+        await adapter.createCatalogTable('shop', {name: 'orders', location: 's3://data/orders', format: 'csv', delimiter: ';', headerRow: true, columns: [{name: 'id', type: 'int'}, {name: 'amount', type: 'DOUBLE'}]})
+
+        const create = sent.find((command) => command instanceof CreateTableCommand) as CreateTableCommand
+        expect(create.input).toEqual({
+            DatabaseName: 'shop',
+            TableInput: {
+                Name: 'orders',
+                TableType: 'EXTERNAL_TABLE',
+                Parameters: {classification: 'csv', EXTERNAL: 'TRUE', 'skip.header.line.count': '1'},
+                StorageDescriptor: {
+                    Columns: [{Name: 'id', Type: 'int'}, {Name: 'amount', Type: 'double'}],
+                    Location: 's3://data/orders/',
+                    InputFormat: 'org.apache.hadoop.mapred.TextInputFormat',
+                    OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
+                    SerdeInfo: {SerializationLibrary: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', Parameters: {'field.delim': ';'}},
+                },
+            },
+        })
+    })
+
+    test('creates JSON and Parquet tables with their own SerDe', async () => {
+        const {adapter, sent} = adapterWith()
+        await adapter.createCatalogTable('shop', {name: 'events', location: 's3://data/events/', format: 'json', columns: [{name: 'kind', type: 'string'}]})
+        await adapter.createCatalogTable('shop', {name: 'facts', location: 's3://data/facts/', format: 'parquet', columns: [{name: 'total', type: 'decimal(12,2)'}]})
+
+        const [json, parquet] = sent.filter((command) => command instanceof CreateTableCommand) as CreateTableCommand[]
+        expect(json.input.TableInput?.StorageDescriptor?.SerdeInfo).toEqual({SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe'})
+        expect(parquet.input.TableInput?.StorageDescriptor?.InputFormat).toBe('org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat')
+        expect(parquet.input.TableInput?.StorageDescriptor?.Columns).toEqual([{Name: 'total', Type: 'decimal(12,2)'}])
+    })
+
+    test('refuses tables Athena could not query', async () => {
+        const {adapter} = adapterWith()
+        const table = {name: 'orders', location: 's3://data/orders/', format: 'csv' as const, columns: [{name: 'id', type: 'int'}]}
+
+        await expect(adapter.createCatalogTable('shop', {...table, columns: []})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.createCatalogTable('shop', {...table, columns: [{name: 'id', type: 'int'}, {name: 'id', type: 'string'}]})).rejects.toThrow(/listed twice/)
+        await expect(adapter.createCatalogTable('shop', {...table, columns: [{name: 'id', type: 'uuid'}]})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.createCatalogTable('shop', {...table, location: '/tmp/orders'})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.createCatalogTable('shop', {...table, delimiter: '||'})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.createCatalogTable('shop', {...table, format: 'avro' as never})).rejects.toBeInstanceOf(ValidationError)
+    })
+
+    test('creating a table in a missing database is a not-found', async () => {
+        const missing = Object.assign(new Error('missing'), {name: 'EntityNotFoundException'})
+        const {adapter} = adapterWith((command) => { if (command instanceof GetDatabaseCommand) throw missing; return undefined })
+        await expect(adapter.createCatalogTable('nope', {name: 'orders', location: 's3://data/orders/', format: 'json', columns: [{name: 'id', type: 'int'}]})).rejects.toBeInstanceOf(NotFoundError)
+    })
+
+    test('deletes tables and databases', async () => {
+        const {adapter, sent} = adapterWith()
+        await adapter.deleteCatalogTable('shop', 'orders')
+        await adapter.delete('shop')
+        expect((sent.find((command) => command instanceof DeleteTableCommand) as DeleteTableCommand).input).toEqual({DatabaseName: 'shop', Name: 'orders'})
+        expect((sent.find((command) => command instanceof DeleteDatabaseCommand) as DeleteDatabaseCommand).input).toEqual({Name: 'shop'})
+    })
+})