feat(api): add EventBridge Pipes to the cloud explorer

Create pipes from SQS, Kinesis, DynamoDB streams or MSK to Lambda, SQS, SNS, EventBridge buses or Step Functions, with an optional filter, enrichment and input template; start and stop them, see the flow with links to each resource, and send a test message to an SQS source. Sources and targets the runtime's poller cannot deliver between are refused on create.
2026-10-25 12:49:15 -04:00
parent 6e0bb98844
commit 12b50f4868
14 changed files with 773 additions and 0 deletions
@@ -0,0 +1,122 @@
+import {describe, expect, test} from 'bun:test'
+import {
+    CreatePipeCommand,
+    DeletePipeCommand,
+    DescribePipeCommand,
+    ListPipesCommand,
+    type PipesClient,
+    StartPipeCommand,
+    StopPipeCommand,
+} from '@aws-sdk/client-pipes'
+import {AwsPipesAdapter, describeArn} from './AwsPipesAdapter'
+import {ValidationError} from '../cloud-spi/errors'
+
+type SendResult = Record<string, unknown>
+
+function stubPipes(handler: (command: object) => SendResult | Promise<SendResult>) {
+    const sent: object[] = []
+    const client = {
+        async send(command: object) {
+            sent.push(command)
+            return handler(command)
+        },
+    } as unknown as PipesClient
+    return {client, sent, adapter: new AwsPipesAdapter(client)}
+}
+
+const SRC = 'arn:aws:sqs:us-east-1:000000000000:pipe-src'
+const DST = 'arn:aws:lambda:us-east-1:000000000000:function:worker'
+const ROLE = 'arn:aws:iam::000000000000:role/pipe-role'
+const described = {
+    Name: 'orders-pipe', Arn: 'arn:aws:pipes:us-east-1:000000000000:pipe/orders-pipe',
+    DesiredState: 'RUNNING', CurrentState: 'RUNNING', Source: SRC, Target: DST, RoleArn: ROLE,
+    SourceParameters: {FilterCriteria: {Filters: [{Pattern: '{"body":{"kind":["order"]}}'}]}, SqsQueueParameters: {BatchSize: 5}},
+    TargetParameters: {InputTemplate: '{"id": <$.messageId>}'},
+    CreationTime: new Date('2026-10-25T16:42:45Z'),
+}
+
+describe('AwsPipesAdapter', () => {
+    test('lists pipes across pages with readable source and target names', async () => {
+        const {adapter} = stubPipes((command) => {
+            if (!(command instanceof ListPipesCommand)) return {}
+            return command.input.NextToken
+                ? {Pipes: [{Name: 'clicks', CurrentState: 'STOPPED', Source: 'arn:aws:kinesis:us-east-1:000000000000:stream/clicks', Target: 'arn:aws:events:us-east-1:000000000000:event-bus/default'}]}
+                : {Pipes: [{Name: 'orders-pipe', CurrentState: 'RUNNING', Source: SRC, Target: DST}], NextToken: 'n'}
+        })
+
+        const pipes = await adapter.list()
+        expect(pipes.map((pipe) => [pipe.name, pipe.status, pipe.metadata.sourceName, pipe.metadata.targetName])).toEqual([
+            ['orders-pipe', 'RUNNING', 'pipe-src (SQS queue)', 'worker (Lambda function)'],
+            ['clicks', 'STOPPED', 'clicks (Kinesis stream)', 'default (EventBridge event bus)'],
+        ])
+        expect((await adapter.list({search: 'kinesis'})).map((pipe) => pipe.name)).toEqual(['clicks'])
+    })
+
+    test('describes the filter, batch size, template and role', async () => {
+        const {adapter} = stubPipes(() => described)
+        expect((await adapter.get('orders-pipe'))?.metadata).toMatchObject({
+            filterPatterns: ['{"body":{"kind":["order"]}}'],
+            batchSize: 5,
+            inputTemplate: '{"id": <$.messageId>}',
+            roleArn: ROLE,
+        })
+    })
+
+    test('a missing pipe is null', async () => {
+        const {adapter} = stubPipes(() => { throw Object.assign(new Error('gone'), {name: 'NotFoundException'}) })
+        expect(await adapter.get('nope')).toBeNull()
+    })
+
+    test('creates an SQS pipe with its filter, batch size and template', async () => {
+        const {adapter, sent} = stubPipes((command) => command instanceof DescribePipeCommand ? described : {})
+
+        await adapter.create({values: {name: 'orders-pipe', source: SRC, target: DST, roleArn: ROLE, batchSize: '5', filterPattern: '{ "body": { "kind": ["order"] } }', inputTemplate: '{"id": <$.messageId>}'}})
+        expect((sent[0] as CreatePipeCommand).input).toEqual({
+            Name: 'orders-pipe',
+            Source: SRC,
+            SourceParameters: {FilterCriteria: {Filters: [{Pattern: '{"body":{"kind":["order"]}}'}]}, SqsQueueParameters: {BatchSize: 5}},
+            Target: DST,
+            TargetParameters: {InputTemplate: '{"id": <$.messageId>}'},
+            RoleArn: ROLE,
+            DesiredState: 'RUNNING',
+        })
+    })
+
+    test('gives stream and MSK sources the starting position they require', async () => {
+        const {adapter, sent} = stubPipes((command) => command instanceof DescribePipeCommand ? described : {})
+
+        await adapter.create({values: {name: 'clicks', source: 'arn:aws:kinesis:us-east-1:000000000000:stream/clicks', target: DST, roleArn: ROLE, startingPosition: 'TRIM_HORIZON', desiredState: 'STOPPED'}})
+        expect((sent[0] as CreatePipeCommand).input).toMatchObject({SourceParameters: {KinesisStreamParameters: {StartingPosition: 'TRIM_HORIZON'}}, DesiredState: 'STOPPED'})
+
+        await adapter.create({values: {name: 'k', source: 'arn:aws:kafka:us-east-1:000000000000:cluster/events/abc', target: DST, roleArn: ROLE, topicName: 'orders'}})
+        expect((sent[2] as CreatePipeCommand).input.SourceParameters).toEqual({ManagedStreamingKafkaParameters: {TopicName: 'orders', StartingPosition: 'LATEST'}})
+    })
+
+    test('refuses pipes the runtime could not run, before calling it', async () => {
+        const {adapter, sent} = stubPipes(() => ({}))
+        const base = {name: 'p', source: SRC, target: DST, roleArn: ROLE}
+        await expect(adapter.create({values: {...base, target: 'arn:aws:s3:::bucket'}})).rejects.toThrow('target must be the ARN of a')
+        await expect(adapter.create({values: {...base, source: 'arn:aws:sns:us-east-1:000000000000:topic'}})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.create({values: {...base, enrichment: 'arn:aws:sqs:us-east-1:000000000000:q'}})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.create({values: {...base, filterPattern: '{kind: order}'}})).rejects.toThrow('filterPattern must be JSON')
+        await expect(adapter.create({values: {...base, batchSize: '20000'}})).rejects.toBeInstanceOf(ValidationError)
+        await expect(adapter.create({values: {...base, source: 'arn:aws:kafka:us-east-1:000000000000:cluster/c/1'}})).rejects.toThrow('topicName is required')
+        await expect(adapter.create({values: {...base, roleArn: 'pipe-role'}})).rejects.toBeInstanceOf(ValidationError)
+        expect(sent).toHaveLength(0)
+    })
+
+    test('starts, stops and deletes by name', async () => {
+        const {adapter, sent} = stubPipes(() => ({}))
+        await adapter.stop('orders-pipe')
+        await adapter.start('orders-pipe')
+        await adapter.delete('orders-pipe')
+        expect(sent.map((command) => command.constructor)).toEqual([StopPipeCommand, StartPipeCommand, DeletePipeCommand])
+        expect(sent.every((command) => (command as StopPipeCommand).input.Name === 'orders-pipe')).toBe(true)
+    })
+
+    test('names every kind of source and target ARN', () => {
+        expect(describeArn('arn:aws:dynamodb:us-east-1:000000000000:table/orders/stream/2026-10-25T00:00:00.000')).toBe('orders (DynamoDB stream)')
+        expect(describeArn('arn:aws:states:us-east-1:000000000000:stateMachine:flow')).toBe('flow (Step Functions state machine)')
+        expect(describeArn('arn:aws:sns:us-east-1:000000000000:alerts')).toBe('alerts (SNS topic)')
+    })
+})
@@ -0,0 +1,237 @@
+import {
+    CreatePipeCommand,
+    DeletePipeCommand,
+    DescribePipeCommand,
+    type DescribePipeCommandOutput,
+    ListPipesCommand,
+    type Pipe,
+    type PipeSourceParameters,
+    type PipesClient,
+    StartPipeCommand,
+    StopPipeCommand,
+} from '@aws-sdk/client-pipes'
+import {RuntimeError, ValidationError} from '../cloud-spi/errors'
+import {
+    awsPipesSchema,
+    PIPE_ENRICHMENT_SERVICES,
+    PIPE_NAME_PATTERN,
+    PIPE_SOURCE_SERVICES,
+    PIPE_STARTING_POSITIONS,
+    PIPE_TARGET_SERVICES,
+} from '../cloud-spi/pipesSchema'
+import type {CloudResource, CloudServiceAdapter, CreateResourceInput, ResourceQuery, ServiceSchema} from '../cloud-spi/types'
+
+const MAX_BATCH_SIZE = 10_000
+
+/**
+ * Talks to EventBridge Pipes, verified against Floci core (community, `latest`).
+ *
+ * Notes from probing the runtime:
+ *  - Pipes really run: the runtime polls SQS, Kinesis, DynamoDB streams and MSK,
+ *    applies the filter, enrichment and input template, and calls the target.
+ *  - Its poller delivers only to Lambda, SQS, SNS, EventBridge buses and Step
+ *    Functions, so create refuses other targets instead of making a pipe that
+ *    would never deliver.
+ *  - ListPipes returns no parameters, so the table shows only what it carries;
+ *    the filter, template and role come from DescribePipe.
+ */
+export class AwsPipesAdapter implements CloudServiceAdapter {
+    readonly cloud = 'aws' as const
+    readonly service = 'pipes' as const
+
+    constructor(private readonly pipes: PipesClient) {}
+
+    schema(): ServiceSchema {
+        return awsPipesSchema()
+    }
+
+    async list(query: ResourceQuery = {}): Promise<CloudResource[]> {
+        const found: Pipe[] = []
+        let next: string | undefined
+        do {
+            const res = await this.pipes.send(new ListPipesCommand({Limit: 100, ...(next ? {NextToken: next} : {})}))
+            found.push(...(res.Pipes ?? []))
+            next = res.NextToken
+        } while (next)
+
+        const search = query.search?.trim().toLowerCase()
+        return found.map((pipe) => toResource(pipe)).filter((pipe) => !search
+            || pipe.name.toLowerCase().includes(search)
+            || String(pipe.metadata.source).toLowerCase().includes(search)
+            || String(pipe.metadata.target).toLowerCase().includes(search))
+    }
+
+    async get(id: string): Promise<CloudResource | null> {
+        try {
+            return toResource(await this.pipes.send(new DescribePipeCommand({Name: id})))
+        } catch (error) {
+            if (errorName(error) === 'NotFoundException') return null
+            throw error
+        }
+    }
+
+    async create(input: CreateResourceInput): Promise<CloudResource> {
+        const values = input.values
+        const name = text(values.name)
+        if (!new RegExp(PIPE_NAME_PATTERN).test(name)) throw new ValidationError('name must be 1 to 64 letters, numbers, dots, dashes or underscores')
+
+        const source = text(values.source)
+        const sourceService = supported(source, PIPE_SOURCE_SERVICES, 'source')
+        const target = text(values.target)
+        supported(target, PIPE_TARGET_SERVICES, 'target')
+        const enrichment = text(values.enrichment)
+        if (enrichment) supported(enrichment, PIPE_ENRICHMENT_SERVICES, 'enrichment')
+        const roleArn = text(values.roleArn)
+        if (!/^arn:aws[a-z-]*:iam::\d{12}:role\/.+$/.test(roleArn)) throw new ValidationError('roleArn must be an IAM role ARN')
+
+        const inputTemplate = text(values.inputTemplate)
+        const description = text(values.description)
+        const desiredState = values.desiredState === 'STOPPED' ? 'STOPPED' : 'RUNNING'
+
+        await this.pipes.send(new CreatePipeCommand({
+            Name: name,
+            Source: source,
+            SourceParameters: sourceParameters(sourceService, values),
+            Target: target,
+            ...(inputTemplate ? {TargetParameters: {InputTemplate: inputTemplate}} : {}),
+            ...(enrichment ? {Enrichment: enrichment} : {}),
+            RoleArn: roleArn,
+            DesiredState: desiredState,
+            ...(description ? {Description: description} : {}),
+        }))
+
+        const created = await this.get(name)
+        if (!created) throw new RuntimeError(`EventBridge Pipes did not return the new pipe ${name}`)
+        return created
+    }
+
+    async delete(id: string): Promise<void> {
+        await this.pipes.send(new DeletePipeCommand({Name: id}))
+    }
+
+    async start(id: string): Promise<void> {
+        await this.pipes.send(new StartPipeCommand({Name: id}))
+    }
+
+    async stop(id: string): Promise<void> {
+        await this.pipes.send(new StopPipeCommand({Name: id}))
+    }
+}
+
+/** Each source kind carries its own parameter block, as CreatePipe expects. */
+function sourceParameters(service: keyof typeof PIPE_SOURCE_SERVICES, values: Record<string, unknown>): PipeSourceParameters {
+    const batchSize = optionalBatchSize(values.batchSize)
+    const filterPattern = text(values.filterPattern)
+    const parameters: PipeSourceParameters = filterPattern ? {FilterCriteria: {Filters: [{Pattern: jsonObject(filterPattern, 'filterPattern')}]}} : {}
+    const batch = batchSize ? {BatchSize: batchSize} : {}
+
+    if (service === 'sqs') return {...parameters, SqsQueueParameters: batch}
+
+    const startingPosition = text(values.startingPosition) || 'LATEST'
+    if (!(PIPE_STARTING_POSITIONS as readonly string[]).includes(startingPosition)) {
+        throw new ValidationError(`startingPosition must be one of ${PIPE_STARTING_POSITIONS.join(', ')}`)
+    }
+    const position = startingPosition as (typeof PIPE_STARTING_POSITIONS)[number]
+    if (service === 'kinesis') return {...parameters, KinesisStreamParameters: {StartingPosition: position, ...batch}}
+    if (service === 'dynamodb') return {...parameters, DynamoDBStreamParameters: {StartingPosition: position, ...batch}}
+
+    const topicName = text(values.topicName)
+    if (!topicName) throw new ValidationError('topicName is required for an MSK source')
+    return {...parameters, ManagedStreamingKafkaParameters: {TopicName: topicName, StartingPosition: position, ...batch}}
+}
+
+function toResource(pipe: Pipe | DescribePipeCommandOutput): CloudResource {
+    const name = pipe.Name ?? ''
+    const detail = 'SourceParameters' in pipe ? pipe : undefined
+    const sourceParameters = detail?.SourceParameters
+    const batchSize = sourceParameters?.SqsQueueParameters?.BatchSize
+        ?? sourceParameters?.KinesisStreamParameters?.BatchSize
+        ?? sourceParameters?.DynamoDBStreamParameters?.BatchSize
+        ?? sourceParameters?.ManagedStreamingKafkaParameters?.BatchSize
+    return {
+        id: name,
+        name,
+        cloud: 'aws',
+        service: 'pipes',
+        type: 'pipe',
+        region: pipe.Arn?.split(':')[3] || null,
+        createdAt: pipe.CreationTime ? pipe.CreationTime.toISOString() : null,
+        status: pipe.CurrentState ?? null,
+        metadata: {
+            arn: pipe.Arn,
+            desiredState: pipe.DesiredState ?? null,
+            currentState: pipe.CurrentState ?? null,
+            stateReason: pipe.StateReason ?? null,
+            source: pipe.Source ?? null,
+            sourceName: describeArn(pipe.Source),
+            target: pipe.Target ?? null,
+            targetName: describeArn(pipe.Target),
+            enrichment: pipe.Enrichment ?? null,
+            lastModified: pipe.LastModifiedTime ? pipe.LastModifiedTime.toISOString() : null,
+            ...(detail ? {
+                description: detail.Description ?? null,
+                roleArn: detail.RoleArn ?? null,
+                filterPatterns: (sourceParameters?.FilterCriteria?.Filters ?? []).map((filter) => filter.Pattern ?? '').filter(Boolean),
+                batchSize: batchSize ?? null,
+                startingPosition: sourceParameters?.KinesisStreamParameters?.StartingPosition
+                    ?? sourceParameters?.DynamoDBStreamParameters?.StartingPosition
+                    ?? sourceParameters?.ManagedStreamingKafkaParameters?.StartingPosition
+                    ?? null,
+                topicName: sourceParameters?.ManagedStreamingKafkaParameters?.TopicName ?? null,
+                inputTemplate: detail.TargetParameters?.InputTemplate ?? null,
+                enrichmentInputTemplate: detail.EnrichmentParameters?.InputTemplate ?? null,
+            } : {}),
+        },
+    }
+}
+
+/**
+ * A readable name for an ARN: `queue (SQS queue)`. The resource part differs by
+ * service — `function:name`, `stream/name`, `table/name/stream/…`, `cluster/name/uuid`.
+ */
+export function describeArn(arn: string | undefined): string | null {
+    if (!arn) return null
+    const parts = arn.split(':')
+    const service = parts[2] ?? ''
+    const resource = parts.slice(5).join(':')
+    const name = service === 'lambda' || service === 'states' ? resource.split(':')[1] ?? resource
+        : service === 'dynamodb' || service === 'kafka' || service === 'kinesis' || service === 'events' ? resource.split('/')[1] ?? resource
+        : resource
+    const kind = (PIPE_SOURCE_SERVICES as Record<string, string>)[service] ?? (PIPE_TARGET_SERVICES as Record<string, string>)[service]
+    return kind ? `${name} (${kind})` : name
+}
+
+function supported<T extends Record<string, string>>(arn: string, services: T, field: string): keyof T {
+    const service = arn.split(':')[2] ?? ''
+    if (!arn.startsWith('arn:') || !(service in services)) {
+        throw new ValidationError(`${field} must be the ARN of a ${Object.values(services).join(', ')}`)
+    }
+    return service as keyof T
+}
+
+function optionalBatchSize(value: unknown): number | undefined {
+    const raw = text(value)
+    if (!raw) return undefined
+    const size = Number(raw)
+    if (!Number.isInteger(size) || size < 1 || size > MAX_BATCH_SIZE) throw new ValidationError(`batchSize must be a whole number from 1 to ${MAX_BATCH_SIZE}`)
+    return size
+}
+
+function jsonObject(value: string, field: string): string {
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(value)
+    } catch {
+        throw new ValidationError(`${field} must be JSON, such as {"body": {"kind": ["order"]}}`)
+    }
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) throw new ValidationError(`${field} must be a JSON object`)
+    return JSON.stringify(parsed)
+}
+
+function text(value: unknown): string {
+    return typeof value === 'string' ? value.trim() : ''
+}
+
+function errorName(error: unknown): string | undefined {
+    return typeof error === 'object' && error !== null ? (error as {name?: string}).name : undefined
+}