feat: MSK, MSK CONNECT and SES
This commit is contained in:
Zakaria
@@ -43,6 +43,7 @@ import {
|
||||
DisassociateRouteTableCommand,
|
||||
GetConsoleOutputCommand,
|
||||
ModifySubnetAttributeCommand,
|
||||
ModifyInstanceAttributeCommand,
|
||||
ModifyVpcAttributeCommand,
|
||||
RebootInstancesCommand,
|
||||
ReleaseAddressCommand,
|
||||
@@ -140,6 +141,14 @@ export type Ec2Instance = {
|
||||
securityGroups: Ec2SecurityGroupRef[]
|
||||
networkInterfaces: Ec2NetworkInterface[]
|
||||
tags: Ec2Tag[]
|
||||
connection?: Ec2InstanceConnection
|
||||
}
|
||||
|
||||
export type Ec2InstanceConnection = {
|
||||
sshHost?: string
|
||||
sshPort?: number
|
||||
containerId?: string
|
||||
containerStatus: 'running' | 'stopped' | 'unavailable'
|
||||
}
|
||||
|
||||
export type Ec2KeyPair = {
|
||||
@@ -572,6 +581,13 @@ export function createEc2Service(client: EC2Client = awsClients.ec2) {
|
||||
}
|
||||
},
|
||||
|
||||
async updateInstanceSecurityGroups(instanceId: string, groupIds: string[]): Promise<void> {
|
||||
await client.send(new ModifyInstanceAttributeCommand({
|
||||
InstanceId: instanceId,
|
||||
Groups: groupIds,
|
||||
}))
|
||||
},
|
||||
|
||||
async createImage(input: CreateAmiInput): Promise<{imageId: string}> {
|
||||
const res = await client.send(new CreateImageCommand({
|
||||
InstanceId: input.instanceId,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
CreateClusterCommand,
|
||||
GetBootstrapBrokersCommand,
|
||||
ListClustersCommand,
|
||||
type KafkaClient,
|
||||
} from "@aws-sdk/client-kafka";
|
||||
import { createMskService, type KafkaClientFactory } from "./msk";
|
||||
|
||||
function createAwsClient() {
|
||||
const commands: unknown[] = [];
|
||||
const client = {
|
||||
send: async (command: unknown) => {
|
||||
commands.push(command);
|
||||
if (command instanceof GetBootstrapBrokersCommand) {
|
||||
return { BootstrapBrokerString: "broker-1:9092,broker-2:9092" };
|
||||
}
|
||||
if (command instanceof ListClustersCommand) {
|
||||
return { ClusterInfoList: [{ ClusterName: "orders" }] };
|
||||
}
|
||||
return { ClusterName: "orders", ClusterArn: "arn:cluster:orders" };
|
||||
},
|
||||
} as unknown as KafkaClient;
|
||||
return { client, commands };
|
||||
}
|
||||
|
||||
function createKafkaFactory() {
|
||||
const calls: string[] = [];
|
||||
const factory: KafkaClientFactory = () => ({
|
||||
admin: () => ({
|
||||
connect: async () => { calls.push("admin.connect"); },
|
||||
disconnect: async () => { calls.push("admin.disconnect"); },
|
||||
fetchTopicMetadata: async () => ({ topics: [{ name: "orders", partitions: [{}, {}] }] }),
|
||||
createTopics: async () => { calls.push("admin.createTopics"); return true; },
|
||||
deleteTopics: async () => { calls.push("admin.deleteTopics"); },
|
||||
fetchTopicOffsets: async () => [{ partition: 0, offset: "12" }],
|
||||
fetchOffsets: async () => [{ topic: "orders", partitions: [{ partition: 0, offset: "7" }] }],
|
||||
}),
|
||||
producer: () => ({
|
||||
connect: async () => { calls.push("producer.connect"); },
|
||||
disconnect: async () => { calls.push("producer.disconnect"); },
|
||||
send: async () => [{ topicName: "orders", partition: 0, baseOffset: "12" }],
|
||||
}),
|
||||
consumer: () => ({
|
||||
connect: async () => { calls.push("consumer.connect"); },
|
||||
disconnect: async () => { calls.push("consumer.disconnect"); },
|
||||
subscribe: async () => { calls.push("consumer.subscribe"); },
|
||||
run: async ({ eachMessage }) => {
|
||||
await eachMessage({
|
||||
partition: 0,
|
||||
message: {
|
||||
key: Buffer.from("key"),
|
||||
value: Buffer.from("value"),
|
||||
headers: { source: Buffer.from("test") },
|
||||
offset: "12",
|
||||
timestamp: "123",
|
||||
},
|
||||
});
|
||||
},
|
||||
stop: async () => { calls.push("consumer.stop"); },
|
||||
}),
|
||||
});
|
||||
return { factory, calls };
|
||||
}
|
||||
|
||||
describe("mskService", () => {
|
||||
test("lists clusters and creates clusters with the MSK SDK", async () => {
|
||||
const { client, commands } = createAwsClient();
|
||||
const service = createMskService(client, createKafkaFactory().factory);
|
||||
|
||||
expect(await service.listClusters()).toEqual([{ ClusterName: "orders" }]);
|
||||
await service.createCluster({
|
||||
clusterName: "orders",
|
||||
kafkaVersion: "3.6.x",
|
||||
numberOfBrokerNodes: 3,
|
||||
brokerNodeGroupInfo: { InstanceType: "kafka.m5.large", ClientSubnets: ["subnet-1", "subnet-2"] },
|
||||
});
|
||||
|
||||
expect(commands[0]).toBeInstanceOf(ListClustersCommand);
|
||||
expect(commands[1]).toBeInstanceOf(CreateClusterCommand);
|
||||
expect((commands[1] as CreateClusterCommand).input).toMatchObject({ ClusterName: "orders" });
|
||||
});
|
||||
|
||||
test("uses server-resolved bootstrap brokers for topic and message operations", async () => {
|
||||
const { client, commands } = createAwsClient();
|
||||
const { factory, calls } = createKafkaFactory();
|
||||
const service = createMskService(client, factory);
|
||||
|
||||
expect(await service.listTopics("arn:cluster:orders")).toEqual([{ name: "orders", partitionCount: 2 }]);
|
||||
await service.createTopic("arn:cluster:orders", { topic: "events", numPartitions: 1, replicationFactor: 1 });
|
||||
await service.deleteTopic("arn:cluster:orders", "events");
|
||||
expect(await service.produceMessage("arn:cluster:orders", "orders", { value: "hello" })).toEqual([
|
||||
{ topicName: "orders", partition: 0, baseOffset: "12" },
|
||||
]);
|
||||
|
||||
expect(commands.filter((command) => command instanceof GetBootstrapBrokersCommand)).toHaveLength(4);
|
||||
expect(calls).toContain("admin.createTopics");
|
||||
expect(calls).toContain("producer.disconnect");
|
||||
});
|
||||
|
||||
test("peeks without committing and calculates consumer group lag", async () => {
|
||||
const { client } = createAwsClient();
|
||||
const { factory, calls } = createKafkaFactory();
|
||||
const service = createMskService(client, factory);
|
||||
|
||||
expect(await service.peekMessages("arn:cluster:orders", "orders", 1)).toEqual([
|
||||
{
|
||||
partition: 0,
|
||||
offset: "12",
|
||||
timestamp: "123",
|
||||
key: "key",
|
||||
value: "value",
|
||||
headers: { source: "test" },
|
||||
},
|
||||
]);
|
||||
expect(await service.consumerGroupLag("arn:cluster:orders", "orders-workers")).toEqual([
|
||||
{
|
||||
topic: "orders",
|
||||
partitions: [{ partition: 0, currentOffset: "7", endOffset: "12", lag: 5 }],
|
||||
totalLag: 5,
|
||||
},
|
||||
]);
|
||||
expect(calls).toContain("consumer.stop");
|
||||
expect(calls).toContain("consumer.disconnect");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,273 @@
|
||||
import {
|
||||
CreateClusterCommand,
|
||||
DeleteClusterCommand,
|
||||
GetBootstrapBrokersCommand,
|
||||
KafkaClient,
|
||||
ListClustersCommand,
|
||||
type BrokerNodeGroupInfo,
|
||||
} from "@aws-sdk/client-kafka";
|
||||
import { Kafka } from "kafkajs";
|
||||
import { awsClients } from "../aws";
|
||||
|
||||
export type CreateMskClusterInput = {
|
||||
clusterName: string;
|
||||
kafkaVersion: string;
|
||||
numberOfBrokerNodes: number;
|
||||
brokerNodeGroupInfo: BrokerNodeGroupInfo;
|
||||
tags?: Record<string, string>;
|
||||
};
|
||||
|
||||
export type CreateTopicInput = {
|
||||
topic: string;
|
||||
numPartitions?: number;
|
||||
replicationFactor?: number;
|
||||
configEntries?: { name: string; value: string }[];
|
||||
};
|
||||
|
||||
export type ProduceMessageInput = {
|
||||
key?: string;
|
||||
value: string;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
|
||||
type KafkaAdmin = {
|
||||
connect(): Promise<void>;
|
||||
disconnect(): Promise<void>;
|
||||
fetchTopicMetadata(input?: { topics?: string[] }): Promise<{
|
||||
topics: { name: string; partitions: unknown[] }[];
|
||||
}>;
|
||||
createTopics(input: { topics: CreateTopicInput[] }): Promise<boolean>;
|
||||
deleteTopics(input: { topics: string[] }): Promise<void>;
|
||||
fetchTopicOffsets(topic: string): Promise<{ partition: number; offset: string }[]>;
|
||||
fetchOffsets(input: {
|
||||
groupId: string;
|
||||
topics?: string[];
|
||||
}): Promise<{ topic: string; partitions: { partition: number; offset: string }[] }[]>;
|
||||
};
|
||||
|
||||
type KafkaProducer = {
|
||||
connect(): Promise<void>;
|
||||
disconnect(): Promise<void>;
|
||||
send(input: {
|
||||
topic: string;
|
||||
messages: { key?: string; value: string; headers?: Record<string, string> }[];
|
||||
}): Promise<{ topicName: string; partition: number; baseOffset?: string }[]>;
|
||||
};
|
||||
|
||||
type KafkaConsumer = {
|
||||
connect(): Promise<void>;
|
||||
disconnect(): Promise<void>;
|
||||
subscribe(input: { topic: string; fromBeginning: boolean }): Promise<void>;
|
||||
run(input: {
|
||||
autoCommit: boolean;
|
||||
eachMessage: (payload: {
|
||||
partition: number;
|
||||
message: {
|
||||
key: Buffer | null;
|
||||
value: Buffer | null;
|
||||
headers?: Record<string, Buffer>;
|
||||
offset: string;
|
||||
timestamp: string;
|
||||
};
|
||||
}) => Promise<void>;
|
||||
}): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
export type KafkaClientFactory = (brokers: string[]) => {
|
||||
admin(): KafkaAdmin;
|
||||
producer(): KafkaProducer;
|
||||
consumer(input: { groupId: string }): KafkaConsumer;
|
||||
};
|
||||
|
||||
function createKafkaClient(brokers: string[]) {
|
||||
return new Kafka({ brokers }) as unknown as ReturnType<KafkaClientFactory>;
|
||||
}
|
||||
|
||||
function splitBootstrapBrokers(bootstrapBrokers: string): string[] {
|
||||
return bootstrapBrokers.split(",").map((broker) => broker.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
export function createMskService(
|
||||
client: KafkaClient = awsClients.kafka,
|
||||
kafkaClientFactory: KafkaClientFactory = createKafkaClient,
|
||||
) {
|
||||
async function brokers(clusterArn: string): Promise<string[]> {
|
||||
const command = new GetBootstrapBrokersCommand({ ClusterArn: clusterArn });
|
||||
command.middlewareStack.add(
|
||||
(next) => async (args) => {
|
||||
const request = args.request as { headers: Record<string, string> };
|
||||
request.headers["x-floci-internal-client"] = "true";
|
||||
return next(args);
|
||||
},
|
||||
{ step: "build", name: "flociInternalMskClient" },
|
||||
);
|
||||
const response = await client.send(command);
|
||||
const bootstrapBrokers = response.BootstrapBrokerString ?? response.BootstrapBrokerStringTls;
|
||||
if (!bootstrapBrokers) {
|
||||
throw new Error(`MSK cluster ${clusterArn} did not return bootstrap brokers`);
|
||||
}
|
||||
return splitBootstrapBrokers(bootstrapBrokers);
|
||||
}
|
||||
|
||||
return {
|
||||
async listClusters() {
|
||||
const clusters = [];
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const response = await client.send(new ListClustersCommand({ NextToken: nextToken }));
|
||||
clusters.push(...(response.ClusterInfoList ?? []));
|
||||
nextToken = response.NextToken;
|
||||
} while (nextToken);
|
||||
return clusters;
|
||||
},
|
||||
|
||||
async createCluster(input: CreateMskClusterInput) {
|
||||
const response = await client.send(new CreateClusterCommand({
|
||||
ClusterName: input.clusterName,
|
||||
KafkaVersion: input.kafkaVersion,
|
||||
NumberOfBrokerNodes: input.numberOfBrokerNodes,
|
||||
BrokerNodeGroupInfo: input.brokerNodeGroupInfo,
|
||||
Tags: input.tags,
|
||||
}));
|
||||
return response;
|
||||
},
|
||||
|
||||
async deleteCluster(clusterArn: string) {
|
||||
return client.send(new DeleteClusterCommand({ ClusterArn: clusterArn }));
|
||||
},
|
||||
|
||||
async getBootstrapBrokers(clusterArn: string) {
|
||||
const response = await client.send(new GetBootstrapBrokersCommand({ ClusterArn: clusterArn }));
|
||||
return response.BootstrapBrokerString ?? response.BootstrapBrokerStringTls;
|
||||
},
|
||||
|
||||
async listTopics(clusterArn: string) {
|
||||
const admin = kafkaClientFactory(await brokers(clusterArn)).admin();
|
||||
await admin.connect();
|
||||
try {
|
||||
const metadata = await admin.fetchTopicMetadata();
|
||||
return metadata.topics
|
||||
.filter((topic) => !topic.name.startsWith("__"))
|
||||
.map((topic) => ({
|
||||
name: topic.name,
|
||||
partitionCount: topic.partitions.length,
|
||||
}));
|
||||
} finally {
|
||||
await admin.disconnect();
|
||||
}
|
||||
},
|
||||
|
||||
async createTopic(clusterArn: string, input: CreateTopicInput) {
|
||||
const admin = kafkaClientFactory(await brokers(clusterArn)).admin();
|
||||
await admin.connect();
|
||||
try {
|
||||
await admin.createTopics({ topics: [input] });
|
||||
} finally {
|
||||
await admin.disconnect();
|
||||
}
|
||||
},
|
||||
|
||||
async deleteTopic(clusterArn: string, topic: string) {
|
||||
const admin = kafkaClientFactory(await brokers(clusterArn)).admin();
|
||||
await admin.connect();
|
||||
try {
|
||||
await admin.deleteTopics({ topics: [topic] });
|
||||
} finally {
|
||||
await admin.disconnect();
|
||||
}
|
||||
},
|
||||
|
||||
async produceMessage(clusterArn: string, topic: string, input: ProduceMessageInput) {
|
||||
const producer = kafkaClientFactory(await brokers(clusterArn)).producer();
|
||||
await producer.connect();
|
||||
try {
|
||||
return await producer.send({ topic, messages: [input] });
|
||||
} finally {
|
||||
await producer.disconnect();
|
||||
}
|
||||
},
|
||||
|
||||
async peekMessages(clusterArn: string, topic: string, limit = 50) {
|
||||
const consumer = kafkaClientFactory(await brokers(clusterArn)).consumer({
|
||||
groupId: `floci-ui-peek-${crypto.randomUUID()}`,
|
||||
});
|
||||
const messages: { partition: number; offset: string; timestamp: string; key?: string; value?: string; headers: Record<string, string> }[] = [];
|
||||
await consumer.connect();
|
||||
try {
|
||||
await consumer.subscribe({ topic, fromBeginning: true });
|
||||
await new Promise<void>(async (resolve, reject) => {
|
||||
const timeout = setTimeout(resolve, 500);
|
||||
try {
|
||||
await consumer.run({
|
||||
autoCommit: false,
|
||||
eachMessage: async ({ partition, message }) => {
|
||||
if (messages.length >= limit) {
|
||||
return;
|
||||
}
|
||||
messages.push({
|
||||
partition,
|
||||
offset: message.offset,
|
||||
timestamp: message.timestamp,
|
||||
key: message.key?.toString(),
|
||||
value: message.value?.toString(),
|
||||
headers: Object.fromEntries(
|
||||
Object.entries(message.headers ?? {}).map(([name, value]) => [name, value.toString()]),
|
||||
),
|
||||
});
|
||||
if (messages.length >= limit) {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
clearTimeout(timeout);
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
return messages;
|
||||
} finally {
|
||||
await consumer.stop();
|
||||
await consumer.disconnect();
|
||||
}
|
||||
},
|
||||
|
||||
async consumerGroupLag(clusterArn: string, groupId: string, topic?: string) {
|
||||
const admin = kafkaClientFactory(await brokers(clusterArn)).admin();
|
||||
await admin.connect();
|
||||
let completed = false;
|
||||
try {
|
||||
const offsets = await admin.fetchOffsets({ groupId, topics: topic ? [topic] : undefined });
|
||||
const lag = await Promise.all(offsets.map(async (topicOffsets) => {
|
||||
const endOffsets = await admin.fetchTopicOffsets(topicOffsets.topic);
|
||||
const endOffsetByPartition = new Map(endOffsets.map((offset) => [offset.partition, offset.offset]));
|
||||
const partitions = topicOffsets.partitions.map((offset) => ({
|
||||
partition: offset.partition,
|
||||
currentOffset: offset.offset,
|
||||
endOffset: endOffsetByPartition.get(offset.partition) ?? "0",
|
||||
lag: Math.max(0, Number(endOffsetByPartition.get(offset.partition) ?? 0) - Math.max(0, Number(offset.offset))),
|
||||
}));
|
||||
return {
|
||||
topic: topicOffsets.topic,
|
||||
partitions,
|
||||
totalLag: partitions.reduce((total, partition) => total + partition.lag, 0),
|
||||
};
|
||||
}));
|
||||
completed = true;
|
||||
return lag;
|
||||
} finally {
|
||||
try {
|
||||
await admin.disconnect();
|
||||
} catch (ignored) {
|
||||
// KafkaJS can close the coordinator socket after returning offsets.
|
||||
if (!completed) {
|
||||
throw ignored;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const mskService = createMskService();
|
||||
@@ -0,0 +1,32 @@
|
||||
import {describe, expect, test} from "bun:test";
|
||||
import {CreateConnectorCommand, CreateCustomPluginCommand, DeleteConnectorCommand, DescribeConnectorCommand, ListConnectorsCommand, ListCustomPluginsCommand, UpdateConnectorCommand, type KafkaConnectClient} from "@aws-sdk/client-kafkaconnect";
|
||||
import {createMskConnectService} from "./mskConnect";
|
||||
|
||||
test("uses Kafka Connect SDK commands with AWS request shapes", async () => {
|
||||
const commands: unknown[] = [];
|
||||
const client = {send: async (command: unknown) => {
|
||||
commands.push(command);
|
||||
if (command instanceof ListCustomPluginsCommand) return {customPlugins: [{name: "s3-sink"}]};
|
||||
if (command instanceof ListConnectorsCommand) return {connectors: [{connectorName: "archive"}]};
|
||||
return {connectorArn: "arn:aws:kafkaconnect:local:connector/archive"};
|
||||
}} as unknown as KafkaConnectClient;
|
||||
const service = createMskConnectService(client);
|
||||
|
||||
expect(await service.listCustomPlugins()).toEqual([{name: "s3-sink"}]);
|
||||
expect(await service.listConnectors()).toEqual([{connectorName: "archive"}]);
|
||||
await service.createCustomPlugin({name: "s3-sink", contentType: "JAR", location: {s3Location: {bucketArn: "arn:aws:s3:::plugins", fileKey: "s3.zip"}}});
|
||||
await service.createConnector({connectorName: "archive", capacity: {provisionedCapacity: {mcuCount: 1, workerCount: 1}}, connectorConfiguration: {topics: "orders"}, kafkaCluster: {apacheKafkaCluster: {bootstrapServers: "broker:9092", vpc: {securityGroups: ["sg-local"], subnets: ["subnet-local"]}}}, kafkaClusterClientAuthentication: {authenticationType: "NONE"}, kafkaClusterEncryptionInTransit: {encryptionType: "PLAINTEXT"}, kafkaConnectVersion: "3.7.x", plugins: [{customPlugin: {customPluginArn: "arn:plugin", revision: 1}}], serviceExecutionRoleArn: "arn:role"});
|
||||
await service.describeConnector("arn:connector");
|
||||
await service.updateConnector({connectorArn: "arn:connector", currentVersion: "1", connectorConfiguration: {topics: "events"}});
|
||||
await service.deleteConnector("arn:connector");
|
||||
|
||||
expect(commands[0]).toBeInstanceOf(ListCustomPluginsCommand);
|
||||
expect(commands[1]).toBeInstanceOf(ListConnectorsCommand);
|
||||
expect(commands[2]).toBeInstanceOf(CreateCustomPluginCommand);
|
||||
expect((commands[2] as CreateCustomPluginCommand).input).toMatchObject({contentType: "JAR", location: {s3Location: {fileKey: "s3.zip"}}});
|
||||
expect(commands[3]).toBeInstanceOf(CreateConnectorCommand);
|
||||
expect((commands[3] as CreateConnectorCommand).input).toMatchObject({kafkaClusterClientAuthentication: {authenticationType: "NONE"}, plugins: [{customPlugin: {revision: 1}}]});
|
||||
expect(commands[4]).toBeInstanceOf(DescribeConnectorCommand);
|
||||
expect(commands[5]).toBeInstanceOf(UpdateConnectorCommand);
|
||||
expect(commands[6]).toBeInstanceOf(DeleteConnectorCommand);
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import {
|
||||
CreateConnectorCommand,
|
||||
CreateCustomPluginCommand,
|
||||
DeleteConnectorCommand,
|
||||
DeleteCustomPluginCommand,
|
||||
DescribeConnectorCommand,
|
||||
KafkaConnectClient,
|
||||
ListConnectorsCommand,
|
||||
ListCustomPluginsCommand,
|
||||
UpdateConnectorCommand,
|
||||
type CreateConnectorRequest,
|
||||
type CreateCustomPluginRequest,
|
||||
type UpdateConnectorRequest,
|
||||
} from "@aws-sdk/client-kafkaconnect";
|
||||
import { awsClients } from "../aws";
|
||||
|
||||
export function createMskConnectService(client: KafkaConnectClient = awsClients.kafkaConnect) {
|
||||
return {
|
||||
async listCustomPlugins() {
|
||||
const plugins = [];
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const response = await client.send(new ListCustomPluginsCommand({ nextToken }));
|
||||
plugins.push(...(response.customPlugins ?? []));
|
||||
nextToken = response.nextToken;
|
||||
} while (nextToken);
|
||||
return plugins;
|
||||
},
|
||||
|
||||
createCustomPlugin(input: CreateCustomPluginRequest) {
|
||||
return client.send(new CreateCustomPluginCommand(input));
|
||||
},
|
||||
|
||||
deleteCustomPlugin(customPluginArn: string) {
|
||||
return client.send(new DeleteCustomPluginCommand({ customPluginArn }));
|
||||
},
|
||||
|
||||
async listConnectors() {
|
||||
const connectors = [];
|
||||
let nextToken: string | undefined;
|
||||
do {
|
||||
const response = await client.send(new ListConnectorsCommand({ nextToken }));
|
||||
connectors.push(...(response.connectors ?? []));
|
||||
nextToken = response.nextToken;
|
||||
} while (nextToken);
|
||||
return connectors;
|
||||
},
|
||||
|
||||
async describeConnector(connectorArn: string) {
|
||||
return client.send(new DescribeConnectorCommand({ connectorArn }));
|
||||
},
|
||||
|
||||
createConnector(input: CreateConnectorRequest) {
|
||||
return client.send(new CreateConnectorCommand(input));
|
||||
},
|
||||
|
||||
deleteConnector(connectorArn: string) {
|
||||
return client.send(new DeleteConnectorCommand({ connectorArn }));
|
||||
},
|
||||
|
||||
updateConnector(input: UpdateConnectorRequest) {
|
||||
return client.send(new UpdateConnectorCommand(input));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const mskConnectService = createMskConnectService();
|
||||
@@ -0,0 +1,95 @@
|
||||
import {describe, expect, test} from "bun:test";
|
||||
import {ListMetricsCommand, type CloudWatchClient} from "@aws-sdk/client-cloudwatch";
|
||||
import {ListEventBusesCommand, type EventBridgeClient} from "@aws-sdk/client-eventbridge";
|
||||
import {
|
||||
CreateConfigurationSetEventDestinationCommand,
|
||||
CreateEmailIdentityCommand,
|
||||
DeleteEmailTemplateCommand,
|
||||
ListSuppressedDestinationsCommand,
|
||||
SendEmailCommand,
|
||||
TestRenderEmailTemplateCommand,
|
||||
type SESv2Client,
|
||||
} from "@aws-sdk/client-sesv2";
|
||||
import {ListTopicsCommand, type SNSClient} from "@aws-sdk/client-sns";
|
||||
import {createSesService} from "./ses";
|
||||
|
||||
describe("sesService", () => {
|
||||
test("uses SESv2 identity, email, template, and event destination command contracts", async () => {
|
||||
const commands: unknown[] = [];
|
||||
const sesv2 = {
|
||||
send: async (command: unknown) => {
|
||||
commands.push(command);
|
||||
if (command instanceof ListSuppressedDestinationsCommand) return {SuppressedDestinationSummaries: []};
|
||||
return {MessageId: "message-1"};
|
||||
},
|
||||
} as unknown as SESv2Client;
|
||||
const service = createSesService(sesv2, {} as SNSClient, {} as EventBridgeClient, {} as CloudWatchClient);
|
||||
|
||||
await service.createIdentity({EmailIdentity: "sender@example.test"});
|
||||
await service.sendSimple({
|
||||
FromEmailAddress: "sender@example.test",
|
||||
Destination: {ToAddresses: ["recipient@example.test"]},
|
||||
Content: {Simple: {Subject: {Data: "Hello"}, Body: {Text: {Data: "Hi"}}}},
|
||||
});
|
||||
await service.sendRaw({
|
||||
FromEmailAddress: "sender@example.test",
|
||||
Destination: {ToAddresses: ["recipient@example.test"]},
|
||||
Content: {Raw: {Data: "U3ViamVjdDogSGVsbG8="}},
|
||||
});
|
||||
await service.testRenderTemplate({TemplateName: "welcome", TemplateData: "{\"name\":\"Ada\"}"});
|
||||
await service.createEventDestination("delivery-events", {
|
||||
EventDestinationName: "to-events",
|
||||
EventDestination: {Enabled: true, MatchingEventTypes: ["SEND"], EventBridgeDestination: {EventBusArn: "arn:aws:events:us-east-1:000000000000:event-bus/default"}},
|
||||
});
|
||||
await service.deleteTemplate("welcome");
|
||||
|
||||
expect(commands[0]).toBeInstanceOf(CreateEmailIdentityCommand);
|
||||
expect((commands[0] as CreateEmailIdentityCommand).input).toEqual({EmailIdentity: "sender@example.test"});
|
||||
expect(commands[1]).toBeInstanceOf(SendEmailCommand);
|
||||
expect((commands[2] as SendEmailCommand).input.Content?.Raw?.Data).toEqual(Uint8Array.from([83, 117, 98, 106, 101, 99, 116, 58, 32, 72, 101, 108, 108, 111]));
|
||||
expect(commands[3]).toBeInstanceOf(TestRenderEmailTemplateCommand);
|
||||
expect(commands[4]).toBeInstanceOf(CreateConfigurationSetEventDestinationCommand);
|
||||
expect((commands[4] as CreateConfigurationSetEventDestinationCommand).input).toMatchObject({ConfigurationSetName: "delivery-events", EventDestinationName: "to-events"});
|
||||
expect(commands[5]).toBeInstanceOf(DeleteEmailTemplateCommand);
|
||||
});
|
||||
|
||||
test("lists live destination inventory with the AWS SDK", async () => {
|
||||
const snsCommands: unknown[] = [];
|
||||
const eventBridgeCommands: unknown[] = [];
|
||||
const cloudWatchCommands: unknown[] = [];
|
||||
const service = createSesService(
|
||||
{} as SESv2Client,
|
||||
{send: async (command: unknown) => {
|
||||
snsCommands.push(command);
|
||||
return {Topics: [{TopicArn: "arn:aws:sns:us-east-1:000000000000:delivery"}]};
|
||||
}} as unknown as SNSClient,
|
||||
{send: async (command: unknown) => {
|
||||
eventBridgeCommands.push(command);
|
||||
return {EventBuses: [{Name: "default"}]};
|
||||
}} as unknown as EventBridgeClient,
|
||||
{send: async (command: unknown) => {
|
||||
cloudWatchCommands.push(command);
|
||||
return {Metrics: [{MetricName: "Send"}]};
|
||||
}} as unknown as CloudWatchClient,
|
||||
);
|
||||
|
||||
expect(await service.listSnsTopics()).toEqual([{TopicArn: "arn:aws:sns:us-east-1:000000000000:delivery"}]);
|
||||
expect(await service.listEventBuses()).toEqual([{Name: "default"}]);
|
||||
expect(await service.listSesMetrics()).toEqual([{MetricName: "Send"}]);
|
||||
|
||||
expect(snsCommands[0]).toBeInstanceOf(ListTopicsCommand);
|
||||
expect(eventBridgeCommands[0]).toBeInstanceOf(ListEventBusesCommand);
|
||||
expect((cloudWatchCommands[0] as ListMetricsCommand).input).toEqual({Namespace: "AWS/SES", NextToken: undefined});
|
||||
});
|
||||
|
||||
test("normalizes Floci configuration-set names for the console", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = (async () => new Response(JSON.stringify({ConfigurationSets: ["mail-events"]}), {headers: {"content-type": "application/json"}})) as unknown as typeof fetch;
|
||||
try {
|
||||
const service = createSesService({} as SESv2Client, {} as SNSClient, {} as EventBridgeClient, {} as CloudWatchClient);
|
||||
expect(await service.listConfigurationSets()).toEqual([{ConfigurationSetName: "mail-events"}]);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,209 @@
|
||||
import {CloudWatchClient, ListMetricsCommand} from "@aws-sdk/client-cloudwatch";
|
||||
import {EventBridgeClient, ListEventBusesCommand} from "@aws-sdk/client-eventbridge";
|
||||
import {
|
||||
CreateConfigurationSetCommand,
|
||||
CreateConfigurationSetEventDestinationCommand,
|
||||
CreateEmailIdentityCommand,
|
||||
CreateEmailTemplateCommand,
|
||||
DeleteConfigurationSetCommand,
|
||||
DeleteConfigurationSetEventDestinationCommand,
|
||||
DeleteEmailIdentityCommand,
|
||||
DeleteEmailTemplateCommand,
|
||||
DeleteSuppressedDestinationCommand,
|
||||
GetConfigurationSetEventDestinationsCommand,
|
||||
GetEmailTemplateCommand,
|
||||
ListEmailTemplatesCommand,
|
||||
ListSuppressedDestinationsCommand,
|
||||
PutSuppressedDestinationCommand,
|
||||
SESv2Client,
|
||||
SendEmailCommand,
|
||||
TestRenderEmailTemplateCommand,
|
||||
UpdateEmailTemplateCommand,
|
||||
type CreateConfigurationSetCommandInput,
|
||||
type CreateConfigurationSetEventDestinationCommandInput,
|
||||
type CreateEmailIdentityCommandInput,
|
||||
type CreateEmailTemplateCommandInput,
|
||||
type PutSuppressedDestinationCommandInput,
|
||||
type SendEmailCommandInput,
|
||||
type TestRenderEmailTemplateCommandInput,
|
||||
type UpdateEmailTemplateCommandInput,
|
||||
} from "@aws-sdk/client-sesv2";
|
||||
import {ListTopicsCommand, SNSClient, type Topic} from "@aws-sdk/client-sns";
|
||||
import {awsClients} from "../aws";
|
||||
|
||||
export type RawEmailInput = Omit<SendEmailCommandInput, "Content"> & {
|
||||
Content: {Raw: {Data: string}};
|
||||
};
|
||||
|
||||
export function createSesService(
|
||||
sesv2: SESv2Client = awsClients.sesv2,
|
||||
sns: SNSClient = awsClients.sns,
|
||||
eventbridge: EventBridgeClient = awsClients.eventbridge,
|
||||
cloudwatch: CloudWatchClient = awsClients.cloudwatch,
|
||||
) {
|
||||
return {
|
||||
async listIdentities() {
|
||||
const identities = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await sesRestRequest<{EmailIdentities?: unknown[]; NextToken?: string}>("/v2/email/identities", NextToken);
|
||||
identities.push(...(response.EmailIdentities ?? []));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return identities;
|
||||
},
|
||||
|
||||
createIdentity(input: CreateEmailIdentityCommandInput) {
|
||||
return sesv2.send(new CreateEmailIdentityCommand(input));
|
||||
},
|
||||
|
||||
deleteIdentity(EmailIdentity: string) {
|
||||
return sesv2.send(new DeleteEmailIdentityCommand({EmailIdentity}));
|
||||
},
|
||||
|
||||
sendSimple(input: SendEmailCommandInput) {
|
||||
return sesv2.send(new SendEmailCommand(input));
|
||||
},
|
||||
|
||||
sendRaw(input: RawEmailInput) {
|
||||
return sesv2.send(new SendEmailCommand({
|
||||
...input,
|
||||
Content: {Raw: {Data: Uint8Array.fromBase64(input.Content.Raw.Data)}},
|
||||
}));
|
||||
},
|
||||
|
||||
sendTemplate(input: SendEmailCommandInput) {
|
||||
return sesv2.send(new SendEmailCommand(input));
|
||||
},
|
||||
|
||||
async listSuppressedDestinations() {
|
||||
const destinations = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await sesv2.send(new ListSuppressedDestinationsCommand({NextToken}));
|
||||
destinations.push(...(response.SuppressedDestinationSummaries ?? []));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return destinations;
|
||||
},
|
||||
|
||||
createSuppression(input: PutSuppressedDestinationCommandInput) {
|
||||
return sesv2.send(new PutSuppressedDestinationCommand(input));
|
||||
},
|
||||
|
||||
deleteSuppression(EmailAddress: string) {
|
||||
return sesv2.send(new DeleteSuppressedDestinationCommand({EmailAddress}));
|
||||
},
|
||||
|
||||
async listTemplates() {
|
||||
const templates = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await sesv2.send(new ListEmailTemplatesCommand({NextToken}));
|
||||
templates.push(...(response.TemplatesMetadata ?? []));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return templates;
|
||||
},
|
||||
|
||||
createTemplate(input: CreateEmailTemplateCommandInput) {
|
||||
return sesv2.send(new CreateEmailTemplateCommand(input));
|
||||
},
|
||||
|
||||
updateTemplate(TemplateName: string, input: Omit<UpdateEmailTemplateCommandInput, "TemplateName">) {
|
||||
return sesv2.send(new UpdateEmailTemplateCommand({...input, TemplateName}));
|
||||
},
|
||||
|
||||
deleteTemplate(TemplateName: string) {
|
||||
return sesv2.send(new DeleteEmailTemplateCommand({TemplateName}));
|
||||
},
|
||||
|
||||
getTemplate(TemplateName: string) {
|
||||
return sesv2.send(new GetEmailTemplateCommand({TemplateName}));
|
||||
},
|
||||
|
||||
testRenderTemplate(input: TestRenderEmailTemplateCommandInput) {
|
||||
return sesv2.send(new TestRenderEmailTemplateCommand(input));
|
||||
},
|
||||
|
||||
async listConfigurationSets() {
|
||||
const configurationSets = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await sesRestRequest<{ConfigurationSets?: unknown[]; NextToken?: string}>("/v2/email/configuration-sets", NextToken);
|
||||
configurationSets.push(...(response.ConfigurationSets ?? []).map((configurationSet) => typeof configurationSet === "string" ? {ConfigurationSetName: configurationSet} : configurationSet));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return configurationSets;
|
||||
},
|
||||
|
||||
createConfigurationSet(input: CreateConfigurationSetCommandInput) {
|
||||
return sesv2.send(new CreateConfigurationSetCommand(input));
|
||||
},
|
||||
|
||||
deleteConfigurationSet(ConfigurationSetName: string) {
|
||||
return sesv2.send(new DeleteConfigurationSetCommand({ConfigurationSetName}));
|
||||
},
|
||||
|
||||
async listEventDestinations(ConfigurationSetName: string) {
|
||||
const response = await sesv2.send(new GetConfigurationSetEventDestinationsCommand({ConfigurationSetName}));
|
||||
return response.EventDestinations ?? [];
|
||||
},
|
||||
|
||||
createEventDestination(ConfigurationSetName: string, input: Omit<CreateConfigurationSetEventDestinationCommandInput, "ConfigurationSetName">) {
|
||||
return sesv2.send(new CreateConfigurationSetEventDestinationCommand({...input, ConfigurationSetName}));
|
||||
},
|
||||
|
||||
deleteEventDestination(ConfigurationSetName: string, EventDestinationName: string) {
|
||||
return sesv2.send(new DeleteConfigurationSetEventDestinationCommand({ConfigurationSetName, EventDestinationName}));
|
||||
},
|
||||
|
||||
async listSnsTopics() {
|
||||
const topics: Topic[] = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await sns.send(new ListTopicsCommand({NextToken}));
|
||||
topics.push(...(response.Topics ?? []));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return topics;
|
||||
},
|
||||
|
||||
async listEventBuses() {
|
||||
const eventBuses = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await eventbridge.send(new ListEventBusesCommand({NextToken}));
|
||||
eventBuses.push(...(response.EventBuses ?? []));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return eventBuses;
|
||||
},
|
||||
|
||||
async listSesMetrics() {
|
||||
const metrics = [];
|
||||
let NextToken: string | undefined;
|
||||
do {
|
||||
const response = await cloudwatch.send(new ListMetricsCommand({Namespace: "AWS/SES", NextToken}));
|
||||
metrics.push(...(response.Metrics ?? []));
|
||||
NextToken = response.NextToken;
|
||||
} while (NextToken);
|
||||
return metrics;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const sesService = createSesService();
|
||||
|
||||
async function sesRestRequest<T>(path: string, nextToken?: string): Promise<T> {
|
||||
const endpoint = process.env.FLOCI_ENDPOINT ?? "http://localhost:4566";
|
||||
const url = new URL(path, endpoint);
|
||||
if (nextToken) {
|
||||
url.searchParams.set("NextToken", nextToken);
|
||||
}
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`SES REST request failed with HTTP ${response.status}`);
|
||||
}
|
||||
return response.json() as Promise<T>;
|
||||
}
|
||||
Reference in New Issue
Block a user