clickhouse-docs
clickhouse-docs copied to clipboard
Kinesis Clickpipe role-based access docs incomplete
The instructions in docs/integrations/data-ingestion/clickpipes/secure-kinesis.md have some gaps. It doesn't explicitly what choice the user is expected to make here:
Assuming the answer is "Custom trust policy" we then have to add two statements. The docs might be easier to follow here if a single JSON blob were provided instead of two distinct steps.
Example:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Statement1",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::191110999071:role/REDACTED-Role"
},
"Action": "sts:AssumeRole"
},
{
"Action": [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords",
"kinesis:ListShards",
"kinesis:SubscribeToShard",
"kinesis:DescribeStreamConsumer",
"kinesis:RegisterStreamConsumer",
"kinesis:DeregisterStreamConsumer",
"kinesis:ListStreamConsumers"
],
"Resource": [
"REDACTED"
],
"Effect": "Allow"
},
{
"Action": [
"kinesis:ListStreams"
],
"Resource": "*",
"Effect": "Allow"
}
]
}
Unfortunately, this results in the following validation errors in the console:
I eventually figured out that for "Trusted Entity Type," one needs to select "AWS Account."
Here's a CDK construct which makes this a lot easier:
import { CfnOutput, Stack } from "aws-cdk-lib";
import { Construct } from "constructs";
import * as iam from "aws-cdk-lib/aws-iam";
import * as kinesis from "aws-cdk-lib/aws-kinesis";
interface ClickHouseAccessRoleProps {
kinesisStreams: kinesis.IStream[];
/**
* Obtaining the ClickHouse service IAM Role Arn:
* 1. Login to your ClickHouse cloud account.
* 2. Select the ClickHouse service you want to create the integration
* 3. Select the Settings tab
* 4. Scroll down to the Network security information section at the bottom of the page
* 5. Copy the Service role ID (IAM) value belong to the service
*
* Pass the value as ```
* assumedBy: new iam.ArnPrincipal("arn:aws:iam::191110999071:role/EXAMPLE-Role")
* ```
*/
assumedBy: iam.IPrincipal;
}
// failed to describe consumer: operation error Kinesis: DescribeStreamConsumer, https response error StatusCode: 400, RequestID: f3ef7433-efc9-3a4d-93b4-cab15dcdbfd4, api error AccessDeniedException: User: arn:aws:sts::054112093691:assumed-role/ClickHouseAccessRole-packstats/clickpipes-b845bea20899a292e2af7652b0da5b7d is not authorized to perform: kinesis:DescribeStreamConsumer on resource: arn:aws:kinesis:us-east-1:054112093691:stream/packstats/consumer/clickpipes-2fae3dc0-89ef-4ece-97b1-52be5c72142d-0:1742847040 because no identity-based policy allows the kinesis:DescribeStreamConsumer action
export class ClickHouseAccessRole extends Construct {
public readonly roleArn: string;
constructor(scope: Construct, id: string, props: ClickHouseAccessRoleProps) {
super(scope, id);
const stack = Stack.of(this);
const role = new iam.Role(this, "Role", {
roleName: `ClickHouseAccessRole-${stack.stackName}`,
assumedBy: props.assumedBy,
inlinePolicies: {
kinesis: new iam.PolicyDocument({
statements: [
new iam.PolicyStatement({
effect: iam.Effect.ALLOW,
actions: [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords",
"kinesis:ListShards",
"kinesis:SubscribeToShard",
"kinesis:DescribeStreamConsumer",
"kinesis:RegisterStreamConsumer",
"kinesis:DeregisterStreamConsumer",
"kinesis:ListStreamConsumers",
],
resources: props.kinesisStreams.flatMap(stream => [stream.streamArn, `${stream.streamArn}/*`]),
}),
new iam.PolicyStatement({
effect: iam.Effect.ALLOW,
actions: ["kinesis:ListStreams"],
resources: ["*"],
}),
],
}),
},
});
new CfnOutput(this, "clickHouseAccessRole", { value: role.roleArn });
this.roleArn = role.roleArn;
}
}