Designer Cloud and EMR Serverless in AWS

Follow this guide to deploy the Designer Cloud module for AWS private data processing.

Prerequisite

Before you deploy the Designer Cloud module, you must complete these steps on the Set Up AWS Account and VPC for Private Data page...

VPC dedicated to Alteryx Analytics Cloud (AAC) has been configured as mentioned in Create a VPC section.
Service account and base IAM policy attached to the service account as mentioned in the Configure IAM section.
Successfully triggered private data processing provisioning as mentioned in the Trigger Private Data Handling Provisioning section.

Account Setup

Step 1: Configure IAM

Step 1a: Create Designer Cloud IAM Policy

You need to create a custom IAM policy. Name it AAC_DesignerCloud_SA_Policy and attach the following policy document. We recommend using the JSON tab instead of the visual editor. AACAAC requires some * permissions to run. Expect some security warnings when you create the policy.

    {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": "iam:PassRole",
            "Resource": "arn:aws:iam::*:role/*",
            "Condition": {
                "StringEqualsIfExists": {
                    "iam:PassedToService": [
                        "ec2.amazonaws.com",
                        "ec2.amazonaws.com.cn"
                    ]
                }
            }
        },
        {
            "Sid": "VisualEditor1",
            "Effect": "Allow",
            "Action": [
                "eks:*",
                "iam:CreateServiceLinkedRole",
                "kms:CreateGrant",
                "kms:Decrypt",
                "kms:DescribeKey",
                "kms:Encrypt",
                "kms:GetKeyPolicy",
                "kms:GetKeyRotationStatus",
                "kms:ListGrants",
                "kms:ListResourceTags",
                "kms:ListRetirableGrants",
                "kms:PutKeyPolicy",
                "kms:RetireGrant",
                "kms:RevokeGrant",
                "kms:ScheduleKeyDeletion",
                "kms:TagResource",
                "kms:UntagResource"
            ],
            "Resource": [
                "arn:aws:eks:*:*:addon/*/*/*",
                "arn:aws:eks:*:*:cluster/*",
                "arn:aws:eks:*:*:nodegroup/*/*/*",
                "arn:aws:eks:*:*:identityproviderconfig/*/*/*/*",
                "arn:aws:eks:*:*:access-entry/*/*/*",
                "arn:aws:kms:*:*:key/*",
                "arn:aws:iam::*:role/*"
            ]
        },
        {
            "Sid": "VisualEditor2",
            "Effect": "Allow",
            "Action": [
                "iam:AttachRolePolicy",
                "iam:CreateOpenIDConnectProvider",
                "iam:CreatePolicy",
                "iam:CreatePolicyVersion",
                "iam:CreateRole",
                "iam:DeleteOpenIDConnectProvider",
                "iam:DeletePolicy",
                "iam:DeletePolicyVersion",
                "iam:DeleteRole",
                "iam:DeleteRolePolicy",
                "iam:DetachRolePolicy",
                "iam:GetOpenIDConnectProvider",
                "iam:GetPolicy",
                "iam:GetPolicyVersion",
                "iam:GetRole",
                "iam:GetRolePolicy",
                "iam:GetUser",
                "iam:GetUserPolicy",
                "iam:ListAttachedRolePolicies",
                "iam:ListAttachedUserPolicies",
                "iam:ListGroupsForUser",
                "iam:ListInstanceProfilesForRole",
                "iam:ListPolicyTags",
                "iam:ListPolicyVersions",
                "iam:ListRolePolicies",
                "iam:PassRole",
                "iam:PutRolePolicy",
                "iam:TagOpenIDConnectProvider",
                "iam:TagPolicy",
                "iam:TagRole",
                "iam:UntagOpenIDConnectProvider",
                "iam:UntagPolicy",
                "iam:UntagRole",
                "iam:UpdateOpenIDConnectProviderThumbprint",
                "iam:UpdateRole",
                "iam:UpdateAssumeRolePolicy"
            ],
            "Resource": [
                "arn:aws:iam::*:policy/*",
                "arn:aws:iam::*:oidc-provider/*",
                "arn:aws:iam::*:user/*",
                "arn:aws:iam::*:role/*"
            ]
        },
        {
            "Sid": "VisualEditor3",
            "Effect": "Allow",
            "Action": [
                "autoscaling:*",
                "ec2:*",
                "eks:CreateCluster",
                "eks:ListClusters",
                "elasticloadbalancing:*",
                "iam:GetAccountName",
                "iam:ListAccountAliases",
                "iam:ListRoles",
                "iam:CreateInstanceProfile",
                "iam:DeleteInstanceProfile",
                "iam:GetInstanceProfile",
                "iam:TagInstanceProfile",
                "iam:UntagInstanceProfile", 
                "iam:RemoveRoleFromInstanceProfile", 
                "iam:AddRoleToInstanceProfile", 
                "kms:CreateKey",
                "logs:CreateLogGroup",
                "logs:DeleteLogGroup",
                "logs:DescribeLogGroups",
                "logs:ListTagsLogGroup",
                "logs:PutRetentionPolicy",
                "logs:TagResource",
                "logs:UntagResource",
                "logs:TagLogGroup",
                "logs:UntagLogGroup",
                "logs:ListTagsForResource",
                "networkmanager:Describe*",
                "networkmanager:Get*",
                "networkmanager:List*",
                "s3:CreateBucket",
                "s3:DeleteBucket",
                "s3:DeleteBucketPolicy",
                "s3:DeleteBucketWebsite",
                "s3:DeleteObject",
                "s3:DeleteObjectVersion",
                "s3:DeleteObjectVersionTagging",
                "s3:GetAccelerateConfiguration",
                "s3:GetBucketAcl",
                "s3:GetBucketCORS",
                "s3:GetBucketLocation",
                "s3:GetBucketLogging",
                "s3:GetBucketObjectLockConfiguration",
                "s3:GetBucketOwnershipControls",
                "s3:GetBucketPolicy",
                "s3:GetBucketPolicyStatus",
                "s3:GetBucketPublicAccessBlock",
                "s3:GetBucketRequestPayment",
                "s3:GetBucketTagging",
                "s3:GetBucketVersioning",
                "s3:GetBucketWebsite",
                "s3:GetEncryptionConfiguration",
                "s3:GetLifecycleConfiguration",
                "s3:GetObject",
                "s3:GetObjectAcl",
                "s3:GetObjectVersion",
                "s3:GetObjectVersionAcl",
                "s3:GetObjectVersionAttributes",
                "s3:GetObjectVersionForReplication",
                "s3:GetObjectVersionTagging",
                "s3:GetObjectVersionTorrent",
                "s3:GetReplicationConfiguration",
                "s3:ListAllMyBuckets",
                "s3:ListBucket",
                "s3:ListBucketVersions",
                "s3:PutAccelerateConfiguration",
                "s3:PutBucketAcl",
                "s3:PutBucketCORS",
                "s3:PutBucketLogging",
                "s3:PutBucketObjectLockConfiguration",
                "s3:PutBucketOwnershipControls",
                "s3:PutBucketPolicy",
                "s3:PutBucketPublicAccessBlock",
                "s3:PutBucketRequestPayment",
                "s3:PutBucketTagging",
                "s3:PutBucketVersioning",
                "s3:PutBucketWebsite",
                "s3:PutEncryptionConfiguration",
                "s3:PutLifecycleConfiguration",
                "s3:PutObject",
                "s3:PutObjectAcl",
                "s3:PutObjectVersionAcl",
                "s3:PutObjectVersionTagging",
                "sts:GetCallerIdentity",
                "memorydb:CreateSubnetGroup",
                "memorydb:CreateUser",
                "memorydb:CreateAcl",
                "memorydb:CreateCluster",
                "memorydb:TagResource",
                "memorydb:DescribeSubnetGroups",
                "memorydb:DescribeUsers",
                "memorydb:DescribeACLs",
                "memorydb:DescribeClusters",
                "memorydb:ListTags",
                "memorydb:DeleteUser",
                "memorydb:DeleteSubnetGroup",
                "memorydb:DeleteAcl",
                "memorydb:DeleteCluster",
                "memorydb:UpdateAcl",
                "memorydb:UpdateCluster",
                "memorydb:UpdateSubnetGroup",
                "memorydb:UpdateUser"
            ],
            "Resource": "*"
        },
        {
            "Sid": "VisualEditor4",
            "Effect": "Allow",
            "Action": "secretsmanager:*",
            "Resource": "arn:aws:secretsmanager:*:*:secret:*"
        }
    ]
}

Step 1b: Tag the IAM Policy

Tag the custom IAM policy created in Step 1a.

Tag Name	Value
AACResource	aac_sa_custom_policy

Step 1c: Attach IAM Policy

Attach the AAC_DesignerCloud_SA_Policy IAM policy to the aac_automation_sa service account created on the Set Up AWS Account and VPC for Private Data page.

注記

AAC_DesignerCloud_SA_Policy is an example policy name. You can choose any name for the policy, but the name must start with AAC_DesignerCloud.

Step 2: Configure Subnet

注記

If you have purchased Designer Cloud and Machine Learning, then configure the subnets as mentioned in Designer Cloud document. Both Designer Cloud and Machine Learning resources share the same subnets.

Designer Cloud in a private data processing environment requires up to 5 subnet groups. Each group contains 3 individual subnets, each in a different availability zone.

eks_control group (required): The EKS control plane uses this subnet to accept incoming job execution requests.
eks_node group (required): The EKS cluster uses this subnet to execute Alteryx software jobs (for example, connectivity, conversion, processing, and publishing).
public group (required): This group doesn’t run any services but the eks_node group uses it for egress out of the cluster.
private group (required): This group runs services private to the private data processing.

Used for EMR

option group (optional): Use this group if you enable EMR processing within your private data processing environment. EMR services do not run in the cluster, but the IP space is needed to interact with the AWS Serverless EMR endpoints.

Step 2a: Create Subnets in the VPC

Configure subnets in the aac_vpc VPC.

Create subnets and tag them following this example. Modify values, as needed, to meet your network architecture…

CIDRs	Subnet Name	Subnet	AZ	Tag Name	Tag Value	Note
10.64.0.0/18	eks_node	10.64.0.0/21	AZa	AACSubnet	eks_node
	eks_node	10.64.8.0/21	AZb	AACSubnet	eks_node
	eks_node	10.64.16.0/21	AZc	AACSubnet	eks_node
		10.64.24.0/21				SPARE
		10.64.32.0/19				SPARE (Can be configured for blue/green upgrade later)
10.10.0.0/21	eks_control	10.10.0.0/27	AZa	AACSubnet	eks_control
	eks_control	10.10.0.32/27	AZb	AACSubnet	eks_control
	eks_control	10.10.0.64/27	AZc	AACSubnet	eks_control
		10.10.0.96/27				SPARE
	public	10.10.0.128/27	AZa	AACSubnet	public
	public	10.10.0.160/27	AZb	AACSubnet	public
	public	10.10.0.192/27	AZc	AACSubnet	public
		10.10.0.224/27				SPARE
	private	10.10.1.0/25	AZa	AACSubnet	private
	private	10.10.1.128/25	AZb	AACSubnet	private
	private	10.10.2.0/25	AZc	AACSubnet	private
		10.10.1.128/25				SPARE
	option	10.10.4.0/24	AZa	AACSubnet	option
	option	10.10.5.0/24	AZa	AACSubnet	option
	option	10.10.6.0/24	AZa	AACSubnet	option
		10.10.7.0/24				SPARE

重要

You must tag subnets with Tag Name and Tag Value as mentioned in the table.

Step 2b: Subnet Route Tables

Create the route table for your subnets.

注記

This route table is an example.

Subnet Name	Route Destination	Target	Comments
eks_node	/18 CIDR Block /21 CIDR Block <s3 prefix id> 0.0.0.0/0	Local Local <vpce endpoint id> <gateway id>	Configure the same routes to all 3 AZs subnet routing tables.
eks_control	/18 CIDR Block /21 CIDR Block <s3 prefix id> 0.0.0.0/0	Local Local <vpce endpoint id> <gateway id>	Configure the same routes to all 3 AZs subnet routing tables.
public	/18 CIDR Block /21 CIDR Block 0.0.0.0/0	Local Local <internet gateway id>	Configure the same routes to all 3 AZs subnet routing tables.
private	/18 CIDR Block /21 CIDR Block <s3 prefix id> 0.0.0.0/0	Local Local <vpce endpoint id> <gateway id>	Configure the same routes to all 3 AZs subnet routing tables. 0.0.0.0/0 should be egressing out to the public network.
option	/18 CIDR Block /21 CIDR Block <s3 prefix id> 0.0.0.0/0	local local <vpce endpoint id> <gateway id>	Configure the same routes to all 3 AZs subnet routing tables. 0.0.0.0/0 should be egressing out to the public network.

注記

Your <gateway id> could be either a zonal NAT gateway that is created per AZ or a transit gateway, depending on your network architecture. If NAT gateway, create NAT gateway per AZ for public subnets.

Step 3: Quota Adjustment

Your private data processing environment requires a quota increase on these services. Adjust the Applied quota value numbers as follows:

Amazon EC2

Quota Name: Running On-Demand Standard (A, C, D, H, I, M, R, T, Z) instances.
Quota Description: Maximum number of vCPUs assigned to the Running On-Demand Standard (A, C, D, H, I, M, R, T, Z) instances.
AWS default quota value: 5
Applied quota value: 2500

Amazon EMR Serverless

Quota Name: Max concurrent vCPUs per account.
Quota Description: Maximum number of vCPUs that can be concurrently run in this account in the current Region.
AWS default quota value: 16
Applied quota value: 1024

Request Quota Increase

Sign in to the AWS account console.
Search for Service Quotas and select the service.
Select AWS Service from left navigation pane.
Search for the service (for example, Amazon EMR Serverless or Amazon EC2).
Select the quota name.
Select Request quota increase.
Request the specified quota increase.

Egress

Host	Port	Protocol	Purpose
us.app.unleash-hosted.com	443	HTTPS	Retrieve feature flags from Unleash.

Private Data Processing

注意

プライベートデータ処理がプロビジョニングされた後で、 AACプロビジョニングされたパブリッククラウドリソースのいずれかの変更または削除をすると、一貫性のない状態になります。この一貫性のない状態は、ジョブの実行中またはプライベートデータプレーン処理設定のプロビジョニング解除中に、エラーを発生させます。

Step 1: Trigger Designer Cloud Deployment

Designer Cloud provisioning triggers from the Admin Console inside AACAAC. You need Workspace Admin privileges within a workspace in order to see it.

From the AACAAC landing page, select the Profile menu and then select Workspace Admin.
From the Admin Console, select Private Data Handling and then select Processing.
Select the Designer Cloud checkbox and then select Update.

Selecting Update triggers the deployment of the cluster and resources in the AWS account. This runs a set of validation checks to verify the correct configuration of the AWS account.

注記

The provisioning process takes approximately 35–40 minutes to complete.

After the provisioning completes, you can view the created resources (for example, EC2 instances and node groups) through the AWS console. It is very important that you don't modify them on your own. Manual changes might cause issues with the function of the private data processing.

Step 2: Append Custom Role’s Trust Relationship

注記

This step is only necessary if you used a cross-account role for permissions when you configured private data storage. If you used an access key for that step, you can skip this step.

重要

You must wait for the successful completion of Step 1 before you proceed with this step.

If your private data storage uses a cross-account role, then in order for your new private data processing environment to be able to read/write from your private data storage, you need to update that role to append a trust relationship with your new Kubernetes cluster role:

{
    "Sid": "",
    "Effect": "Allow",
    "Principal": {
         "AWS": "arn:aws:iam::<accountid>:role/aac-<xxxx-xxxxxxxxxxxx>-cluster-role"
    },
    "Action": "sts:AssumeRole"
}

注記

Replace AWS Principal with the ARN of the IAM role created by the private data handling provisioning process.

<accountid>: AWS account number where private data processing environment handling has been provisioned.

<xxxx-xxxxxxxxxxxx>: Last 2 segments of Private Data Processing Environment ID. You can locate this ID in the Admin UI after the private data processing environment has been successfully provisioned.

Example Scenario:

Account ID: 123456789012

Private Data Processing Environment ID: b2a65fbd-95dc-490a-b69b-a1dc92df224e

Role ARN: arn:aws:iam::123456789012:role/aac-b69b-a1dc92df224e-cluster-role

For more information, go to https://docs.aws.amazon.com/directoryservice/latest/admin-guide/edit_trust.html.

Step 3: EMR Serverless (Optional)

Configure EMR serverless if you're using Spark/EMR processing.

Enable EMR

Sign in to AACAAC.
From the Profile menu, select Admin Console.
From the left navigation panel, select Private Data Handling and then select Processing.
Select Edit.
Select EMR and then select Update.

Update Custom Role Created for S3 Connection

Append the custom policy and custom role from AWS S3 as Private Data Storage with these permissions and trust relationships for EMR serverless:

Append Custom Policy Document

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "EMRServerlessAccess",
            "Effect": "Allow",
            "Action": [
                "emr-serverless:CreateApplication",
                "emr-serverless:UpdateApplication",
                "emr-serverless:DeleteApplication",
                "emr-serverless:ListApplications",
                "emr-serverless:GetApplication",
                "emr-serverless:StartApplication",
                "emr-serverless:StopApplication",
                "emr-serverless:StartJobRun",
                "emr-serverless:CancelJobRun",
                "emr-serverless:ListJobRuns",
                "emr-serverless:GetJobRun"
            ],
            "Resource": "*"
        },
        {
            "Sid": "AllowNetworkInterfaceCreationViaEMRServerless",
            "Effect": "Allow",
            "Action": "ec2:CreateNetworkInterface",
            "Resource": [
                "arn:aws:ec2:*:*:network-interface/*",
                "arn:aws:ec2:*:*:security-group/*",
                "arn:aws:ec2:*:*:subnet/*"
            ],
            "Condition": {
                "StringEquals": {
                    "aws:CalledViaLast": "ops.emr-serverless.amazonaws.com"
                }
            }
        },
        {
            "Sid":"AllowEMRServerlessServiceLinkedRoleCreation",
            "Effect":"Allow",
            "Action":"iam:CreateServiceLinkedRole",
            "Resource":"arn:aws:iam:::role/aws-service-role/ops.emr-serverless.amazonaws.com/AWSServiceRoleForAmazonEMRServerless"
        },
        {
            "Sid": "AllowPassingRuntimeRole",
            "Effect": "Allow",
            "Action": "iam:PassRole",
            "Resource": "arn:aws:iam:::role/aac--emr-serverless-spark-execution",
            "Condition": {
                "StringLike": {
                    "iam:PassedToService": "emr-serverless.amazonaws.com"
                }
            }
        },
        {
            "Sid": "S3ResourceBucketAccess",
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:DeleteObject"
            ],
            "Resource": [
                "arn:aws:s3:::aac--emr-logs",
                "arn:aws:s3:::aac--emr-logs/*"
            ]
        }
    ]
}

Append Custom Role's Trust Relationship

{
    "Sid": "",
    "Effect": "Allow",
    "Principal": {
        "AWS": "arn:aws:iam:::role/aac--emr-serverless-spark-execution"
    },
    "Action": "sts:AssumeRole"
},
{
    "Sid": "",
    "Effect": "Allow",
    "Principal": {
        "Service": "emr-serverless.amazonaws.com"
    },
    "Action": "sts:AssumeRole"
}

注記

When you delete Private Data Handling, AWS replaces the trust relationship of aac-<xxxx-xxxxxxxxxxxx>-cluster-role ARN with an access key. You must also delete the trust relationship from the UI.

注記