# ARO Deployment Guide of Ultimate Winning ## Prerequisites - Azure CLI 2.84+ - Create a file called `rh-pull-secret.json` with your Red Hat Pull Secret: https://console.redhat.com/openshift/install/azure/aro-provisioned - Microsoft Documentation: https://learn.microsoft.com/en-us/azure/openshift/howto-create-openshift-cluster?pivots=aro-deploy-az-cli - Will deploy the latest version of OpenShift in Azure ```bash= #################################################################################### ## Set some Azure variables export SUBSCRIPTION_ID="some-long-string" ## Switch to the proper Azure Subscription ID export AZURE_LOCATION="centralus" ## Cluster Location ## Set a Cluster Name export ARO_CLUSTER_NAME="my-handsome-aro" # Configure Resource Groups export CREATE_RESOURCE_GROUPS="true" export RESOURCE_GROUP="${ARO_CLUSTER_NAME}-rg" ## ARO Resource Group Name export VNET_RESOURCE_GROUP="vnet-${RESOURCE_GROUP}" ## VNet RG export INFRASTRUCTURE_RESOURCE_GROUP="infra-${RESOURCE_GROUP}" # Infrastructure RG, do NOT pre-create, will be handled by ARM # Configure VNet export CREATE_VNET="true" export VNET_NAME="aro-vnet" ## Define the VNet Name export VNET_CIDR="10.42.0.0/16" ## Define the whole VNet CIDR Network ## Define the Subnet CIDRs and Names export VNET_CONTROL_PLANE_SUBNET_CIDR="10.42.0.0/23" export VNET_APP_NODE_SUBNET_CIDR="10.42.2.0/23" export VNET_CONTROL_PLANE_SUBNET_NAME="${ARO_CLUSTER_NAME}-cp-sn" export VNET_APP_NODE_SUBNET_NAME="${ARO_CLUSTER_NAME}-app-sn" # Configuration Options export CREATE_MANAGED_IDENTITIES="true" export POD_CIDR_SUBNET="100.80.0.0/14" # NOT 100.64.0.0/16 OR 100.88.0.0/16, USED BY OVN-K export SERVICE_CIDR_SUBNET="100.84.0.0/16" # NOT 100.64.0.0/16 OR 100.88.0.0/16, USED BY OVN-K export CLUSTER_EXPOSURE="Private" # Private or Public export WORKER_VM_SIZE="Standard_D4s_v5" #################################################################################### ## Start the creation process ## Set the proper subscription context az account set --subscription $SUBSCRIPTION_ID ## Enable the resources one by one az provider register -n Microsoft.RedHatOpenShift --wait az provider register -n Microsoft.Compute --wait az provider register -n Microsoft.Storage --wait az provider register -n Microsoft.Network --wait az provider register -n Microsoft.Authorization --wait az provider register -n Microsoft.ContainerRegistry --wait az provider register -n Microsoft.Quota --wait az provider register -n Microsoft.Subscription --wait #################################################################################### ## Resource Groups if [ $CREATE_RESOURCE_GROUPS = "true" ]; then echo "Creating Resource Groups..." ## Create the ARO Resource Group if [ $(az group exists -n $RESOURCE_GROUP) = "false" ]; then az group create --name $RESOURCE_GROUP --location $AZURE_LOCATION fi ## Create the VNet Resource Group if [ $(az group exists -n $VNET_RESOURCE_GROUP) = "false" ]; then az group create --name $VNET_RESOURCE_GROUP --location $AZURE_LOCATION fi # Validation Check # The Infrastructure Resource Group is Automatically Created, an RG name is optional and will be randomly generated otherwise if [ $(az group exists -n $INFRASTRUCTURE_RESOURCE_GROUP) = "false" ]; then echo " - PASS: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP does not exist!" else echo " - FAIL: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP already exists!" exit 1 fi else echo "Skipping Resource Group Creation..." echo "Using:" echo " - ARO Resource Group: $RESOURCE_GROUP" echo " - ARO VNet Resource Group: $VNET_RESOURCE_GROUP" echo " - ARO Infrastructure Resource Group: $INFRASTRUCTURE_RESOURCE_GROUP" echo "" echo "Checking for required Resource Groups..." if [ $(az group exists -n $RESOURCE_GROUP) = "false" ]; then echo " - FAIL: ARO Resource Group $RESOURCE_GROUP does not exist!" exit 1 else echo " - PASS: ARO Resource Group exists" fi if [ $(az group exists -n $VNET_RESOURCE_GROUP) = "false" ]; then echo " - FAIL: ARO Resource Group $VNET_RESOURCE_GROUP does not exist!" exit 1 else echo " - PASS: ARO VNet Resource Group exists" fi if [ $(az group exists -n $INFRASTRUCTURE_RESOURCE_GROUP) = "false" ]; then echo " - PASS: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP does not exist!" else echo " - FAIL: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP already exists!" exit 1 fi fi #################################################################################### ## Networking if [ $CREATE_VNET = "true" ]; then echo "Creating VNet and Subnets..." ## Create the VNet if [ $(az network vnet list --resource-group $VNET_RESOURCE_GROUP --query "[?contains(name, '"$VNET_NAME"')]" | jq -r 'length') = "0" ]; then az network vnet create \ --resource-group $VNET_RESOURCE_GROUP \ --location $AZURE_LOCATION \ --name $VNET_NAME \ --address-prefixes "${VNET_CIDR}" else echo "$VNET_NAME already exists!" fi ## Create the Control Plane Subnet if [ $(az network vnet subnet list -g $VNET_RESOURCE_GROUP --vnet-name $VNET_NAME --query "[?contains(name, '"$VNET_CONTROL_PLANE_SUBNET_NAME"')]" | jq -r 'length') = "0" ]; then az network vnet subnet create \ --resource-group $VNET_RESOURCE_GROUP \ --vnet-name $VNET_NAME \ --name $VNET_CONTROL_PLANE_SUBNET_NAME \ --address-prefixes "${VNET_CONTROL_PLANE_SUBNET_CIDR}" \ --service-endpoints Microsoft.ContainerRegistry else echo "Subnet $VNET_CONTROL_PLANE_SUBNET_NAME in $VNET_NAME already exists!" fi ## Create the Application Node Subnet if [ $(az network vnet subnet list -g $VNET_RESOURCE_GROUP --vnet-name $VNET_NAME --query "[?contains(name, '"$VNET_APP_NODE_SUBNET_NAME"')]" | jq -r 'length') = "0" ]; then az network vnet subnet create \ --resource-group $VNET_RESOURCE_GROUP \ --vnet-name $VNET_NAME \ --name $VNET_APP_NODE_SUBNET_NAME \ --address-prefixes "${VNET_APP_NODE_SUBNET_CIDR}" \ --service-endpoints Microsoft.ContainerRegistry else echo "Subnet $VNET_APP_NODE_SUBNET_NAME in $VNET_NAME already exists!" fi ## Disable subnet private endpoints az network vnet subnet update \ --name $VNET_CONTROL_PLANE_SUBNET_NAME \ --resource-group $VNET_RESOURCE_GROUP \ --vnet-name $VNET_NAME \ --private-link-service-network-policies Disabled fi #################################################################################### ## Create the Managed Identities if [ $CREATE_MANAGED_IDENTITIES = "true" ]; then az identity create --resource-group $RESOURCE_GROUP --name aro-cluster --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name cloud-controller-manager --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name ingress --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name machine-api --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name disk-csi-driver --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name cloud-network-config --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name image-registry --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name file-csi-driver --location $AZURE_LOCATION az identity create --resource-group $RESOURCE_GROUP --name aro-operator --location $AZURE_LOCATION fi ######################## ## Associate the Managed Identities to Roles # assign cluster identity permissions over identities previously created az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-operator" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/machine-api" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver" ######################## # VNet Role Assignment # assign vnet-level permissions for operators that require it, and subnets-level permission for operators that require it az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name cloud-controller-manager --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name cloud-controller-manager --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name ingress --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name ingress --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name machine-api --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0358943c-7e01-48ba-8889-02cc51d78637" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name machine-api --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0358943c-7e01-48ba-8889-02cc51d78637" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name cloud-network-config --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/be7a6435-15ae-4171-8f30-4a343eff9e8f" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name file-csi-driver --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0d7aedc0-15fd-4a67-a412-efad370c947e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name image-registry --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/8b32b316-c2f5-4ddf-b05b-83dacd2d08b5" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-operator --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/4436bae4-7702-4c84-919b-c4069ff25ee2" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-operator --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/4436bae4-7702-4c84-919b-c4069ff25ee2" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME" az role assignment create --assignee-object-id "$(az ad sp list --display-name "Azure Red Hat OpenShift RP" --query '[0].id' -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/42f3c60f-e7b1-46d7-ba56-6de681664342" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME" #################################################################################### ## Create the ARO Cluster and make it privately available az aro create \ --name $ARO_CLUSTER_NAME \ \ --location ${AZURE_LOCATION} \ --resource-group $RESOURCE_GROUP \ --cluster-resource-group $INFRASTRUCTURE_RESOURCE_GROUP \ --vnet-resource-group $VNET_RESOURCE_GROUP \ --vnet $VNET_NAME \ --master-subnet $VNET_CONTROL_PLANE_SUBNET_NAME \ --worker-subnet $VNET_APP_NODE_SUBNET_NAME \ --worker-vm-size $WORKER_VM_SIZE \ --apiserver-visibility $CLUSTER_EXPOSURE \ --ingress-visibility $CLUSTER_EXPOSURE \ --version $(az aro get-versions --location $AZURE_LOCATION | jq -r '.[-1]') \ --pull-secret "@rh-pull-secret.json" \ --enable-managed-identity \ --assign-cluster-identity /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-cluster \ --assign-platform-workload-identity file-csi-driver /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver \ --assign-platform-workload-identity cloud-controller-manager /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager \ --assign-platform-workload-identity ingress /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress \ --assign-platform-workload-identity image-registry /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry \ --assign-platform-workload-identity machine-api /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/machine-api \ --assign-platform-workload-identity cloud-network-config /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config \ --assign-platform-workload-identity aro-operator /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-operator \ --assign-platform-workload-identity disk-csi-driver /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver \ --pod-cidr $POD_CIDR_SUBNET \ --service-cidr $SERVICE_CIDR_SUBNET \ --debug #################################################################################### ## Read out cluster connection information az aro show --name $ARO_CLUSTER_NAME --resource-group $RESOURCE_GROUP | jq -r '.consoleProfile.url' az aro list-credentials \ --name $ARO_CLUSTER_NAME \ --resource-group $RESOURCE_GROUP ``` ## Justin Case ```bash= ## Delete the ARO Cluster and resources az aro delete --resource-group $RESOURCE_GROUP --name $ARO_CLUSTER_NAME ## Delete the Managed Identities az identity delete -g $RESOURCE_GROUP -n aro-cluster az identity delete -g $RESOURCE_GROUP -n aro-operator az identity delete -g $RESOURCE_GROUP -n cloud-controller-manager az identity delete -g $RESOURCE_GROUP -n cloud-network-config az identity delete -g $RESOURCE_GROUP -n disk-csi-driver az identity delete -g $RESOURCE_GROUP -n file-csi-driver az identity delete -g $RESOURCE_GROUP -n image-registry az identity delete -g $RESOURCE_GROUP -n ingress az identity delete -g $RESOURCE_GROUP -n machine-api ```