# ARO Deployment Guide of Ultimate Winning
## Prerequisites
- Azure CLI 2.84+
- Create a file called `rh-pull-secret.json` with your Red Hat Pull Secret: https://console.redhat.com/openshift/install/azure/aro-provisioned
- Microsoft Documentation: https://learn.microsoft.com/en-us/azure/openshift/howto-create-openshift-cluster?pivots=aro-deploy-az-cli
- Will deploy the latest version of OpenShift in Azure
```bash=
####################################################################################
## Set some Azure variables
export SUBSCRIPTION_ID="some-long-string" ## Switch to the proper Azure Subscription ID
export AZURE_LOCATION="centralus" ## Cluster Location
## Set a Cluster Name
export ARO_CLUSTER_NAME="my-handsome-aro"
# Configure Resource Groups
export CREATE_RESOURCE_GROUPS="true"
export RESOURCE_GROUP="${ARO_CLUSTER_NAME}-rg" ## ARO Resource Group Name
export VNET_RESOURCE_GROUP="vnet-${RESOURCE_GROUP}" ## VNet RG
export INFRASTRUCTURE_RESOURCE_GROUP="infra-${RESOURCE_GROUP}" # Infrastructure RG, do NOT pre-create, will be handled by ARM
# Configure VNet
export CREATE_VNET="true"
export VNET_NAME="aro-vnet" ## Define the VNet Name
export VNET_CIDR="10.42.0.0/16" ## Define the whole VNet CIDR Network
## Define the Subnet CIDRs and Names
export VNET_CONTROL_PLANE_SUBNET_CIDR="10.42.0.0/23"
export VNET_APP_NODE_SUBNET_CIDR="10.42.2.0/23"
export VNET_CONTROL_PLANE_SUBNET_NAME="${ARO_CLUSTER_NAME}-cp-sn"
export VNET_APP_NODE_SUBNET_NAME="${ARO_CLUSTER_NAME}-app-sn"
# Configuration Options
export CREATE_MANAGED_IDENTITIES="true"
export POD_CIDR_SUBNET="100.80.0.0/14" # NOT 100.64.0.0/16 OR 100.88.0.0/16, USED BY OVN-K
export SERVICE_CIDR_SUBNET="100.84.0.0/16" # NOT 100.64.0.0/16 OR 100.88.0.0/16, USED BY OVN-K
export CLUSTER_EXPOSURE="Private" # Private or Public
export WORKER_VM_SIZE="Standard_D4s_v5"
####################################################################################
## Start the creation process
## Set the proper subscription context
az account set --subscription $SUBSCRIPTION_ID
## Enable the resources one by one
az provider register -n Microsoft.RedHatOpenShift --wait
az provider register -n Microsoft.Compute --wait
az provider register -n Microsoft.Storage --wait
az provider register -n Microsoft.Network --wait
az provider register -n Microsoft.Authorization --wait
az provider register -n Microsoft.ContainerRegistry --wait
az provider register -n Microsoft.Quota --wait
az provider register -n Microsoft.Subscription --wait
####################################################################################
## Resource Groups
if [ $CREATE_RESOURCE_GROUPS = "true" ]; then
echo "Creating Resource Groups..."
## Create the ARO Resource Group
if [ $(az group exists -n $RESOURCE_GROUP) = "false" ]; then
az group create --name $RESOURCE_GROUP --location $AZURE_LOCATION
fi
## Create the VNet Resource Group
if [ $(az group exists -n $VNET_RESOURCE_GROUP) = "false" ]; then
az group create --name $VNET_RESOURCE_GROUP --location $AZURE_LOCATION
fi
# Validation Check
# The Infrastructure Resource Group is Automatically Created, an RG name is optional and will be randomly generated otherwise
if [ $(az group exists -n $INFRASTRUCTURE_RESOURCE_GROUP) = "false" ]; then
echo " - PASS: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP does not exist!"
else
echo " - FAIL: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP already exists!"
exit 1
fi
else
echo "Skipping Resource Group Creation..."
echo "Using:"
echo " - ARO Resource Group: $RESOURCE_GROUP"
echo " - ARO VNet Resource Group: $VNET_RESOURCE_GROUP"
echo " - ARO Infrastructure Resource Group: $INFRASTRUCTURE_RESOURCE_GROUP"
echo ""
echo "Checking for required Resource Groups..."
if [ $(az group exists -n $RESOURCE_GROUP) = "false" ]; then
echo " - FAIL: ARO Resource Group $RESOURCE_GROUP does not exist!"
exit 1
else
echo " - PASS: ARO Resource Group exists"
fi
if [ $(az group exists -n $VNET_RESOURCE_GROUP) = "false" ]; then
echo " - FAIL: ARO Resource Group $VNET_RESOURCE_GROUP does not exist!"
exit 1
else
echo " - PASS: ARO VNet Resource Group exists"
fi
if [ $(az group exists -n $INFRASTRUCTURE_RESOURCE_GROUP) = "false" ]; then
echo " - PASS: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP does not exist!"
else
echo " - FAIL: ARO Infrastructure Resource Group $INFRASTRUCTURE_RESOURCE_GROUP already exists!"
exit 1
fi
fi
####################################################################################
## Networking
if [ $CREATE_VNET = "true" ]; then
echo "Creating VNet and Subnets..."
## Create the VNet
if [ $(az network vnet list --resource-group $VNET_RESOURCE_GROUP --query "[?contains(name, '"$VNET_NAME"')]" | jq -r 'length') = "0" ]; then
az network vnet create \
--resource-group $VNET_RESOURCE_GROUP \
--location $AZURE_LOCATION \
--name $VNET_NAME \
--address-prefixes "${VNET_CIDR}"
else
echo "$VNET_NAME already exists!"
fi
## Create the Control Plane Subnet
if [ $(az network vnet subnet list -g $VNET_RESOURCE_GROUP --vnet-name $VNET_NAME --query "[?contains(name, '"$VNET_CONTROL_PLANE_SUBNET_NAME"')]" | jq -r 'length') = "0" ]; then
az network vnet subnet create \
--resource-group $VNET_RESOURCE_GROUP \
--vnet-name $VNET_NAME \
--name $VNET_CONTROL_PLANE_SUBNET_NAME \
--address-prefixes "${VNET_CONTROL_PLANE_SUBNET_CIDR}" \
--service-endpoints Microsoft.ContainerRegistry
else
echo "Subnet $VNET_CONTROL_PLANE_SUBNET_NAME in $VNET_NAME already exists!"
fi
## Create the Application Node Subnet
if [ $(az network vnet subnet list -g $VNET_RESOURCE_GROUP --vnet-name $VNET_NAME --query "[?contains(name, '"$VNET_APP_NODE_SUBNET_NAME"')]" | jq -r 'length') = "0" ]; then
az network vnet subnet create \
--resource-group $VNET_RESOURCE_GROUP \
--vnet-name $VNET_NAME \
--name $VNET_APP_NODE_SUBNET_NAME \
--address-prefixes "${VNET_APP_NODE_SUBNET_CIDR}" \
--service-endpoints Microsoft.ContainerRegistry
else
echo "Subnet $VNET_APP_NODE_SUBNET_NAME in $VNET_NAME already exists!"
fi
## Disable subnet private endpoints
az network vnet subnet update \
--name $VNET_CONTROL_PLANE_SUBNET_NAME \
--resource-group $VNET_RESOURCE_GROUP \
--vnet-name $VNET_NAME \
--private-link-service-network-policies Disabled
fi
####################################################################################
## Create the Managed Identities
if [ $CREATE_MANAGED_IDENTITIES = "true" ]; then
az identity create --resource-group $RESOURCE_GROUP --name aro-cluster --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name cloud-controller-manager --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name ingress --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name machine-api --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name disk-csi-driver --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name cloud-network-config --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name image-registry --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name file-csi-driver --location $AZURE_LOCATION
az identity create --resource-group $RESOURCE_GROUP --name aro-operator --location $AZURE_LOCATION
fi
########################
## Associate the Managed Identities to Roles
# assign cluster identity permissions over identities previously created
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-operator"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/machine-api"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-cluster --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/ef318e2a-8334-4a05-9e4a-295a196c6a6e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver"
########################
# VNet Role Assignment
# assign vnet-level permissions for operators that require it, and subnets-level permission for operators that require it
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name cloud-controller-manager --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name cloud-controller-manager --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name ingress --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name ingress --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name machine-api --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0358943c-7e01-48ba-8889-02cc51d78637" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name machine-api --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0358943c-7e01-48ba-8889-02cc51d78637" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name cloud-network-config --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/be7a6435-15ae-4171-8f30-4a343eff9e8f" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name file-csi-driver --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/0d7aedc0-15fd-4a67-a412-efad370c947e" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name image-registry --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/8b32b316-c2f5-4ddf-b05b-83dacd2d08b5" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-operator --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/4436bae4-7702-4c84-919b-c4069ff25ee2" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_CONTROL_PLANE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az identity show --resource-group $RESOURCE_GROUP --name aro-operator --query principalId -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/4436bae4-7702-4c84-919b-c4069ff25ee2" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME/subnets/$VNET_APP_NODE_SUBNET_NAME"
az role assignment create --assignee-object-id "$(az ad sp list --display-name "Azure Red Hat OpenShift RP" --query '[0].id' -o tsv)" --assignee-principal-type ServicePrincipal --role "/subscriptions/$SUBSCRIPTION_ID/providers/Microsoft.Authorization/roleDefinitions/42f3c60f-e7b1-46d7-ba56-6de681664342" --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_RESOURCE_GROUP/providers/Microsoft.Network/virtualNetworks/$VNET_NAME"
####################################################################################
## Create the ARO Cluster and make it privately available
az aro create \
--name $ARO_CLUSTER_NAME \ \
--location ${AZURE_LOCATION} \
--resource-group $RESOURCE_GROUP \
--cluster-resource-group $INFRASTRUCTURE_RESOURCE_GROUP \
--vnet-resource-group $VNET_RESOURCE_GROUP \
--vnet $VNET_NAME \
--master-subnet $VNET_CONTROL_PLANE_SUBNET_NAME \
--worker-subnet $VNET_APP_NODE_SUBNET_NAME \
--worker-vm-size $WORKER_VM_SIZE \
--apiserver-visibility $CLUSTER_EXPOSURE \
--ingress-visibility $CLUSTER_EXPOSURE \
--version $(az aro get-versions --location $AZURE_LOCATION | jq -r '.[-1]') \
--pull-secret "@rh-pull-secret.json" \
--enable-managed-identity \
--assign-cluster-identity /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-cluster \
--assign-platform-workload-identity file-csi-driver /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver \
--assign-platform-workload-identity cloud-controller-manager /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager \
--assign-platform-workload-identity ingress /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress \
--assign-platform-workload-identity image-registry /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry \
--assign-platform-workload-identity machine-api /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/machine-api \
--assign-platform-workload-identity cloud-network-config /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config \
--assign-platform-workload-identity aro-operator /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-operator \
--assign-platform-workload-identity disk-csi-driver /subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver \
--pod-cidr $POD_CIDR_SUBNET \
--service-cidr $SERVICE_CIDR_SUBNET \
--debug
####################################################################################
## Read out cluster connection information
az aro show --name $ARO_CLUSTER_NAME --resource-group $RESOURCE_GROUP | jq -r '.consoleProfile.url'
az aro list-credentials \
--name $ARO_CLUSTER_NAME \
--resource-group $RESOURCE_GROUP
```
## Justin Case
```bash=
## Delete the ARO Cluster and resources
az aro delete --resource-group $RESOURCE_GROUP --name $ARO_CLUSTER_NAME
## Delete the Managed Identities
az identity delete -g $RESOURCE_GROUP -n aro-cluster
az identity delete -g $RESOURCE_GROUP -n aro-operator
az identity delete -g $RESOURCE_GROUP -n cloud-controller-manager
az identity delete -g $RESOURCE_GROUP -n cloud-network-config
az identity delete -g $RESOURCE_GROUP -n disk-csi-driver
az identity delete -g $RESOURCE_GROUP -n file-csi-driver
az identity delete -g $RESOURCE_GROUP -n image-registry
az identity delete -g $RESOURCE_GROUP -n ingress
az identity delete -g $RESOURCE_GROUP -n machine-api
```