diff --git a/.github/skills/validate-devops-lab/azure/scripts/run-full-validation.sh b/.github/skills/validate-devops-lab/azure/scripts/run-full-validation.sh deleted file mode 100755 index 21a2d34..0000000 --- a/.github/skills/validate-devops-lab/azure/scripts/run-full-validation.sh +++ /dev/null @@ -1,745 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================= -# DEVOPS LAB - FULL END-TO-END VALIDATION -# Applies fixes, deploys to Azure, validates, generates token, destroys -# ============================================================================= - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="${SCRIPT_DIR}/../../../../.." -AZURE_DIR="${REPO_ROOT}/azure" -TERRAFORM_DIR="${AZURE_DIR}/terraform" -VALIDATE_SCRIPT="${AZURE_DIR}/scripts/validate.sh" - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -CYAN='\033[0;36m' -NC='\033[0m' - -SKIP_DEPLOY=false -SKIP_DESTROY=false -for arg in "$@"; do - case "$arg" in - --skip-deploy) SKIP_DEPLOY=true ;; - --skip-destroy) SKIP_DESTROY=true ;; - esac -done - -TOTAL_STEPS=9 -PASSED=0 -FAILED=0 -RESULTS=() - -log_result() { - local STEP="$1" - local STATUS="$2" - if [ "$STATUS" = "PASS" ]; then - RESULTS+=("PASS:$STEP") - PASSED=$((PASSED + 1)) - else - RESULTS+=("FAIL:$STEP") - FAILED=$((FAILED + 1)) - fi -} - -cleanup() { - echo "" - echo -e "${YELLOW}Cleaning up...${NC}" - - # Restore broken files - cd "$REPO_ROOT" - git checkout -- azure/ 2>/dev/null || true - - # Destroy Azure resources - if [ "$SKIP_DESTROY" = false ] && [ -f "${TERRAFORM_DIR}/terraform.tfstate" ]; then - local RG - RG=$(cd "$TERRAFORM_DIR" && terraform output -raw resource_group_name 2>/dev/null || echo "") - if [ -n "$RG" ]; then - echo "Deleting resource group: $RG" - az group delete -n "$RG" --yes --no-wait 2>/dev/null || true - fi - rm -rf "${TERRAFORM_DIR}/.terraform" "${TERRAFORM_DIR}/.terraform.lock.hcl" "${TERRAFORM_DIR}/terraform.tfstate"* - fi -} - -trap cleanup EXIT - -echo "" -echo "╔══════════════════════════════════════════════════════════════╗" -echo "║ DEVOPS LAB - FULL VALIDATION ║" -echo "╚══════════════════════════════════════════════════════════════╝" -echo "" - -SUB_ID=$(az account show --query id -o tsv 2>/dev/null) -echo "Azure Subscription: $SUB_ID" -echo "" - -# ───────────────────────────────────────────────────────────────── -# Step 1: Fix INC-001 (Dockerfile) -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[1/$TOTAL_STEPS] Fixing INC-001 (Dockerfile)...${NC}" - -cat > "${AZURE_DIR}/docker/Dockerfile" << 'EOF' -FROM python:3.11-slim -WORKDIR /app -COPY app/requirements.txt . -RUN pip install -r requirements.txt -COPY app/ . -EXPOSE 8000 -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] -EOF - -if docker build -f "${AZURE_DIR}/docker/Dockerfile" -t devops-lab-test "${AZURE_DIR}" > /dev/null 2>&1; then - CID=$(docker run -d -p 18000:8000 -e REDIS_HOST=localhost devops-lab-test 2>/dev/null || echo "") - sleep 3 - RESP=$(curl -s --max-time 5 http://localhost:18000/health 2>/dev/null || echo "") - docker rm -f "$CID" > /dev/null 2>&1 || true - docker rmi devops-lab-test > /dev/null 2>&1 || true - if echo "$RESP" | grep -q "healthy"; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Fix INC-001 (Dockerfile)" "PASS" - else - echo -e " ${RED}✗ FAIL - Container didn't respond${NC}" - log_result "Fix INC-001 (Dockerfile)" "FAIL" - fi -else - echo -e " ${RED}✗ FAIL - Build failed${NC}" - log_result "Fix INC-001 (Dockerfile)" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 2: Fix INC-002 (Docker Compose) -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[2/$TOTAL_STEPS] Fixing INC-002 (Docker Compose)...${NC}" - -cat > "${AZURE_DIR}/docker/docker-compose.yml" << 'EOF' -services: - app: - build: - context: .. - dockerfile: docker/Dockerfile - ports: - - "8000:8000" - environment: - - REDIS_HOST=redis - - REDIS_PORT=6379 - depends_on: - - redis - networks: - - backend - - redis: - image: redis:alpine - ports: - - "6379:6379" - volumes: - - redis_data:/data - networks: - - backend - -networks: - backend: - -volumes: - redis_data: -EOF - -docker compose -f "${AZURE_DIR}/docker/docker-compose.yml" up -d --build > /dev/null 2>&1 || true -sleep 5 -RESP=$(curl -s --max-time 5 http://localhost:8000/health 2>/dev/null || echo "") -docker compose -f "${AZURE_DIR}/docker/docker-compose.yml" down > /dev/null 2>&1 || true - -if echo "$RESP" | grep -q '"redis":"connected"'; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Fix INC-002 (Docker Compose)" "PASS" -else - echo -e " ${RED}✗ FAIL - Services didn't communicate${NC}" - log_result "Fix INC-002 (Docker Compose)" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 3: Fix INC-003 (CI Pipeline) -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[3/$TOTAL_STEPS] Fixing INC-003 (CI Pipeline)...${NC}" - -cat > "${AZURE_DIR}/github-actions/ci.yml" << 'EOF' -name: CI Pipeline - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - build-and-test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install dependencies - working-directory: ./azure/app - run: | - pip install -r requirements.txt - - - name: Run tests - working-directory: ./azure/app - run: | - python -m pytest tests/ -v - - - name: Build Docker image - run: | - docker build -f azure/docker/Dockerfile -t devops-lab-app . -EOF - -if python3 -c "import yaml; yaml.safe_load(open('${AZURE_DIR}/github-actions/ci.yml'))" 2>/dev/null; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Fix INC-003 (CI Pipeline)" "PASS" -else - echo -e " ${RED}✗ FAIL - YAML invalid${NC}" - log_result "Fix INC-003 (CI Pipeline)" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 4: Fix INC-004 (Terraform) + Deploy -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[4/$TOTAL_STEPS] Fixing INC-004 (Terraform) + Deploy...${NC}" - -cat > "${AZURE_DIR}/terraform/main.tf" << 'TFEOF' -terraform { - required_version = ">= 1.0" - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 4.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } -} - -provider "azurerm" { - features {} - subscription_id = var.subscription_id -} - -resource "random_id" "deployment" { - byte_length = 4 -} - -resource "azurerm_resource_group" "main" { - name = "rg-devopslab-${random_id.deployment.hex}" - location = var.location -} - -resource "azurerm_virtual_network" "main" { - name = "vnet-devopslab-${random_id.deployment.hex}" - address_space = ["10.0.0.0/16"] - location = azurerm_resource_group.main.location - resource_group_name = azurerm_resource_group.main.name -} - -resource "azurerm_subnet" "aks" { - name = "snet-aks" - resource_group_name = azurerm_resource_group.main.name - virtual_network_name = azurerm_virtual_network.main.name - address_prefixes = ["10.0.1.0/24"] -} - -resource "azurerm_container_registry" "main" { - name = "acrdevopslab${random_id.deployment.hex}" - resource_group_name = azurerm_resource_group.main.name - location = azurerm_resource_group.main.location - sku = "Basic" - admin_enabled = true -} - -resource "azurerm_log_analytics_workspace" "main" { - name = "law-devopslab-${random_id.deployment.hex}" - location = azurerm_resource_group.main.location - resource_group_name = azurerm_resource_group.main.name - sku = "PerGB2018" - retention_in_days = 30 -} - -resource "azurerm_kubernetes_cluster" "main" { - name = "aks-devopslab-${random_id.deployment.hex}" - location = azurerm_resource_group.main.location - resource_group_name = azurerm_resource_group.main.name - dns_prefix = "devopslab${random_id.deployment.hex}" - - default_node_pool { - name = "default" - node_count = 1 - vm_size = "Standard_B4ms" - vnet_subnet_id = azurerm_subnet.aks.id - } - - identity { - type = "SystemAssigned" - } - - oms_agent { - log_analytics_workspace_id = azurerm_log_analytics_workspace.main.id - } - - network_profile { - network_plugin = "azure" - service_cidr = "172.16.0.0/16" - dns_service_ip = "172.16.0.10" - } -} - -resource "azurerm_role_assignment" "aks_acr" { - scope = azurerm_container_registry.main.id - role_definition_name = "AcrPull" - principal_id = azurerm_kubernetes_cluster.main.kubelet_identity[0].object_id -} -TFEOF - -cat > "${AZURE_DIR}/terraform/outputs.tf" << 'EOF' -output "resource_group_name" { - value = azurerm_resource_group.main.name -} - -output "acr_login_server" { - value = azurerm_container_registry.main.login_server -} - -output "acr_name" { - value = azurerm_container_registry.main.name -} - -output "aks_cluster_name" { - value = azurerm_kubernetes_cluster.main.name -} - -output "deployment_id" { - value = random_id.deployment.hex -} - -output "log_analytics_workspace_id" { - value = azurerm_log_analytics_workspace.main.id -} -EOF - -if [ "$SKIP_DEPLOY" = false ]; then - cd "$TERRAFORM_DIR" - terraform init > /dev/null 2>&1 - if terraform validate > /dev/null 2>&1; then - echo " Deploying to Azure (this takes ~5-10 minutes)..." - if terraform apply -auto-approve -var subscription_id="$SUB_ID" > /tmp/tf-apply.log 2>&1; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Fix INC-004 (Terraform Deploy)" "PASS" - else - echo -e " ${RED}✗ FAIL - terraform apply failed${NC}" - tail -20 /tmp/tf-apply.log - log_result "Fix INC-004 (Terraform Deploy)" "FAIL" - fi - else - echo -e " ${RED}✗ FAIL - terraform validate failed${NC}" - terraform validate - log_result "Fix INC-004 (Terraform Deploy)" "FAIL" - fi - cd "$SCRIPT_DIR" -else - echo -e " ${YELLOW}SKIPPED (--skip-deploy)${NC}" - log_result "Fix INC-004 (Terraform Deploy)" "PASS" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 5: Fix INC-005 (CD Pipeline) -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[5/$TOTAL_STEPS] Fixing INC-005 (CD Pipeline)...${NC}" - -cat > "${AZURE_DIR}/github-actions/cd.yml" << 'EOF' -name: CD Pipeline - -on: - workflow_dispatch: - push: - branches: [main] - -env: - ACR_NAME: ${{ secrets.ACR_NAME }} - AKS_CLUSTER: ${{ secrets.AKS_CLUSTER_NAME }} - RESOURCE_GROUP: ${{ secrets.RESOURCE_GROUP }} - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Azure Login - uses: azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - - name: Login to ACR - run: az acr login --name ${{ env.ACR_NAME }} - - - name: Build and push image - run: | - docker build -f azure/docker/Dockerfile -t ${{ env.ACR_NAME }}.azurecr.io/devops-lab-app:${{ github.sha }} . - docker push ${{ env.ACR_NAME }}.azurecr.io/devops-lab-app:${{ github.sha }} - - deploy-to-aks: - runs-on: ubuntu-latest - needs: build-and-push - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Azure Login - uses: azure/login@v2 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - - - name: Get AKS credentials - run: | - az aks get-credentials --resource-group ${{ env.RESOURCE_GROUP }} --name ${{ env.AKS_CLUSTER }} - - - name: Deploy to AKS - run: | - kubectl set image deployment/devops-lab-app app=${{ env.ACR_NAME }}.azurecr.io/devops-lab-app:${{ github.sha }} -n devops-lab -EOF - -if ! grep -q "credentials:" "${AZURE_DIR}/github-actions/cd.yml" && grep -q "creds:" "${AZURE_DIR}/github-actions/cd.yml"; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Fix INC-005 (CD Pipeline)" "PASS" -else - echo -e " ${RED}✗ FAIL${NC}" - log_result "Fix INC-005 (CD Pipeline)" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 6: Fix INC-006 (Kubernetes) + Deploy to AKS -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[6/$TOTAL_STEPS] Fixing INC-006 (Kubernetes) + Deploy...${NC}" - -if [ "$SKIP_DEPLOY" = false ] && [ -f "${TERRAFORM_DIR}/terraform.tfstate" ]; then - cd "$TERRAFORM_DIR" - ACR_NAME=$(terraform output -raw acr_name 2>/dev/null) - ACR_SERVER=$(terraform output -raw acr_login_server 2>/dev/null) - AKS_NAME=$(terraform output -raw aks_cluster_name 2>/dev/null) - RG_NAME=$(terraform output -raw resource_group_name 2>/dev/null) - cd "$SCRIPT_DIR" - - # Push image to ACR - echo " Pushing image to ACR..." - az acr login --name "$ACR_NAME" > /dev/null 2>&1 - docker buildx build --platform linux/amd64 -f "${AZURE_DIR}/docker/Dockerfile" \ - -t "${ACR_SERVER}/devops-lab-app:latest" --push "${AZURE_DIR}" > /dev/null 2>&1 - - # Get AKS credentials - az aks get-credentials --resource-group "$RG_NAME" --name "$AKS_NAME" --overwrite-existing > /dev/null 2>&1 -fi - -# Write fixed K8s manifests -ACR_SERVER_FOR_K8S="${ACR_SERVER:-YOURACR.azurecr.io}" - -cat > "${AZURE_DIR}/kubernetes/app-deployment.yaml" << EOF -apiVersion: apps/v1 -kind: Deployment -metadata: - name: devops-lab-app - namespace: devops-lab -spec: - replicas: 2 - selector: - matchLabels: - app: devops-lab-app - template: - metadata: - labels: - app: devops-lab-app - spec: - containers: - - name: app - image: ${ACR_SERVER_FOR_K8S}/devops-lab-app:latest - ports: - - containerPort: 8000 - env: - - name: REDIS_HOST - value: "redis" - - name: REDIS_PORT - value: "6379" - livenessProbe: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 10 - readinessProbe: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 5 -EOF - -cat > "${AZURE_DIR}/kubernetes/app-service.yaml" << 'EOF' -apiVersion: v1 -kind: Service -metadata: - name: devops-lab-app - namespace: devops-lab -spec: - type: LoadBalancer - selector: - app: devops-lab-app - ports: - - protocol: TCP - port: 80 - targetPort: 8000 -EOF - -cat > "${AZURE_DIR}/kubernetes/redis-deployment.yaml" << 'EOF' -apiVersion: apps/v1 -kind: Deployment -metadata: - name: redis - namespace: devops-lab -spec: - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - labels: - app: redis - spec: - containers: - - name: redis - image: redis:alpine - ports: - - containerPort: 6379 -EOF - -cat > "${AZURE_DIR}/kubernetes/redis-service.yaml" << 'EOF' -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: devops-lab -spec: - selector: - app: redis - ports: - - protocol: TCP - port: 6379 - targetPort: 6379 -EOF - -if [ "$SKIP_DEPLOY" = false ] && [ -n "$AKS_NAME" ]; then - echo " Deploying to AKS..." - kubectl apply -f "${AZURE_DIR}/kubernetes/namespace.yaml" > /dev/null 2>&1 - kubectl apply -f "${AZURE_DIR}/kubernetes/" > /dev/null 2>&1 - - echo " Waiting for pods..." - for i in $(seq 1 12); do - sleep 10 - PODS_READY=$(kubectl get pods -n devops-lab --no-headers 2>/dev/null | grep -c "Running" || echo "0") - if [ "$PODS_READY" -ge 3 ]; then - break - fi - echo " ($i/12) $PODS_READY pods running..." - done - if [ "$PODS_READY" -ge 3 ]; then - echo -e " ${GREEN}✓ PASS ($PODS_READY pods running)${NC}" - log_result "Fix INC-006 (Kubernetes)" "PASS" - else - echo -e " ${RED}✗ FAIL - Only $PODS_READY pods running${NC}" - kubectl get pods -n devops-lab 2>/dev/null - log_result "Fix INC-006 (Kubernetes)" "FAIL" - fi -else - # Local validation only - if ! grep -q "v1beta1" "${AZURE_DIR}/kubernetes/app-deployment.yaml" && \ - ! grep -q "ACR_LOGIN_SERVER" "${AZURE_DIR}/kubernetes/app-deployment.yaml" && \ - ! grep -q "containerPort: 5000" "${AZURE_DIR}/kubernetes/app-deployment.yaml"; then - echo -e " ${GREEN}✓ PASS (local validation)${NC}" - log_result "Fix INC-006 (Kubernetes)" "PASS" - else - echo -e " ${RED}✗ FAIL${NC}" - log_result "Fix INC-006 (Kubernetes)" "FAIL" - fi -fi - -# ───────────────────────────────────────────────────────────────── -# Step 7: Fix INC-007 (Monitoring) -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[7/$TOTAL_STEPS] Fixing INC-007 (Monitoring)...${NC}" - -cat > "${AZURE_DIR}/monitoring/alerts.json" << 'EOF' -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "workspaceId": { "type": "string" }, - "aksClusterName": { "type": "string" }, - "resourceGroupName": { "type": "string" } - }, - "resources": [ - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "high-cpu-alert", - "location": "global", - "properties": { - "description": "Alert when CPU exceeds 90%", - "severity": 2, - "enabled": true, - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', parameters('aksClusterName'))]" - ], - "evaluationFrequency": "PT1M", - "windowSize": "PT5M", - "criteria": { - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria", - "allOf": [ - { - "name": "cpu-check", - "metricName": "node_cpu_usage_percentage", - "metricNamespace": "Microsoft.ContainerService/managedClusters", - "operator": "GreaterThan", - "threshold": 90, - "timeAggregation": "Average" - } - ] - } - } - }, - { - "type": "Microsoft.Insights/metricAlerts", - "apiVersion": "2018-03-01", - "name": "pod-restart-alert", - "location": "global", - "properties": { - "description": "Alert on pod restarts", - "severity": 2, - "enabled": true, - "scopes": [ - "[resourceId('Microsoft.ContainerService/managedClusters', parameters('aksClusterName'))]" - ], - "evaluationFrequency": "PT1M", - "windowSize": "PT30M", - "criteria": { - "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria", - "allOf": [ - { - "name": "restart-check", - "metricName": "kube_pod_status_restarts_total", - "metricNamespace": "Microsoft.ContainerService/managedClusters", - "operator": "GreaterThan", - "threshold": 3, - "timeAggregation": "Maximum" - } - ] - } - } - } - ] -} -EOF - -ENABLED=$(python3 -c " -import json -with open('${AZURE_DIR}/monitoring/alerts.json') as f: - data = json.load(f) -for r in data.get('resources', []): - if r.get('name') == 'pod-restart-alert': - print(r['properties']['enabled']) -" 2>/dev/null || echo "false") - -if [ "$ENABLED" = "True" ]; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Fix INC-007 (Monitoring)" "PASS" -else - echo -e " ${RED}✗ FAIL${NC}" - log_result "Fix INC-007 (Monitoring)" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 8: Token Generation -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[8/$TOTAL_STEPS] Testing token generation...${NC}" - -TOKEN=$(echo "testuser" | "$VALIDATE_SCRIPT" export 2>/dev/null | grep -A1 "BEGIN L2C" | tail -1) - -if [ -n "$TOKEN" ] && [ ${#TOKEN} -gt 20 ]; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Token Generation" "PASS" -else - echo -e " ${RED}✗ FAIL - No token generated${NC}" - log_result "Token Generation" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Step 9: Token Verification -# ───────────────────────────────────────────────────────────────── -echo -e "${CYAN}[9/$TOTAL_STEPS] Testing token verification...${NC}" - -if [ -n "$TOKEN" ]; then - VERIFY_OUTPUT=$("$VALIDATE_SCRIPT" verify "$TOKEN" 2>&1) - if echo "$VERIFY_OUTPUT" | grep -q "VALID"; then - echo -e " ${GREEN}✓ PASS${NC}" - log_result "Token Verification" "PASS" - else - echo -e " ${RED}✗ FAIL - Token verification failed${NC}" - echo "$VERIFY_OUTPUT" - log_result "Token Verification" "FAIL" - fi -else - echo -e " ${RED}✗ FAIL - No token to verify${NC}" - log_result "Token Verification" "FAIL" -fi - -# ───────────────────────────────────────────────────────────────── -# Summary -# ───────────────────────────────────────────────────────────────── -echo "" -echo "╔══════════════════════════════════════════════════════════════╗" -echo "║ VALIDATION SUMMARY ║" -echo "╚══════════════════════════════════════════════════════════════╝" -echo "" -echo "┌────────────────────────────────────┬──────────┐" -echo "│ Step │ Result │" -echo "├────────────────────────────────────┼──────────┤" - -for entry in "${RESULTS[@]}"; do - STATUS="${entry%%:*}" - STEP="${entry#*:}" - if [ "$STATUS" = "PASS" ]; then - printf "│ %-34s │ ${GREEN}✓ PASS${NC} │\n" "$STEP" - else - printf "│ %-34s │ ${RED}✗ FAIL${NC} │\n" "$STEP" - fi -done - -echo "└────────────────────────────────────┴──────────┘" -echo "" - -if [ $FAILED -eq 0 ]; then - echo "══════════════════════════════════════════════════════════════" - echo -e " ${GREEN}ALL TESTS PASSED ($PASSED/$TOTAL_STEPS)${NC}" - echo " Lab validation complete — ready for student use" - echo "══════════════════════════════════════════════════════════════" - exit 0 -else - echo "══════════════════════════════════════════════════════════════" - echo -e " ${RED}SOME TESTS FAILED ($PASSED passed, $FAILED failed)${NC}" - echo "══════════════════════════════════════════════════════════════" - exit 1 -fi diff --git a/.gitignore b/.gitignore index 354aba1..e24081b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ venv/ # Solutions (maintainer only) **/solutions/ + +# Full validation scripts (contain solutions) +.github/skills/validate-devops-lab/aws/scripts/run-full-validation.sh +.github/skills/validate-devops-lab/azure/scripts/run-full-validation.sh diff --git a/README.md b/README.md index 31b4894..1369e1a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ You've just joined a startup as the DevOps engineer. The previous engineer left, | Provider | Status | Guide | |----------|--------|-------| | Azure | ✅ Available | [azure/README.md](azure/README.md) | -| AWS | 🚧 Coming soon | — | +| AWS | ✅ Available | [aws/README.md](aws/README.md) | | GCP | 🚧 Coming soon | — | ## How It Works diff --git a/aws/README.md b/aws/README.md index e812b7e..823da1f 100644 --- a/aws/README.md +++ b/aws/README.md @@ -1,14 +1,167 @@ # AWS DevOps Lab -🚧 **Coming soon.** +Fix a broken DevOps pipeline deployed to AWS. Work through 7 incidents to get the application running. -The AWS version of this lab will use equivalent services: +``` +┌─────────────────────────────────────────────────────────────┐ +│ AWS Resources │ +│ │ +│ ┌──────────┐ ┌──────────┐ ┌────────────────────────┐ │ +│ │ VPC │ │ ECR │ │ EKS │ │ +│ │ │ │ (images) │──▶│ ┌─────┐ ┌───────┐ │ │ +│ │ Subnet │ │ │ │ │ App │──│ Redis │ │ │ +│ │ │ └──────────┘ │ └─────┘ └───────┘ │ │ +│ └──────────┘ └────────────────────────┘ │ +│ │ +│ ┌──────────────────┐ ┌───────────────────────────────┐ │ +│ │ CloudWatch │ │ Container Insights │ │ +│ │ Log Group │ │ + Alarms │ │ +│ └──────────────────┘ └───────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` -| Azure | AWS Equivalent | -|-------|---------------| -| ACR | ECR | -| AKS | EKS | -| Azure Monitor | CloudWatch | -| VNet | VPC | +## Prerequisites -Want to help build it? See our [Contributing Guide](../CONTRIBUTING.md). +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) +- [Terraform](https://developer.hashicorp.com/terraform/install) (v1.0+) +- [Docker](https://docs.docker.com/get-docker/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/) + +## Getting Started + +1. Clone this repo and navigate to the AWS scripts: + ```bash + git clone https://github.com/learntocloud/devops-lab + cd devops-lab/aws/scripts + ``` + +2. Log in to AWS: + ```bash + aws configure + ``` + +3. Run the setup script: + ```bash + chmod +x *.sh + ./setup.sh + ``` + +**Cost**: ~$3-5/session. Destroy resources when done. + +--- + +## Incident Queue + +You're the new DevOps engineer. Seven incidents are waiting. Diagnose and fix each one. + +--- + +### 🎫 INC-001: Container Image Won't Build + +**Priority:** High +**Reported by:** Development Team +**Tools:** `docker` CLI + +> "We can't build the app's Docker image. The `docker build` command fails immediately with errors. The Dockerfile is at `aws/docker/Dockerfile`. We need the image to build successfully and the container to start and respond on the correct port." + +**What to fix:** `aws/docker/Dockerfile` + +--- + +### 🎫 INC-002: Local Dev Environment Broken + +**Priority:** High +**Reported by:** Development Team +**Tools:** `docker compose` CLI + +> "Docker Compose won't bring up our local environment. The app can't connect to Redis, and the port mapping seems wrong. The compose file is at `aws/docker/docker-compose.yml`. We need both services (app + redis) to start and communicate." + +**What to fix:** `aws/docker/docker-compose.yml` + +--- + +### 🎫 INC-003: CI Pipeline is Broken + +**Priority:** High +**Reported by:** Engineering Manager +**Tools:** GitHub Actions YAML reference + +> "Our CI workflow has YAML errors and the steps are in the wrong order. Tests run before dependencies are installed, and some action versions look wrong. The workflow is at `aws/github-actions/ci.yml`." + +**What to fix:** `aws/github-actions/ci.yml` + +--- + +### 🎫 INC-004: Terraform Can't Provision Infrastructure + +**Priority:** Critical +**Reported by:** Platform Team +**Tools:** `terraform` CLI, `aws` CLI + +> "Terraform plan fails with multiple errors. There are typos in resource types, something is wrong with the IAM role policies, and the cluster networking configuration has conflicts. The config is at `aws/terraform/`. We need the VPC, ECR, EKS cluster, and monitoring log group to all deploy successfully." + +**What to fix:** `aws/terraform/main.tf`, `aws/terraform/outputs.tf` + +--- + +### 🎫 INC-005: Deployment Pipeline Failing + +**Priority:** High +**Reported by:** Release Team +**Tools:** GitHub Actions YAML reference, `aws` CLI + +> "The CD pipeline can't deploy to EKS. The AWS credentials action is misconfigured, and the deployment steps aren't right. The workflow is at `aws/github-actions/cd.yml`." + +**What to fix:** `aws/github-actions/cd.yml` + +**Note:** OIDC is the recommended approach. + +--- + +### 🎫 INC-006: Kubernetes Deployment Crashing + +**Priority:** Critical +**Reported by:** SRE Team +**Tools:** `kubectl` CLI + +> "Pods won't start in EKS. The deployments have wrong API versions, label selectors don't match between deployments and services, container ports are wrong, and the readiness probe is hitting an endpoint that doesn't exist. Manifests are in `aws/kubernetes/`." + +**What to fix:** `aws/kubernetes/app-deployment.yaml`, `aws/kubernetes/app-service.yaml`, `aws/kubernetes/redis-deployment.yaml`, `aws/kubernetes/redis-service.yaml` + +--- + +### 🎫 INC-007: Monitoring Not Working + +**Priority:** Medium +**Reported by:** Observability Team +**Tools:** `aws` CLI + +> "The pod restart alarm is disabled and should be enabled. We need Container Insights running on EKS, and our alarm configuration at `aws/monitoring/alerts.json` needs fixing. The alarm for pod restarts should be severity 2 (not 1), and it should evaluate every minute (not every 5 minutes)." + +**What to fix:** `aws/monitoring/alerts.json` + +--- + +## Verify Your Fixes + +Check incident status anytime: + +```bash +cd aws/scripts +./validate.sh +``` + +Generate your completion token after all incidents are resolved: + +```bash +./validate.sh export +``` + +## Clean Up + +**Always destroy resources when done to avoid charges:** + +```bash +cd aws/scripts +./destroy.sh +``` diff --git a/aws/app/app.py b/aws/app/app.py new file mode 100644 index 0000000..0e5a439 --- /dev/null +++ b/aws/app/app.py @@ -0,0 +1,39 @@ +from fastapi import FastAPI +import redis +import os + +app = FastAPI(title="DevOps Lab App") + +REDIS_HOST = os.getenv("REDIS_HOST", "localhost") +REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) + + +def get_redis(): + try: + r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True) + r.ping() + return r + except redis.ConnectionError: + return None + + +@app.get("/health") +def health(): + r = get_redis() + redis_status = "connected" if r else "disconnected" + return {"status": "healthy", "redis": redis_status} + + +@app.get("/api/status") +def status(): + r = get_redis() + if r: + visits = r.incr("visits") + else: + visits = -1 + return { + "app": "devops-lab", + "version": "1.0.0", + "visits": visits, + "redis": "connected" if r else "disconnected", + } diff --git a/aws/app/requirements.txt b/aws/app/requirements.txt new file mode 100644 index 0000000..eceecf6 --- /dev/null +++ b/aws/app/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.115.0 +uvicorn==0.30.0 +redis==5.0.0 diff --git a/aws/app/tests/test_app.py b/aws/app/tests/test_app.py new file mode 100644 index 0000000..5a3202e --- /dev/null +++ b/aws/app/tests/test_app.py @@ -0,0 +1,19 @@ +from fastapi.testclient import TestClient +from app import app + +client = TestClient(app) + + +def test_health(): + response = client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + + +def test_status(): + response = client.get("/api/status") + assert response.status_code == 200 + data = response.json() + assert data["app"] == "devops-lab" + assert data["version"] == "1.0.0" diff --git a/aws/docker/Dockerfile b/aws/docker/Dockerfile new file mode 100644 index 0000000..0fdbc85 --- /dev/null +++ b/aws/docker/Dockerfile @@ -0,0 +1,14 @@ +# Dockerfile for DevOps Lab App +FROM python:3.11-slm + +WORKDIR /src + +COPY app/requirements.txt . + +RUN pip install -r requirements.txt + +COPY app/ . + +EXPOSE 5000 + +CMD ["python", "app.py"] diff --git a/aws/docker/docker-compose.yml b/aws/docker/docker-compose.yml new file mode 100644 index 0000000..fa40f27 --- /dev/null +++ b/aws/docker/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.8" + +services: + app: + build: + context: . + dockerfile: Dockerfile + ports: + - "8000:5000" + environment: + - REDIS_HOST=cache + - REDIS_PORT=6379 + depends_on: + - cache + networks: + - backend + + redis: + image: redis:alpine + ports: + - "6379:6379" + volumes: + - redis_data:/data + +volumes: + redis_data: diff --git a/aws/github-actions/cd.yml b/aws/github-actions/cd.yml new file mode 100644 index 0000000..4bae7e0 --- /dev/null +++ b/aws/github-actions/cd.yml @@ -0,0 +1,55 @@ +name: CD Pipeline + +on: + workflow_dispatch: + push: + branches: [main] + +env: + AWS_REGION: ${{ secrets.AWS_REGION }} + ECR_REPO: ${{ secrets.ECR_REPO }} + EKS_CLUSTER: ${{ secrets.EKS_CLUSTER_NAME }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + credentials: ${{ secrets.AWS_CREDENTIALS }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to ECR + run: | + aws ecr get-login-password --region ${{ env.AWS_REGION }} | \ + docker login --username AWS --password-stdin ${{ env.ECR_REPO }} + + - name: Build and push image + run: | + docker build -f aws/docker/Dockerfile -t ${{ env.ECR_REPO }}:latest . + docker push ${{ env.ECR_REPO }}:latest + + deploy-to-eks: + runs-on: ubuntu-latest + needs: build-and-push + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + credentials: ${{ secrets.AWS_CREDENTIALS }} + aws-region: ${{ env.AWS_REGION }} + + - name: Get EKS credentials + run: | + aws eks update-kubeconfig --region ${{ env.AWS_REGION }} --name ${{ env.EKS_CLUSTER }} + + - name: Deploy to EKS + run: | + kubectl set image deployment/devops-lab-app app=${{ env.ECR_REPO }}:latest -n devops-lab diff --git a/aws/github-actions/ci.yml b/aws/github-actions/ci.yml new file mode 100644 index 0000000..88f111a --- /dev/null +++ b/aws/github-actions/ci.yml @@ -0,0 +1,33 @@ +name: CI Pipeline + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v99 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run tests + working-directory: ./aws/app + run: | + python -m pytest tests/ -v + + - name: Install dependencies + working-directory: ./aws/app + run: | + pip install -r requirements.txt + + - name: Build Docker image + run: | + docker build -f aws/docker/Dockerfile -t devops-lab-app . diff --git a/aws/kubernetes/app-deployment.yaml b/aws/kubernetes/app-deployment.yaml new file mode 100644 index 0000000..560e557 --- /dev/null +++ b/aws/kubernetes/app-deployment.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: devops-lab-app + namespace: devops-lab +spec: + replicas: 2 + selector: + matchLabels: + app: devopslab-app + template: + metadata: + labels: + app: devops-lab-app + spec: + containers: + - name: app + image: ECR_REGISTRY/devops-lab-app:latest + ports: + - containerPort: 5000 + env: + - name: REDIS_HOST + value: "redis" + - name: REDIS_PORT + value: "6379" + livenessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 5000 + initialDelaySeconds: 5 diff --git a/aws/kubernetes/app-service.yaml b/aws/kubernetes/app-service.yaml new file mode 100644 index 0000000..0cf9091 --- /dev/null +++ b/aws/kubernetes/app-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: devops-lab-app + namespace: devops-lab +spec: + type: LoadBalancer + selector: + app: devopslab-app + ports: + - protocol: TCP + port: 80 + targetPort: 5000 diff --git a/aws/kubernetes/namespace.yaml b/aws/kubernetes/namespace.yaml new file mode 100644 index 0000000..5dedf0b --- /dev/null +++ b/aws/kubernetes/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: devops-lab diff --git a/aws/kubernetes/redis-deployment.yaml b/aws/kubernetes/redis-deployment.yaml new file mode 100644 index 0000000..2fbea7c --- /dev/null +++ b/aws/kubernetes/redis-deployment.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: devops-lab +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis-cache + spec: + containers: + - name: redis + image: redis:alpine + ports: + - containerPort: 6380 diff --git a/aws/kubernetes/redis-service.yaml b/aws/kubernetes/redis-service.yaml new file mode 100644 index 0000000..153300b --- /dev/null +++ b/aws/kubernetes/redis-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: devops-lab +spec: + selector: + app: redis + ports: + - protocol: TCP + port: 6379 + targetPort: 6380 diff --git a/aws/monitoring/alerts.json b/aws/monitoring/alerts.json new file mode 100644 index 0000000..41217a8 --- /dev/null +++ b/aws/monitoring/alerts.json @@ -0,0 +1,28 @@ +{ + "alarms": [ + { + "AlarmName": "high-cpu-alarm", + "Namespace": "ContainerInsights", + "MetricName": "node_cpu_utilization", + "Statistic": "Average", + "Period": 60, + "EvaluationPeriods": 5, + "Threshold": 90, + "ComparisonOperator": "GreaterThanThreshold", + "ActionsEnabled": true, + "Severity": 2 + }, + { + "AlarmName": "pod-restart-alarm", + "Namespace": "ContainerInsights", + "MetricName": "pod_number_of_container_restarts", + "Statistic": "Maximum", + "Period": 300, + "EvaluationPeriods": 1, + "Threshold": 3, + "ComparisonOperator": "GreaterThanThreshold", + "ActionsEnabled": false, + "Severity": 1 + } + ] +} diff --git a/aws/scripts/destroy.sh b/aws/scripts/destroy.sh new file mode 100755 index 0000000..5fbe916 --- /dev/null +++ b/aws/scripts/destroy.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# ============================================================================= +# DEVOPS LAB - DESTROY SCRIPT +# Tears down all AWS resources +# ============================================================================= + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TERRAFORM_DIR="${SCRIPT_DIR}/../terraform" + +AUTO_APPROVE=false +for arg in "$@"; do + case "$arg" in + --yes|--auto-approve) AUTO_APPROVE=true ;; + esac +done + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo "" +echo -e "${RED}============================================${NC}" +echo -e "${RED} DEVOPS LAB - DESTROY RESOURCES${NC}" +echo -e "${RED}============================================${NC}" +echo "" + +REGION=$(aws configure get region 2>/dev/null || echo "") +if [ -z "$REGION" ]; then + REGION="us-east-1" +fi + +empty_ecr_repository() { + local REPO="$1" + local IMAGE_IDS BATCHES + IMAGE_IDS=$(aws ecr list-images --repository-name "$REPO" \ + --query 'imageIds' --output json 2>/dev/null || echo "[]") + if [ -z "$IMAGE_IDS" ] || [ "$IMAGE_IDS" = "[]" ]; then + return + fi + + BATCHES=$(printf '%s' "$IMAGE_IDS" | python3 - <<'PY' +import json +import sys +try: + data = json.load(sys.stdin) +except json.JSONDecodeError: + sys.exit(0) +if not data: + sys.exit(0) +for i in range(0, len(data), 100): + print(json.dumps(data[i:i+100])) +PY +) + while IFS= read -r batch; do + [ -z "$batch" ] && continue + aws ecr batch-delete-image --repository-name "$REPO" \ + --image-ids "$batch" >/dev/null 2>&1 || true + done <<< "$BATCHES" +} + +delete_elbs_for_vpc() { + local VPC_ID="$1" + local ELB_NAMES ELB_ARNS + + ELB_NAMES=$(aws elb describe-load-balancers \ + --query "LoadBalancerDescriptions[?VPCId=='$VPC_ID'].LoadBalancerName" \ + --output text 2>/dev/null || echo "") + for name in $ELB_NAMES; do + aws elb delete-load-balancer --load-balancer-name "$name" >/dev/null 2>&1 || true + done + + ELB_ARNS=$(aws elbv2 describe-load-balancers \ + --query "LoadBalancers[?VpcId=='$VPC_ID'].LoadBalancerArn" \ + --output text 2>/dev/null || echo "") + for arn in $ELB_ARNS; do + aws elbv2 delete-load-balancer --load-balancer-arn "$arn" >/dev/null 2>&1 || true + done +} + +wait_for_elb_enis_gone() { + local VPC_ID="$1" + local REPEAT=20 + + while [ $REPEAT -gt 0 ]; do + local ENI_COUNT + ENI_COUNT=$(aws ec2 describe-network-interfaces \ + --filters Name=vpc-id,Values="$VPC_ID" Name=description,Values="ELB*" \ + --query 'length(NetworkInterfaces)' --output text 2>/dev/null || echo "0") + if [ "$ENI_COUNT" = "0" ]; then + return + fi + sleep 10 + REPEAT=$((REPEAT - 1)) + done +} + +unmap_public_addresses() { + local VPC_ID="$1" + local ENIS + + ENIS=$(aws ec2 describe-network-interfaces \ + --filters Name=vpc-id,Values="$VPC_ID" \ + --query 'NetworkInterfaces[].NetworkInterfaceId' --output text 2>/dev/null || echo "") + for eni in $ENIS; do + local ASSOC_ID ALLOC_ID + ASSOC_ID=$(aws ec2 describe-addresses \ + --filters Name=network-interface-id,Values="$eni" \ + --query 'Addresses[].AssociationId' --output text 2>/dev/null || echo "") + for assoc in $ASSOC_ID; do + aws ec2 disassociate-address --association-id "$assoc" >/dev/null 2>&1 || true + done + + ALLOC_ID=$(aws ec2 describe-addresses \ + --filters Name=network-interface-id,Values="$eni" \ + --query 'Addresses[].AllocationId' --output text 2>/dev/null || echo "") + for alloc in $ALLOC_ID; do + aws ec2 release-address --allocation-id "$alloc" >/dev/null 2>&1 || true + done + done +} + +delete_available_enis() { + local VPC_ID="$1" + local ENIS + + ENIS=$(aws ec2 describe-network-interfaces \ + --filters Name=vpc-id,Values="$VPC_ID" \ + --query 'NetworkInterfaces[?Status==`available`].NetworkInterfaceId' \ + --output text 2>/dev/null || echo "") + for eni in $ENIS; do + aws ec2 delete-network-interface --network-interface-id "$eni" >/dev/null 2>&1 || true + done +} + +delete_security_groups_for_vpc() { + local VPC_ID="$1" + local SG_GROUPS + + SG_GROUPS=$(aws ec2 describe-security-groups \ + --filters Name=vpc-id,Values="$VPC_ID" \ + --query 'SecurityGroups[?GroupName!=`default`].GroupId' --output text 2>/dev/null || echo "") + for sg in $SG_GROUPS; do + aws ec2 delete-security-group --group-id "$sg" >/dev/null 2>&1 || true + done +} + +if [ -f "${TERRAFORM_DIR}/terraform.tfstate" ]; then + echo "Found Terraform state. Attempting terraform destroy..." + + echo -e "${YELLOW}This will destroy ALL resources in the Terraform state.${NC}" + echo "" + if [ "$AUTO_APPROVE" = false ]; then + read -p "Continue? (y/N) " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 + fi + fi + + cd "$TERRAFORM_DIR" + ECR_REPO=$(terraform output -raw ecr_repository_name 2>/dev/null || echo "") + EKS_NAME=$(terraform output -raw eks_cluster_name 2>/dev/null || echo "") + VPC_ID=$(terraform output -raw vpc_id 2>/dev/null || echo "") + if [ -n "$ECR_REPO" ]; then + empty_ecr_repository "$ECR_REPO" + fi + + if [ -n "$EKS_NAME" ]; then + echo "Cleaning Kubernetes load balancers..." + aws eks update-kubeconfig --region "$REGION" --name "$EKS_NAME" >/dev/null 2>&1 || true + kubectl delete svc devops-lab-app -n devops-lab >/dev/null 2>&1 || true + kubectl delete namespace devops-lab >/dev/null 2>&1 || true + fi + + if [ -n "$VPC_ID" ]; then + echo "Releasing public addresses and load balancers..." + delete_elbs_for_vpc "$VPC_ID" + wait_for_elb_enis_gone "$VPC_ID" + unmap_public_addresses "$VPC_ID" + delete_available_enis "$VPC_ID" + delete_security_groups_for_vpc "$VPC_ID" + fi + + echo "Preparing EKS resources for destroy..." + terraform destroy -auto-approve -var region="$REGION" -target=aws_eks_node_group.main >/dev/null 2>&1 || true + terraform destroy -auto-approve -var region="$REGION" -target=aws_eks_cluster.main >/dev/null 2>&1 || true + + set +e + terraform destroy -auto-approve -var region="$REGION" + TF_DESTROY_EXIT=$? + set -e + + if [ $TF_DESTROY_EXIT -eq 0 ]; then + rm -f terraform.tfstate terraform.tfstate.backup + rm -rf .terraform .terraform.lock.hcl + echo -e "${GREEN}Terraform state cleaned up.${NC}" + else + echo -e "${YELLOW}Terraform destroy failed.${NC}" + fi +else + echo "No Terraform state found." +fi diff --git a/aws/scripts/setup.sh b/aws/scripts/setup.sh new file mode 100644 index 0000000..44f5c4d --- /dev/null +++ b/aws/scripts/setup.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# ============================================================================= +# DEVOPS LAB - SETUP SCRIPT +# Initializes the lab and optionally deploys infrastructure +# ============================================================================= + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TERRAFORM_DIR="${SCRIPT_DIR}/../terraform" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +echo "" +echo -e "${BLUE}============================================${NC}" +echo -e "${BLUE} DEVOPS LAB - SETUP${NC}" +echo -e "${BLUE}============================================${NC}" +echo "" + +# Pre-flight checks +echo "Checking prerequisites..." + +if ! command -v aws &> /dev/null; then + echo -e "${RED}Error: AWS CLI not found.${NC}" + echo "Install: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" + exit 1 +fi +echo -e " ${GREEN}✓${NC} AWS CLI found" + +if ! aws sts get-caller-identity &> /dev/null; then + echo -e "${YELLOW}Not logged in to AWS. Running 'aws configure'...${NC}" + aws configure +fi +ACCOUNT_ID=$(aws sts get-caller-identity --query Account -o text) +REGION=$(aws configure get region) +if [ -z "$REGION" ]; then + REGION="us-east-1" +fi +echo -e " ${GREEN}✓${NC} Logged in: $ACCOUNT_ID ($REGION)" + +if ! command -v terraform &> /dev/null; then + echo -e "${RED}Error: Terraform not found.${NC}" + echo "Install: https://www.terraform.io/downloads" + exit 1 +fi +echo -e " ${GREEN}✓${NC} Terraform found" + +if ! command -v docker &> /dev/null; then + echo -e "${YELLOW}Warning: Docker not found. You'll need it for INC-001 and INC-002.${NC}" +else + echo -e " ${GREEN}✓${NC} Docker found" +fi + +if ! command -v kubectl &> /dev/null; then + echo -e "${YELLOW}Warning: kubectl not found. You'll need it for INC-006.${NC}" +else + echo -e " ${GREEN}✓${NC} kubectl found" +fi + +echo "" +echo -e "${YELLOW}This lab deploys AWS resources that cost ~\$3-5/session.${NC}" +echo -e "${YELLOW}Account: $ACCOUNT_ID ($REGION)${NC}" +echo "" +read -p "Continue? (y/N) " -n 1 -r +echo "" + +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 +fi + +echo "" +echo "Note: Before deploying, you need to fix the Terraform configuration (INC-004)." +echo "The Terraform files have intentional errors that must be fixed first." +echo "" +echo "To start working on the lab:" +echo " 1. Fix INC-001 (Dockerfile): aws/docker/Dockerfile" +echo " 2. Fix INC-002 (Compose): aws/docker/docker-compose.yml" +echo " 3. Fix INC-003 (CI): aws/github-actions/ci.yml" +echo " 4. Fix INC-004 (Terraform): aws/terraform/main.tf" +echo "" +echo "Once INC-004 is fixed, deploy infrastructure:" +echo " cd aws/terraform" +echo " terraform init" +echo " terraform plan -var region=\"$REGION\"" +echo " terraform apply -var region=\"$REGION\"" +echo "" +echo "Then continue with INC-005 through INC-007." +echo "" +echo "Validate progress: ./validate.sh" +echo "Destroy resources: ./destroy.sh" +echo "" diff --git a/aws/scripts/validate.sh b/aws/scripts/validate.sh new file mode 100644 index 0000000..b7abc2d --- /dev/null +++ b/aws/scripts/validate.sh @@ -0,0 +1,415 @@ +#!/usr/bin/env bash +# ============================================================================= +# DEVOPS LAB - VALIDATION SCRIPT +# Validates incident resolution and generates completion tokens +# ============================================================================= + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AWS_DIR="${SCRIPT_DIR}/.." +TERRAFORM_DIR="${AWS_DIR}/terraform" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +INC_001="pending" +INC_002="pending" +INC_003="pending" +INC_004="pending" +INC_005="pending" +INC_006="pending" +INC_007="pending" + +MASTER_SECRET="L2C_CTF_MASTER_2024" + +# ============================================================================= +# INC-001: Dockerfile +# ============================================================================= +validate_inc_001() { + local DOCKERFILE="${AWS_DIR}/docker/Dockerfile" + [ ! -f "$DOCKERFILE" ] && return + + if docker build -f "$DOCKERFILE" -t devops-lab-app-test "${AWS_DIR}" > /dev/null 2>&1; then + local CID + CID=$(docker run -d -p 18000:8000 -e REDIS_HOST=localhost devops-lab-app-test 2>/dev/null || echo "") + if [ -n "$CID" ]; then + sleep 3 + local RESP + RESP=$(curl -s --max-time 5 http://localhost:18000/health 2>/dev/null || echo "") + docker rm -f "$CID" > /dev/null 2>&1 || true + if echo "$RESP" | grep -q "healthy"; then + INC_001="resolved" + fi + fi + docker rmi devops-lab-app-test > /dev/null 2>&1 || true + fi +} + +# ============================================================================= +# INC-002: Docker Compose +# ============================================================================= +validate_inc_002() { + local COMPOSE="${AWS_DIR}/docker/docker-compose.yml" + [ ! -f "$COMPOSE" ] && return + + if ! docker compose -f "$COMPOSE" config > /dev/null 2>&1; then + return + fi + + docker compose -f "$COMPOSE" up -d --build > /dev/null 2>&1 || true + sleep 5 + + local RESP + RESP=$(curl -s --max-time 5 http://localhost:8000/health 2>/dev/null || echo "") + docker compose -f "$COMPOSE" down > /dev/null 2>&1 || true + + if echo "$RESP" | grep -q '"redis":"connected"'; then + INC_002="resolved" + fi +} + +# ============================================================================= +# INC-003: CI Workflow +# ============================================================================= +validate_inc_003() { + local CI="${AWS_DIR}/github-actions/ci.yml" + [ ! -f "$CI" ] && return + + if ! python3 -c "import yaml; yaml.safe_load(open('$CI'))" 2>/dev/null; then + return + fi + + if grep -q "@v99" "$CI" 2>/dev/null; then return; fi + if ! grep -q "runs-on:" "$CI" 2>/dev/null; then return; fi + + local INSTALL_LINE TEST_LINE + INSTALL_LINE=$(grep -n "Install dependencies" "$CI" 2>/dev/null | head -1 | cut -d: -f1) + TEST_LINE=$(grep -n "Run tests" "$CI" 2>/dev/null | head -1 | cut -d: -f1) + + if [ -n "$INSTALL_LINE" ] && [ -n "$TEST_LINE" ]; then + if [ "$TEST_LINE" -lt "$INSTALL_LINE" ]; then return; fi + fi + + INC_003="resolved" +} + +# ============================================================================= +# INC-004: Terraform +# ============================================================================= +validate_inc_004() { + local TF="${AWS_DIR}/terraform" + [ ! -f "${TF}/main.tf" ] && return + + if grep -q "aws_vcp" "${TF}/main.tf" 2>/dev/null; then return; fi + if grep -q "AmazonEKSCluserPolicy" "${TF}/main.tf" 2>/dev/null; then return; fi + if grep -q 'service_ipv4_cidr.*=.*"10\.0\.' "${TF}/main.tf" 2>/dev/null; then return; fi + + cd "$TF" + if terraform init -backend=false > /dev/null 2>&1 && terraform validate > /dev/null 2>&1; then + INC_004="resolved" + fi + cd "$SCRIPT_DIR" +} + +# ============================================================================= +# INC-005: CD Workflow +# ============================================================================= +validate_inc_005() { + local CD="${AWS_DIR}/github-actions/cd.yml" + [ ! -f "$CD" ] && return + + if ! python3 -c "import yaml; yaml.safe_load(open('$CD'))" 2>/dev/null; then return; fi + if ! grep -q "aws-actions/configure-aws-credentials@v4" "$CD" 2>/dev/null; then return; fi + if grep -q "credentials:" "$CD" 2>/dev/null; then return; fi + local HAS_KEYS=false + local HAS_OIDC=false + if grep -q "aws-access-key-id" "$CD" 2>/dev/null && grep -q "aws-secret-access-key" "$CD" 2>/dev/null; then + HAS_KEYS=true + fi + if grep -q "role-to-assume" "$CD" 2>/dev/null; then + HAS_OIDC=true + fi + if [ "$HAS_KEYS" = false ] && [ "$HAS_OIDC" = false ]; then return; fi + if ! grep -q "aws-region" "$CD" 2>/dev/null; then return; fi + if ! grep -q "aws eks update-kubeconfig" "$CD" 2>/dev/null; then return; fi + if ! grep -q "kubectl" "$CD" 2>/dev/null; then return; fi + + INC_005="resolved" +} + +# ============================================================================= +# INC-006: Kubernetes +# ============================================================================= +validate_inc_006() { + local K="${AWS_DIR}/kubernetes" + [ ! -f "${K}/app-deployment.yaml" ] && return + + if grep -q "v1beta1" "${K}/app-deployment.yaml" 2>/dev/null; then return; fi + + local SEL TMPL + SEL=$(grep -A2 "matchLabels" "${K}/app-deployment.yaml" 2>/dev/null | grep "app:" | head -1 | awk '{print $2}') + TMPL=$(sed -n '/template:/,$ p' "${K}/app-deployment.yaml" 2>/dev/null | grep -A2 "labels:" | grep "app:" | head -1 | awk '{print $2}') + if [ "$SEL" != "$TMPL" ]; then return; fi + + local RSEL RTMPL + RSEL=$(grep -A2 "matchLabels" "${K}/redis-deployment.yaml" 2>/dev/null | grep "app:" | head -1 | awk '{print $2}') + RTMPL=$(sed -n '/template:/,$ p' "${K}/redis-deployment.yaml" 2>/dev/null | grep -A2 "labels:" | grep "app:" | head -1 | awk '{print $2}') + if [ "$RSEL" != "$RTMPL" ]; then return; fi + + if grep -q "containerPort: 6380" "${K}/redis-deployment.yaml" 2>/dev/null; then return; fi + if grep -q "ECR_REGISTRY" "${K}/app-deployment.yaml" 2>/dev/null; then return; fi + if grep -q "/ready" "${K}/app-deployment.yaml" 2>/dev/null; then return; fi + if grep -q "containerPort: 5000" "${K}/app-deployment.yaml" 2>/dev/null; then return; fi + + INC_006="resolved" +} + +# ============================================================================= +# INC-007: Monitoring +# ============================================================================= +validate_inc_007() { + local ALERTS="${AWS_DIR}/monitoring/alerts.json" + [ ! -f "$ALERTS" ] && return + + local ENABLED SEVERITY PERIOD + ENABLED=$(python3 -c " +import json +with open('$ALERTS') as f: + data = json.load(f) +for alarm in data.get('alarms', []): + if alarm.get('AlarmName') == 'pod-restart-alarm': + print(alarm.get('ActionsEnabled')) +" 2>/dev/null || echo "false") + + SEVERITY=$(python3 -c " +import json +with open('$ALERTS') as f: + data = json.load(f) +for alarm in data.get('alarms', []): + if alarm.get('AlarmName') == 'pod-restart-alarm': + print(alarm.get('Severity')) +" 2>/dev/null || echo "") + + PERIOD=$(python3 -c " +import json +with open('$ALERTS') as f: + data = json.load(f) +for alarm in data.get('alarms', []): + if alarm.get('AlarmName') == 'pod-restart-alarm': + print(alarm.get('Period')) +" 2>/dev/null || echo "") + + if [ "$ENABLED" = "True" ] && [ "$SEVERITY" = "2" ] && [ "$PERIOD" = "60" ]; then + INC_007="resolved" + fi +} + +# ============================================================================= +# Token Generation +# ============================================================================= +get_deployment_id() { + if [ -f "${TERRAFORM_DIR}/terraform.tfstate" ]; then + cd "$TERRAFORM_DIR" + local DID + DID=$(terraform output -raw deployment_id 2>/dev/null || echo "") + cd "$SCRIPT_DIR" + if [ -n "$DID" ]; then + echo "$DID" + return + fi + fi + echo "local-$(date +%s | shasum -a 256 | head -c 16)" +} + +generate_verification_token() { + local GITHUB_USER="$1" + local DEPLOYMENT_ID + DEPLOYMENT_ID=$(get_deployment_id) + + local TIMESTAMP=$(date +%s) + local COMPLETION_DATE=$(date -u +"%Y-%m-%d") + local COMPLETION_TIME=$(date -u +"%H:%M:%S") + + local VERIFICATION_SECRET + VERIFICATION_SECRET=$(echo -n "${MASTER_SECRET}:${DEPLOYMENT_ID}" | shasum -a 256 | cut -d' ' -f1) + + local PAYLOAD='{"github_username":"'"$GITHUB_USER"'","date":"'"$COMPLETION_DATE"'","time":"'"$COMPLETION_TIME"'","timestamp":'"$TIMESTAMP"',"challenge":"devops-lab-aws","challenges":7,"instance_id":"'"$DEPLOYMENT_ID"'"}' + + local SIGNATURE + SIGNATURE=$(echo -n "$PAYLOAD" | openssl dgst -sha256 -hmac "$VERIFICATION_SECRET" | sed 's/^.* //') + + local TOKEN_DATA='{"payload":'"$PAYLOAD"',"signature":"'"$SIGNATURE"'"}' + echo -n "$TOKEN_DATA" | base64 +} + +# ============================================================================= +# Display +# ============================================================================= +show_status() { + echo "" + echo "============================================" + echo " DevOps Lab - Incident Status" + echo "============================================" + + local RESOLVED=0 + + for INC_VAR in INC_001 INC_002 INC_003 INC_004 INC_005 INC_006 INC_007; do + local NUM="${INC_VAR#INC_}" + local LABEL="" + case "$NUM" in + 001) LABEL="Dockerfile" ;; + 002) LABEL="Docker Compose" ;; + 003) LABEL="CI Pipeline" ;; + 004) LABEL="Terraform" ;; + 005) LABEL="CD Pipeline" ;; + 006) LABEL="Kubernetes" ;; + 007) LABEL="Monitoring" ;; + esac + + local STATUS="${!INC_VAR}" + if [ "$STATUS" = "resolved" ]; then + echo -e " ${GREEN}✓${NC} INC-${NUM} - ${LABEL}" + RESOLVED=$((RESOLVED + 1)) + else + echo -e " ${RED}✗${NC} INC-${NUM} - ${LABEL}" + fi + done + + echo "" + echo " Resolved: $RESOLVED / 7" + echo "" + + if [ $RESOLVED -eq 7 ]; then + echo -e "${GREEN}============================================${NC}" + echo -e "${GREEN} ALL INCIDENTS RESOLVED${NC}" + echo -e "${GREEN}============================================${NC}" + echo "" + echo -e " Run ${CYAN}./validate.sh export${NC} to generate" + echo " your completion token." + echo "" + fi +} + +export_token() { + validate_inc_001 + validate_inc_002 + validate_inc_003 + validate_inc_004 + validate_inc_005 + validate_inc_006 + validate_inc_007 + + local RESOLVED=0 + for INC_VAR in INC_001 INC_002 INC_003 INC_004 INC_005 INC_006 INC_007; do + [ "${!INC_VAR}" = "resolved" ] && RESOLVED=$((RESOLVED + 1)) + done + + if [ $RESOLVED -ne 7 ]; then + show_status + echo -e "${RED}Error: Not all incidents resolved.${NC}" + exit 1 + fi + + echo "" + echo -e "${GREEN}============================================${NC}" + echo -e "${GREEN} DEVOPS LAB - EXPORT TOKEN${NC}" + echo -e "${GREEN}============================================${NC}" + echo "" + echo "Enter your GitHub username:" + echo -n "> " + read GITHUB_USER + + if [ -z "$GITHUB_USER" ]; then + echo -e "${RED}Error: GitHub username required.${NC}" + exit 1 + fi + + echo "" + echo "Generating completion token..." + echo "" + + local TOKEN + TOKEN=$(generate_verification_token "$GITHUB_USER") + + echo -e "${GREEN}Your completion token:${NC}" + echo "" + echo "--- BEGIN L2C DEVOPS LAB TOKEN ---" + echo "$TOKEN" + echo "--- END L2C DEVOPS LAB TOKEN ---" + echo "" + echo "Token details:" + echo " GitHub User: $GITHUB_USER" + echo " Challenge: devops-lab-aws" + echo " Completed: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + echo "" + echo -e "${CYAN}Submit this token at: https://learntocloud.guide${NC}" + echo "" +} + +verify_token() { + local TOKEN="$1" + [ -z "$TOKEN" ] && echo "Usage: $0 verify " && exit 1 + + echo "" + echo "Verifying token..." + + local DECODED + DECODED=$(echo "$TOKEN" | base64 -d 2>/dev/null || echo "") + [ -z "$DECODED" ] && echo -e "${RED}Error: Invalid token.${NC}" && exit 1 + + local PAYLOAD PROVIDED_SIG INSTANCE_ID + PAYLOAD=$(echo "$DECODED" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps(d['payload'],separators=(',',':')))" 2>/dev/null) + PROVIDED_SIG=$(echo "$DECODED" | python3 -c "import sys,json; print(json.load(sys.stdin)['signature'])" 2>/dev/null) + INSTANCE_ID=$(echo "$DECODED" | python3 -c "import sys,json; print(json.load(sys.stdin)['payload']['instance_id'])" 2>/dev/null) + + [ -z "$PAYLOAD" ] || [ -z "$PROVIDED_SIG" ] && echo -e "${RED}Error: Parse failed.${NC}" && exit 1 + + local VSECRET EXPECTED_SIG + VSECRET=$(echo -n "${MASTER_SECRET}:${INSTANCE_ID}" | shasum -a 256 | cut -d' ' -f1) + EXPECTED_SIG=$(echo -n "$PAYLOAD" | openssl dgst -sha256 -hmac "$VSECRET" | sed 's/^.* //') + + if [ "$PROVIDED_SIG" = "$EXPECTED_SIG" ]; then + echo -e "${GREEN}✓ Token is VALID${NC}" + echo "" + echo "$DECODED" | python3 -c "import sys,json; d=json.load(sys.stdin); [print(f' {k}: {v}') for k,v in d['payload'].items()]" + else + echo -e "${RED}✗ Token is INVALID${NC}" + exit 1 + fi + echo "" +} + +# ============================================================================= +# Main +# ============================================================================= +main() { + local CMD="${1:-status}" + case "$CMD" in + status|all) + validate_inc_001 + validate_inc_002 + validate_inc_003 + validate_inc_004 + validate_inc_005 + validate_inc_006 + validate_inc_007 + show_status + ;; + export) + export_token + ;; + verify) + verify_token "$2" + ;; + *) + echo "Usage: $0 [status|export|verify ]" + exit 1 + ;; + esac +} + +main "$@" diff --git a/aws/terraform/main.tf b/aws/terraform/main.tf new file mode 100644 index 0000000..8f6140b --- /dev/null +++ b/aws/terraform/main.tf @@ -0,0 +1,142 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} + +provider "aws" { + region = var.region +} + +resource "random_id" "deployment" { + byte_length = 4 +} + +resource "aws_vcp" "main" { + cidr_block = "10.0.0.0/16" + tags = { + Name = "vpc-devopslab-${random_id.deployment.hex}" + } +} + +resource "aws_subnet" "eks_a" { + vpc_id = aws_vcp.main.id + cidr_block = "10.0.1.0/24" + availability_zone = "${var.region}a" + tags = { + Name = "snet-eks-a" + } +} + +resource "aws_subnet" "eks_b" { + vpc_id = aws_vcp.main.id + cidr_block = "10.0.2.0/24" + availability_zone = "${var.region}b" + tags = { + Name = "snet-eks-b" + } +} + +resource "aws_ecr_repository" "main" { + name = "devopslab-${random_id.deployment.hex}" +} + +resource "aws_cloudwatch_log_group" "eks" { + name = "/aws/eks/devopslab-${random_id.deployment.hex}" + retention_in_days = 30 +} + +resource "aws_iam_role" "eks_cluster" { + name = "eks-cluster-role-${random_id.deployment.hex}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "eks.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "eks_cluster_policy" { + role = aws_iam_role.eks_cluster.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSCluserPolicy" +} + +resource "aws_iam_role" "eks_node" { + name = "eks-node-role-${random_id.deployment.hex}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "ec2.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "eks_node_policy" { + role = aws_iam_role.eks_node.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" +} + +resource "aws_iam_role_policy_attachment" "eks_cni_policy" { + role = aws_iam_role.eks_node.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" +} + +resource "aws_iam_role_policy_attachment" "eks_ecr_policy" { + role = aws_iam_role.eks_node.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +} + +resource "aws_eks_cluster" "main" { + name = "eks-devopslab-${random_id.deployment.hex}" + role_arn = aws_iam_role.eks_cluster.arn + + vpc_config { + subnet_ids = [aws_subnet.eks_a.id, aws_subnet.eks_b.id] + } + + kubernetes_network_config { + service_ipv4_cidr = "10.0.0.0/16" + } +} + +resource "aws_eks_node_group" "main" { + cluster_name = aws_eks_cluster.main.name + node_group_name = "ng-devopslab-${random_id.deployment.hex}" + node_role_arn = aws_iam_role.eks_node.arn + subnet_ids = [aws_subnet.eks_a.id, aws_subnet.eks_b.id] + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + instance_types = ["t3.medium"] + + depends_on = [ + aws_iam_role_policy_attachment.eks_node_policy, + aws_iam_role_policy_attachment.eks_cni_policy, + aws_iam_role_policy_attachment.eks_ecr_policy + ] +} diff --git a/aws/terraform/outputs.tf b/aws/terraform/outputs.tf new file mode 100644 index 0000000..20c869a --- /dev/null +++ b/aws/terraform/outputs.tf @@ -0,0 +1,23 @@ +output "vpc_id" { + value = aws_vcp.main.id +} + +output "ecr_repository_url" { + value = aws_ecr_repository.main.repository_url +} + +output "ecr_repository_name" { + value = aws_ecr_repository.main.name +} + +output "eks_cluster_name" { + value = aws_eks_cluster.main.name +} + +output "deployment_id" { + value = random_id.deployment.hex +} + +output "log_group_name" { + value = aws_cloudwatch_log_group.eks.name +} diff --git a/aws/terraform/variables.tf b/aws/terraform/variables.tf new file mode 100644 index 0000000..2ccfc46 --- /dev/null +++ b/aws/terraform/variables.tf @@ -0,0 +1,5 @@ +variable "region" { + description = "AWS region to deploy resources" + type = string + default = "us-east-1" +}