Sync Secrets with External Secrets Operator

# Create the secret in AWS Secrets Manager
aws secretsmanager create-secret \
  --name "prod/myapp/database" \
  --description "Database credentials for myapp" \
  --secret-string '{
    "username": "app_user",
    "password": "initial-password-123",
    "host": "db.example.com",
    "port": "5432",
    "database": "myapp_prod"
  }' \
  --region us-east-1

# Note the ARN — you'll need it for the SecretStore
aws secretsmanager describe-secret \
  --secret-id "prod/myapp/database" \
  --query 'ARN' --output text

# Add ESO Helm repository
helm repo add external-secrets https://charts.external-secrets.io
helm repo update

# Install ESO in its own namespace
helm install external-secrets \
  external-secrets/external-secrets \
  --namespace external-secrets \
  --create-namespace \
  --set installCRDs=true \
  --set webhook.port=9443 \
  --wait

# Verify ESO is running
kubectl get pods -n external-secrets
kubectl get crd | grep external-secrets
# Should see: externalsecrets.external-secrets.io
#             secretstores.external-secrets.io
#             clustersecretstores.external-secrets.io

# Create IAM policy allowing read access to specific secrets
cat > eso-policy.json << 'EOF'
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "secretsmanager:GetSecretValue",
        "secretsmanager:DescribeSecret"
      ],
      "Resource": [
        "arn:aws:secretsmanager:us-east-1:ACCOUNT_ID:secret:prod/myapp/*"
      ]
    }
  ]
}
EOF

aws iam create-policy \
  --policy-name ESOSecretsReadPolicy \
  --policy-document file://eso-policy.json

# Create IAM role with trust policy for ESO ServiceAccount
# (Replace ACCOUNT_ID and OIDC_PROVIDER with your values)
OIDC_PROVIDER=$(aws eks describe-cluster \
  --name my-cluster \
  --query "cluster.identity.oidc.issuer" \
  --output text | sed 's|https://||')

cat > trust-policy.json << EOF
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Principal": {
      "Federated": "arn:aws:iam::ACCOUNT_ID:oidc-provider/${OIDC_PROVIDER}"
    },
    "Action": "sts:AssumeRoleWithWebIdentity",
    "Condition": {
      "StringEquals": {
        "${OIDC_PROVIDER}:sub": "system:serviceaccount:external-secrets:external-secrets"
      }
    }
  }]
}
EOF

aws iam create-role \
  --role-name ESORole \
  --assume-role-policy-document file://trust-policy.json

aws iam attach-role-policy \
  --role-name ESORole \
  --policy-arn arn:aws:iam::ACCOUNT_ID:policy/ESOSecretsReadPolicy

# secret-store.yaml
cat << 'EOF' | kubectl apply -f -
apiVersion: external-secrets.io/v1beta1
kind: SecretStore
metadata:
  name: aws-secretsmanager
  namespace: default
spec:
  provider:
    aws:
      service: SecretsManager
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: external-secrets   # ESO service account with IRSA
            namespace: external-secrets
EOF

# Check SecretStore status
kubectl get secretstore aws-secretsmanager -n default
# STATUS should be: Valid
kubectl describe secretstore aws-secretsmanager -n default

# external-secret.yaml
cat << 'EOF' | kubectl apply -f -
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
  namespace: default
spec:
  refreshInterval: 1h        # Check for changes every hour
  secretStoreRef:
    name: aws-secretsmanager
    kind: SecretStore
  target:
    name: db-credentials     # Name of the K8s Secret to create
    creationPolicy: Owner    # ESO owns and manages this Secret
    deletionPolicy: Delete   # Delete K8s Secret when ExternalSecret is deleted
    template:
      type: Opaque
      metadata:
        labels:
          managed-by: external-secrets
  data:
  - secretKey: db-username   # Key in the K8s Secret
    remoteRef:
      key: prod/myapp/database   # AWS Secrets Manager secret name
      property: username         # JSON field within the secret
  - secretKey: db-password
    remoteRef:
      key: prod/myapp/database
      property: password
  - secretKey: db-host
    remoteRef:
      key: prod/myapp/database
      property: host
EOF

# Wait for sync and check status
kubectl wait externalsecret db-credentials \
  --for=condition=Ready --timeout=60s

kubectl get externalsecret db-credentials
# READY: True  SYNCED: True

# Verify the K8s Secret exists and has the correct keys
kubectl get secret db-credentials -o jsonpath='{.data}' | \
  python3 -c "
import sys, json, base64
data = json.load(sys.stdin)
for k, v in data.items():
    print(f'{k}: {base64.b64decode(v).decode()}')"

# Expected output:
# db-host: db.example.com
# db-password: initial-password-123
# db-username: app_user

# Verify the Secret is owned by the ExternalSecret
kubectl get secret db-credentials -o jsonpath='{.metadata.ownerReferences}'

# Rotate the password in Secrets Manager
aws secretsmanager put-secret-value \
  --secret-id "prod/myapp/database" \
  --secret-string '{
    "username": "app_user",
    "password": "rotated-password-456",
    "host": "db.example.com",
    "port": "5432",
    "database": "myapp_prod"
  }'

# Force immediate re-sync (don't wait for refreshInterval)
kubectl annotate externalsecret db-credentials \
  force-sync=$(date +%s) --overwrite

# Wait a moment for sync
sleep 10

# Verify the K8s Secret has the new password
kubectl get secret db-credentials \
  -o jsonpath='{.data.db-password}' | base64 -d
# Expected: rotated-password-456

# Check ESO status for sync time
kubectl describe externalsecret db-credentials | grep -A10 "Status"

# azure-secret-store.yaml (AKS with Workload Identity)
apiVersion: external-secrets.io/v1beta1
kind: SecretStore
metadata:
  name: azure-key-vault
spec:
  provider:
    azurekv:
      authType: WorkloadIdentity
      vaultUrl: "https://my-keyvault.vault.azure.net"
      serviceAccountRef:
        name: external-secrets
        namespace: external-secrets
---
# azure-external-secret.yaml
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: azure-key-vault
    kind: SecretStore
  target:
    name: db-credentials
  data:
  - secretKey: db-password
    remoteRef:
      key: myapp-db-password   # Key Vault secret name