Canary k8s deployments with Flagger
# scripts to to run anc create a test/sample canary deployment
kubectl create ns ingress-nginx
helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \
--namespace ingress-nginx \
--set controller.metrics.enabled=true \
--set controller.podAnnotations."prometheus\.io/scrape"=true \
--set controller.podAnnotations."prometheus\.io/port"=10254
# metrics-server
helm upgrade --install metrics-server metrics-server/metrics-server \
--namespace kube-system \
--set args.0="--kubelet-insecure-tls"
# flagger
helm upgrade --install flagger flagger/flagger \
--namespace ingress-nginx \
--set prometheus.install=true \
--set meshProvider=nginx
# monitor canaries
watch kubectl get canaries --all-namespaces
# install test http-echo app
k create namespace localhost
kcn localhost
k apply -n localhost -f http-echo-manifest.yaml
# flagger load tester to load test the app
helm upgrade --install flagger-loadtester flagger/loadtester \
--namespace=localhost# http-echo-manifest.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: http-echo
labels:
app: http-echo
spec:
replicas: 1
selector:
matchLabels:
app: http-echo
template:
metadata:
labels:
app: http-echo
spec:
containers:
- name: http-echo
image: mendhak/http-https-echo:34
ports:
- containerPort: 8080
resources:
requests:
cpu: 5m
---
apiVersion: v1
kind: Service
metadata:
name: http-echo
labels:
app: http-echo
spec:
type: ClusterIP
ports:
- port: 80
targetPort: 8080
name: http
selector:
app: http-echo
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: http-echo
labels:
app: http-echo
spec:
ingressClassName: nginx
rules:
- http:
paths:
- path: "/ping"
pathType: Exact
backend:
service:
name: http-echo
port:
number: 80
host: http-echo.localhost
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: http-echo
labels:
app: http-echo
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: http-echo
minReplicas: 2
maxReplicas: 4
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
# scale up if usage is above
# 99% of the requested CPU (10m)
averageUtilization: 99
---
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: http-echo
spec:
provider: nginx
# deployment reference
targetRef:
apiVersion: apps/v1
kind: Deployment
name: http-echo
# ingress reference
ingressRef:
apiVersion: networking.k8s.io/v1
kind: Ingress
name: http-echo
# HPA reference (optional)
autoscalerRef:
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
name: http-echo
# the maximum time in seconds for the canary deployment
# to make progress before it is rollback (default 600s)
progressDeadlineSeconds: 60
service:
# ClusterIP port number
port: 80
# container port number or name
targetPort: 8080
analysis:
# schedule interval (default 60s)
interval: 10s
# max number of failed metric checks before rollback
threshold: 10
# max traffic percentage routed to canary
# percentage (0-100)
maxWeight: 50
# canary increment step
# percentage (0-100)
stepWeight: 5
# NGINX Prometheus checks
metrics:
- name: request-success-rate
# minimum req success rate (non 5xx responses)
# percentage (0-100)
thresholdRange:
min: 99
interval: 1m
# testing (optional)
webhooks:
- name: acceptance-test
type: pre-rollout
url: http://flagger-loadtester.localhost/
timeout: 30s
metadata:
type: bash
cmd: "curl -sd 'test' http://http-echo-canary/ping | grep ping"
- name: load-test
url: http://flagger-loadtester.localhost/
timeout: 5s
metadata:
cmd: "hey -z 1m -q 3 -c 1 http://http-echo.localhost/ping"