feat(ansible): add init and activate scripts for virtual environment setup

chore: add .gitignore for virtual environment files
refactor: move etcd configuration to manifests and remove obsolete files

Signed-off-by: 孙振宇 <>

.gitignore

@@ -0,0 +1 @@
+cluster/ansible/venv

cluster/ansible/activate.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# Check if `init.sh` has not run
+if [ ! -d venv ]; then
+    echo "Please run init.sh first."
+    exit 1
+fi
+
+# Allows venv/bin/activate has permission to run
+chmod +x venv/bin/activate
+
+# Activate virtual env
+source venv/bin/activate

cluster/ansible/init.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# Check if there has no Python 3 installed
+if ! command -v python3 &> /dev/null; then
+    echo "Python3 is not installed. Please install python3 first."
+    exit 1
+fi
+
+# Check if there has no virtualenv installed
+if ! command -v virtualenv &> /dev/null; then
+    echo "Virtualenv is not installed. Please install virtualenv first."
+    exit 1
+fi
+
+# Create virtual env in current workspace
+virtualenv venv --python=python3 --prompt="(freeleaps-cluster-maintain)" --clear 
+
+# Install requirements with ../../3rd/kubespary/requirements.txt
+source venv/bin/activate
+pip install -r ../../3rd/kubespray/requirements.txt
+deactivate

cluster/ansible/manifests/group_vars/all/all.yml

@@ -90,7 +90,7 @@ cert_management: script
 # kube_read_only_port: 10255
 
 ## Set true to download and cache container
-# download_container: true
+download_container: true
 
 ## Deploy container engine
 # Set false if you want to deploy container engine manually.

cluster/ansible/manifests/group_vars/all/docker.yml

@@ -0,0 +1,59 @@
+---
+## Uncomment this if you want to force overlay/overlay2 as docker storage driver
+## Please note that overlay2 is only supported on newer kernels
+# docker_storage_options: -s overlay2
+
+## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7.
+docker_container_storage_setup: false
+
+## It must be define a disk path for docker_container_storage_setup_devs.
+## Otherwise docker-storage-setup will be executed incorrectly.
+# docker_container_storage_setup_devs: /dev/vdb
+
+## Uncomment this if you want to change the Docker Cgroup driver (native.cgroupdriver)
+## Valid options are systemd or cgroupfs, default is systemd
+# docker_cgroup_driver: systemd
+
+## Only set this if you have more than 3 nameservers:
+## If true Kubespray will only use the first 3, otherwise it will fail
+docker_dns_servers_strict: false
+
+# Path used to store Docker data
+docker_daemon_graph: "/var/lib/docker"
+
+## Used to set docker daemon iptables options to true
+docker_iptables_enabled: "false"
+
+# Docker log options
+# Rotate container stderr/stdout logs at 50m and keep last 5
+docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5"
+
+# define docker bin_dir
+docker_bin_dir: "/usr/bin"
+
+# keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'
+# kubespray deletes the docker package on each run, so caching the package makes sense
+docker_rpm_keepcache: 1
+
+## An obvious use case is allowing insecure-registry access to self hosted registries.
+## Can be ipaddress and domain_name.
+## example define 172.19.16.11 or mirror.registry.io
+# docker_insecure_registries:
+#   - mirror.registry.io
+#   - 172.19.16.11
+
+## Add other registry,example China registry mirror.
+# docker_registry_mirrors:
+#   - https://registry.docker-cn.com
+#   - https://mirror.aliyuncs.com
+
+## If non-empty will override default system MountFlags value.
+## This option takes a mount propagation flag: shared, slave
+## or private, which control whether mounts in the file system
+## namespace set up for docker will receive or propagate mounts
+## and unmounts. Leave empty for system default
+# docker_mount_flags:
+
+## A string of extra options to pass to the docker daemon.
+## This string should be exactly as you wish it to appear.
+# docker_options: ""

cluster/ansible/manifests/group_vars/all/etcd.yml

@@ -9,7 +9,7 @@ etcd_data_dir: /var/lib/etcd
 ## If this is not set, container manager will be inherited from the Kubespray defaults
 ## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want.
 ## Also this makes possible to use different container manager for etcd nodes.
-# container_manager: containerd
+container_manager: docker
 
 ## Settings for etcd deployment type
 # Set this to docker if you are using container_manager: docker

cluster/ansible/manifests/group_vars/k8s_cluster/k8s-cluster.yml

@@ -222,7 +222,7 @@ dns_domain: "{{ cluster_name }}"
 ## Container runtime
 ## docker for docker, crio for cri-o and containerd for containerd.
 ## Default: containerd
-container_manager: containerd
+container_manager: docker
 
 # Additional container runtimes
 kata_containers_enabled: false
@@ -240,11 +240,11 @@ kubernetes_audit: true
 default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
 
 # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
-# kubeconfig_localhost: false
+kubeconfig_localhost: true
 # Use ansible_host as external api ip when copying over kubeconfig.
 # kubeconfig_localhost_ansible_host: false
 # Download kubectl onto the host that runs Ansible in {{ bin_dir }}
-# kubectl_localhost: false
+kubectl_localhost: true
 
 # A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.
 # Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "".

cluster/ansible/manifests/inventory.ini

@@ -0,0 +1,19 @@
+[bastion]
+bastion ansible_host=jumper.mathmast.com ansible_user=zhenyus@mathmast.com
+
+[kube_control_plane]
+prod-usw2-k8s-freeleaps-master-01 ansible_host=prod-usw2-k8s-freeleaps-master-01.mathmast.com ansible_user=zhenyus@mathmast.com etcd_member_name=freeleaps-etcd-01
+prod-usw2-k8s-freeleaps-master-02 ansible_host=prod-usw2-k8s-freeleaps-master-02.mathmast.com ansible_user=zhenyus@mathmast.com etcd_member_name=freeleaps-etcd-02
+prod-usw2-k8s-freeleaps-master-03 ansible_host=prod-usw2-k8s-freeleaps-master-03.mathmast.com ansible_user=zhenyus@mathmast.com etcd_member_name=freeleaps-etcd-03
+
+[etcd:children]
+kube_control_plane
+
+[kube_node]
+prod-usw2-k8s-freeleaps-worker-01 ansible_host=prod-usw2-k8s-freeleaps-worker-01.mathmast.com ansible_user=zhenyus@mathmast.com
+prod-usw2-k8s-freeleaps-worker-02 ansible_host=prod-usw2-k8s-freeleaps-worker-02.mathmast.com ansible_user=zhenyus@mathmast.com
+prod-usw2-k8s-freeleaps-worker-03 ansible_host=prod-usw2-k8s-freeleaps-worker-03.mathmast.com ansible_user=zhenyus@mathmast.com
+prod-usw2-k8s-freeleaps-worker-04 ansible_host=prod-usw2-k8s-freeleaps-worker-04.mathmast.com ansible_user=zhenyus@mathmast.com
+prod-usw2-k8s-freeleaps-ingress-worker-01 ansible_host=prod-usw2-k8s-freeleaps-ingress-worker-01.mathmast.com ansible_user=zhenyus@mathmast.com
+prod-usw2-k8s-freeleaps-ingress-worker-02 ansible_host=prod-usw2-k8s-freeleaps-ingress-worker-02.mathmast.com ansible_user=zhenyus@mathmast.com
+prod-usw2-k8s-freeleaps-ingress-worker-03 ansible_host=prod-usw2-k8s-freeleaps-ingress-worker-03.mathmast.com ansible_user=zhenyus@mathmast.com

cluster/group_vars/all/containerd.yml

@@ -1,59 +0,0 @@
----
-# Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
-
-# containerd_storage_dir: "/var/lib/containerd"
-# containerd_state_dir: "/run/containerd"
-# containerd_oom_score: 0
-
-# containerd_default_runtime: "runc"
-# containerd_snapshotter: "native"
-
-# containerd_runc_runtime:
-#   name: runc
-#   type: "io.containerd.runc.v2"
-#   engine: ""
-#   root: ""
-
-# containerd_additional_runtimes:
-# Example for Kata Containers as additional runtime:
-#   - name: kata
-#     type: "io.containerd.kata.v2"
-#     engine: ""
-#     root: ""
-
-# containerd_grpc_max_recv_message_size: 16777216
-# containerd_grpc_max_send_message_size: 16777216
-
-# Containerd debug socket location: unix or tcp format
-# containerd_debug_address: ""
-
-# Containerd log level
-# containerd_debug_level: "info"
-
-# Containerd logs format, supported values: text, json
-# containerd_debug_format: ""
-
-# Containerd debug socket UID
-# containerd_debug_uid: 0
-
-# Containerd debug socket GID
-# containerd_debug_gid: 0
-
-# containerd_metrics_address: ""
-
-# containerd_metrics_grpc_histogram: false
-
-# Registries defined within containerd.
-# containerd_registries_mirrors:
-#  - prefix: docker.io
-#    mirrors:
-#     - host: https://registry-1.docker.io
-#       capabilities: ["pull", "resolve"]
-#       skip_verify: false
-
-# containerd_max_container_log_line_size: 16384
-
-# containerd_registry_auth:
-#   - registry: 10.0.0.2:5000
-#     username: user
-#     password: pass

cluster/inventory.ini

@@ -1,8 +0,0 @@
-# Configure 'ip' variable to bind kubernetes services on a different ip than the default iface
-# We should set etcd_member_name for etcd cluster. The node that are not etcd members do not need to set the value,
-# or can set the empty string value.
-[kube_control_plane]
-
-[etcd:children]
-
-[kube_node]