Kubernetes 1.36 叢集部署（CRI-O + Calico / Rocky Linux 9）

# 0. 全部節點：前置環境（swap / SELinux / 核心模組 / sysctl）後重開機
curl -fsSL https://raw.githubusercontent.com/CTJ425/script-docs/main/container/k8s_env_init/k8s_env_initialization.sh | sudo bash -s -- --yes

# 1. Control Plane：安裝 CRI-O + kubeadm 並初始化，含 Calico
curl -fsSL https://raw.githubusercontent.com/CTJ425/script-docs/main/container/k8s_install/k8s_cluster_install.sh | sudo bash -s -- \
  --role cp --apiserver-advertise-address 10.0.1.10 --yes

# 2. 每個 Worker：貼上 CP 最後印出的 join 指令
curl -fsSL https://raw.githubusercontent.com/CTJ425/script-docs/main/container/k8s_install/k8s_cluster_install.sh | sudo bash -s -- \
  --role worker --yes \
  --join "kubeadm join 10.0.1.10:6443 --token <TOKEN> --discovery-token-ca-cert-hash sha256:<HASH>"

kubectl get nodes -o wide
kubectl get pods -A
kubectl get tigerastatus

sudo hostnamectl set-hostname k8s-cp01     # 依節點調整

cat <<EOF | sudo tee -a /etc/hosts
10.0.1.10 k8s-cp01
10.0.1.11 k8s-wk01
10.0.1.12 k8s-wk02
EOF

# swap
sudo swapoff -a
sudo sed -r -i '/\s+swap\s+/s/^#*/#/' /etc/fstab

# SELinux（permissive 或用 selinux=0 核心參數）
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config

# 核心模組
cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
overlay
br_netfilter
EOF
sudo modprobe overlay && sudo modprobe br_netfilter

# sysctl
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sudo sysctl --system

# 時間同步
sudo dnf install -y chrony && sudo systemctl enable --now chronyd

# Control Plane
sudo firewall-cmd --permanent --add-port={6443,2379-2380,10250-10259}/tcp
# Worker
sudo firewall-cmd --permanent --add-port={10250,10256}/tcp --add-port=30000-32767/tcp
# 全部節點（Calico VXLAN）
sudo firewall-cmd --permanent --add-port=4789/udp
sudo firewall-cmd --reload

export CRIO_VERSION=v1.36

cat <<EOF | sudo tee /etc/yum.repos.d/cri-o.repo
[cri-o]
name=CRI-O
baseurl=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/$CRIO_VERSION/rpm/
enabled=1
gpgcheck=1
gpgkey=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/$CRIO_VERSION/rpm/repodata/repomd.xml.key
EOF

sudo dnf install -y cri-o

sudo mkdir -p /etc/crio/crio.conf.d
cat <<EOF | sudo tee /etc/crio/crio.conf.d/02-cgroup-manager.conf
[crio.runtime]
cgroup_manager = "systemd"
conmon_cgroup = "pod"
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now crio

export KUBERNETES_VERSION=v1.36

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF

sudo dnf install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable --now kubelet

kubeadm version && kubectl version --client && crictl --version && crio --version

sudo kubeadm init \
  --pod-network-cidr=10.244.0.0/16 \
  --apiserver-advertise-address=10.0.1.10 \
  --cri-socket=unix:///var/run/crio/crio.sock \
  --kubernetes-version=v1.36.2

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubeadm token create --print-join-command

sudo kubeadm join 10.0.1.10:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH> \
  --cri-socket=unix:///var/run/crio/crio.sock

CALICO_VERSION=v3.32.1

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/v1_crd_projectcalico_org.yaml
kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/tigera-operator.yaml

kubectl wait --for=condition=Available deployment/tigera-operator -n tigera-operator --timeout=120s

curl -O https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/custom-resources.yaml
sed -i 's#cidr: 192\.168\.0\.0/16#cidr: 10.244.0.0/16#' custom-resources.yaml
kubectl create -f custom-resources.yaml

kubectl wait --for=condition=Available tigerastatus/calico --timeout=300s
kubectl get pods -n calico-system

kubectl get installation default -o jsonpath='{.status.calicoVersion}'; echo

curl -L https://github.com/projectcalico/calico/releases/download/${CALICO_VERSION}/calicoctl-linux-amd64 -o calicoctl
chmod +x calicoctl && sudo mv calicoctl /usr/local/bin/

METALLB_VERSION=v0.16.1
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/config/manifests/metallb-native.yaml

kubectl wait --namespace metallb-system --for=condition=ready pod \
  --selector=app=metallb --timeout=120s

cat <<EOF | kubectl apply -f -
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: default-pool
  namespace: metallb-system
spec:
  addresses:
  - 10.0.1.200-10.0.1.220
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: default-l2
  namespace: metallb-system
spec:
  ipAddressPools:
  - default-pool
EOF

kubectl create deployment nginx-demo --image=nginx --port=80
kubectl expose deployment nginx-demo --port=80 --type=LoadBalancer --name=nginx-lb
kubectl get svc nginx-lb -w        # 等 EXTERNAL-IP 從 <pending> 變成實際 IP
curl http://10.0.1.200

kubectl delete svc nginx-lb && kubectl delete deployment nginx-demo

kubectl get configmap kube-proxy -n kube-system -o yaml | \
  sed -e "s/strictARP: false/strictARP: true/" | kubectl apply -f - -n kube-system
kubectl rollout restart daemonset kube-proxy -n kube-system

# 確認 CPU 虛擬化擴充（實體機需在 BIOS 開 VT-x/AMD-V；節點本身是 VM 則需開巢狀虛擬化）
lscpu | grep -E 'Virtualization'

# 載入 KVM 模組並確認 /dev/kvm 存在
sudo modprobe kvm
sudo modprobe kvm_intel        # AMD 用 kvm_amd
ls -l /dev/kvm

# （選用）驗證環境
sudo dnf install -y qemu-kvm libvirt virt-host-validate
sudo virt-host-validate qemu

export KUBEVIRT_VERSION=v1.8.4

kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml
kubectl wait --for=condition=Ready pod -l kubevirt.io=virt-operator -n kubevirt --timeout=120s

kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml
kubectl -n kubevirt wait kv kubevirt --for condition=Available --timeout=300s

kubectl get pods -n kubevirt

ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
curl -L -o virtctl \
  https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/virtctl-${KUBEVIRT_VERSION}-linux-${ARCH}
sudo install -m 0755 virtctl /usr/local/bin/virtctl
virtctl version

cat <<EOF | kubectl apply -f -
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  name: testvm
  namespace: default
spec:
  running: false
  template:
    metadata:
      labels:
        kubevirt.io/domain: testvm
    spec:
      domain:
        cpu:
          cores: 1
        resources:
          requests:
            memory: 512Mi
        devices:
          disks:
            - name: containerdisk
              disk:
                bus: virtio
            - name: cloudinitdisk
              disk:
                bus: virtio
      volumes:
        - name: containerdisk
          containerDisk:
            image: quay.io/containerdisks/cirros:latest
        - name: cloudinitdisk
          cloudInitNoCloud:
            userData: |
              #cloud-config
              password: kubevirt
              chpasswd: { expire: False }
EOF

virtctl start testvm
kubectl get vm,vmi
virtctl console testvm      # 離開按 Ctrl+]

virtctl stop testvm && kubectl delete vm testvm

# 1. Gateway API CRDs（Standard channel）
kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.0/standard-install.yaml
kubectl get crd | grep gateway.networking.k8s.io

# 2. 啟用 Calico Ingress Gateway，Operator 會自動建立 tigera-gateway-class
cat <<EOF | kubectl apply -f -
apiVersion: operator.tigera.io/v1
kind: GatewayAPI
metadata:
  name: default
EOF

kubectl get tigerastatus gateway-api-connectivity -w    # 約 1~2 分鐘
kubectl get gatewayclass

kubectl create deployment web-demo --image=nginx --port=80
kubectl expose deployment web-demo --port=80

cat <<EOF | kubectl apply -f -
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: demo-gateway
  namespace: default
spec:
  gatewayClassName: tigera-gateway-class
  listeners:
  - name: http
    port: 80
    protocol: HTTP
    allowedRoutes:
      namespaces:
        from: Same
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: web-demo-route
  namespace: default
spec:
  parentRefs:
  - name: demo-gateway
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /
    backendRefs:
    - name: web-demo
      port: 80
EOF

kubectl wait --for=condition=Programmed gateway/demo-gateway --timeout=120s
kubectl get gateway demo-gateway -o jsonpath='{.status.addresses[0].value}'; echo

sudo dnf install -y iscsi-initiator-utils
sudo systemctl enable --now iscsid

BASE=https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master
kubectl apply -f $BASE/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
kubectl apply -f $BASE/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
kubectl apply -f $BASE/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
kubectl apply -f $BASE/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
kubectl apply -f $BASE/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml

curl -O https://raw.githubusercontent.com/truenas/truenas-csi/master/deploy/truenas-csi-driver.yaml

truenasURL: "wss://<TRUENAS_IP>/api/current"
truenasInsecure: "true"           # TrueNAS 用自簽憑證時
defaultPool: "tank"
nfsServer: "<TRUENAS_IP>"
iscsiPortal: "<TRUENAS_IP>:3260"  # 只有要用 iSCSI 才需要
iscsiIQNBase: "iqn.2005-10.org.freenas.ctl"

kubectl apply -f truenas-csi-driver.yaml
kubectl get pods -n truenas-csi
kubectl get csidrivers

cat <<EOF | kubectl apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: truenas-nfs
provisioner: csi.truenas.io
parameters:
  protocol: nfs
  pool: tank
  compression: "lz4"
  sync: "standard"
reclaimPolicy: Delete
allowVolumeExpansion: true
volumeBindingMode: Immediate
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: demo-nfs-pvc
  namespace: default
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: truenas-nfs
  resources:
    requests:
      storage: 5Gi
EOF

kubectl get pvc demo-nfs-pvc

cat <<EOF | kubectl apply -f -
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
  name: truenas-snapshot-class
driver: csi.truenas.io
deletionPolicy: Delete
---
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
  name: demo-nfs-snapshot
  namespace: default
spec:
  volumeSnapshotClassName: truenas-snapshot-class
  source:
    persistentVolumeClaimName: demo-nfs-pvc
EOF

# 在 CP 產生 join 指令（token 預設 24 小時過期）
kubeadm token create --print-join-command

kubectl get nodes -o wide
kubectl -n calico-system get pods -o wide

sudo kubeadm init \
  --pod-network-cidr=10.244.0.0/16 \
  --control-plane-endpoint="10.0.1.100:6443" \
  --upload-certs \
  --cri-socket=unix:///var/run/crio/crio.sock \
  --kubernetes-version=v1.36.2

sudo kubeadm join 10.0.1.100:6443 \
  --token <TOKEN> \
  --discovery-token-ca-cert-hash sha256:<HASH> \
  --control-plane \
  --certificate-key <CERTIFICATE_KEY> \
  --cri-socket=unix:///var/run/crio/crio.sock

# 1. 建好 LB/VIP 後修改 controlPlaneEndpoint
kubectl -n kube-system edit cm kubeadm-config
#    將 ClusterConfiguration.controlPlaneEndpoint 設為 "10.0.1.100:6443"

# 2. 重新簽發含新 SAN 的 API Server 憑證
sudo cp -a /etc/kubernetes/pki /etc/kubernetes/pki.backup.$(date +%Y%m%d%H%M%S)
sudo mv /etc/kubernetes/pki/apiserver.crt /etc/kubernetes/pki/apiserver.crt.bak
sudo mv /etc/kubernetes/pki/apiserver.key /etc/kubernetes/pki/apiserver.key.bak
sudo kubeadm init phase certs apiserver --control-plane-endpoint "10.0.1.100:6443"
sudo systemctl restart kubelet

# 3. 確認所有節點的 kubeconfig 改指向新位址後，再照建議路徑加入新 CP

sudo ETCDCTL_API=3 etcdctl snapshot save /root/etcd-backup-$(date +%Y%m%d%H%M).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key