Skip to main content

prepare offline resource for kubespray installation

ben.wangzAbout 4 min

prepare offline resource for kubespray installation

container runtime

  1. install podman 
    • sudo dnf -y install podman
  2. specify container runtime(recommended) 
    • export CONTAINER_RUNTIME=podman
  3. verify/find container runtime automatically 
    • function find_container_runtime(){
    if command -v nerdctl 1>/dev/null 2>&1; then
        echo "nerdctl"
    elif command -v podman 1>/dev/null 2>&1; then
        echo "podman"
    elif command -v docker 1>/dev/null 2>&1; then
        echo "docker"
    else
        echo "No supported container runtime found"
        exit 1
    fi
}
if [ -z "$CONTAINER_RUNTIME" ]; then
    CONTAINER_RUNTIME=$(find_container_runtime)
    echo "CONTAINER_RUNTIME=$CONTAINER_RUNTIME found"
else
    if command -v $CONTAINER_RUNTIME 1>/dev/null 2>&1; then
        echo "CONTAINER_RUNTIME=$CONTAINER_RUNTIME verified"
    else
        echo "CONTAINER_RUNTIME=$CONTAINER_RUNTIME but not exits"
    fi
fi

turn off selinux and firewalld

  1. turn off selinux permanently 
    • sudo sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
sudo setenforce 0
  2. turn off firewalld permanently 
    • sudo systemctl stop firewalld
sudo systemctl disable firewalld

offline resource directory

  1. specify offline resource directory 
    • export OFFLINE_RESOURCE_DIRECTORY=$HOME/kubespray-offline-resource
  2. create offline resource directory 
    • mkdir -p $OFFLINE_RESOURCE_DIRECTORY

build inventory for kubespray

  1. specify kubespary image 
    • export KUBESPRAY_IMAGE=quay.io/kubespray/kubespray:v2.23.1
  2. copy sample inventory 
    • INVENTORY_DIRECTORY=$OFFLINE_RESOURCE_DIRECTORY/inventory
mkdir -p $INVENTORY_DIRECTORY
$CONTAINER_RUNTIME run --rm \
    -v $INVENTORY_DIRECTORY:/my-inventory \
    -it $KUBESPRAY_IMAGE \
    /bin/cp -rfp /kubespray/inventory/sample /my-inventory/single-master
  3. create inventory file 
    • INVENTORY_FILE=$INVENTORY_DIRECTORY/single-master/hosts.yaml
declare -a IPS=(192.168.1.105)
$CONTAINER_RUNTIME run --rm \
    -v $INVENTORY_DIRECTORY:/my-inventory \
    -e CONFIG_FILE=/my-inventory/single-master/hosts.yaml \
    -it $KUBESPRAY_IMAGE \
    python3 /kubespray/contrib/inventory_builder/inventory.py ${IPS[@]}

download files and images

  1. generate offline files and image list 
    • $CONTAINER_RUNTIME run --rm \
    -v $INVENTORY_DIRECTORY:/my-inventory \
    -it $KUBESPRAY_IMAGE \
    bash -c '/kubespray/contrib/offline/generate_list.sh -i /my-inventory/single-master/hosts.yaml \
        && /bin/cp /kubespray/contrib/offline/temp/files.list /my-inventory/single-master/files.list \
        && /bin/cp /kubespray/contrib/offline/temp/images.list /my-inventory/single-master/images.list'
  2. modify files.list as aria2 input file 
    • cat << 'EOF' > $INVENTORY_DIRECTORY/aria2_input_file_transform.py
import os
from urllib.parse import urlparse

download_path = os.environ.get("DOWNLOAD_PATH", "/tmp/download")
input_file_path = os.environ.get("INPUT_FILE_PATH", "files.list")
output_file_path = os.environ.get("OUTPUT_FILE_PATH", "files.list.aria2")
with open(output_file_path, "w") as output_file:
    with open(input_file_path) as input_file:
        for line in input_file:
            url = line.strip()
            url_parsed = urlparse(url)
            filename = os.path.basename(url_parsed.path)
            path = os.path.dirname(url_parsed.path)
            output_file.write(url + "\n")
            output_file.write(" dir=" + os.path.join(download_path, path.strip("/")) + "\n")
            output_file.write(" out=" + filename + "\n")
EOF

    • DOWNLOAD_FILES_DIRECTORY=$INVENTORY_DIRECTORY/single-master/files
mkdir -p $DOWNLOAD_FILES_DIRECTORY
FILES_LIST=$INVENTORY_DIRECTORY/single-master/files.list
FILES_LIST_ARIA2=$DOWNLOAD_FILES_DIRECTORY/files.list.aria2
$CONTAINER_RUNTIME run --rm \
    -v $DOWNLOAD_FILES_DIRECTORY:/download/files \
    -v $INVENTORY_DIRECTORY/single-master/files.list:/download/files.list:ro \
    -v $INVENTORY_DIRECTORY/aria2_input_file_transform.py:/tmp/aria2_input_file_transform.py:ro \
    -e DOWNLOAD_PATH=/download/files \
    -e INPUT_FILE_PATH=/download/files.list \
    -e OUTPUT_FILE_PATH=/download/files/files.list.aria2 \
    -it docker.io/library/python:3.12.1-alpine3.19 \
    python3 /tmp/aria2_input_file_transform.py
  3. download files 
    • $CONTAINER_RUNTIME run --rm \
    -v $DOWNLOAD_FILES_DIRECTORY:/download/files \
    -v $FILES_LIST_ARIA2:/download/files.list.aria2:ro \
    -it docker.io/library/alpine:3.19.0 sh -c 'apk add aria2 && aria2c -i /download/files.list.aria2 --max-tries=0 --continue'
  4. download images 
    • DOWNLOAD_IMAGES_DIRECTORY=$INVENTORY_DIRECTORY/single-master/images
mkdir -p $DOWNLOAD_IMAGES_DIRECTORY
while read IMAGE
do
    FILE_NAME="$(echo $IMAGE | sed s@"/"@"-"@g | sed s/":"/"-"/g)".dim
    IMAGE_FILE=$DOWNLOAD_IMAGES_DIRECTORY/$FILE_NAME
    $CONTAINER_RUNTIME image inspect $IMAGE > /dev/null || $CONTAINER_RUNTIME pull $IMAGE
    $CONTAINER_RUNTIME save -o $IMAGE_FILE $IMAGE
done < $INVENTORY_DIRECTORY/single-master/images.list

start nginx to serve files

  1. prepare nginx.conf 
    • cat << 'EOF' > $INVENTORY_DIRECTORY/single-master/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;
    default_type        application/octet-stream;
    include /etc/nginx/conf.d/*.conf;
    server {
        listen       80 default_server;
        listen       [::]:80 default_server;
        server_name  _;
        include /etc/nginx/default.d/*.conf;
        location / {
            root    /usr/share/nginx/html/download;
        autoindex on;
        autoindex_exact_size off;
        autoindex_localtime on;
        }
        error_page 404 /404.html;
            location = /40x.html {
        }
        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }
}
EOF
  2. start nginx container 
    • $CONTAINER_RUNTIME run --restart always \
    -v $INVENTORY_DIRECTORY/single-master/nginx.conf:/etc/nginx/nginx.conf:ro \
    -v $DOWNLOAD_FILES_DIRECTORY:/usr/share/nginx/html/download:ro \
    -p 8080:80 \
    -d docker.io/library/nginx:1.19.7-alpine

start registry to serve images

  1. generate ssl 
    • IP_ADDRESS=192.168.1.105
REGISTRY_DIRECTORY=$INVENTORY_DIRECTORY/single-master/registry
mkdir -p $REGISTRY_DIRECTORY
cat << EOF > $REGISTRY_DIRECTORY/san.conf
[req]
default_bits = 4096
default_md = sha256
distinguished_name = req_distinguished_name
x509_extensions = v3_req
prompt = no
[req_distinguished_name]
C = CN
ST = ZheJiang
L = HangZhou
O = ZhejiangLab
OU = astronomy
CN = $IP_ADDRESS
[v3_req]
keyUsage = keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
IP.1 = $IP_ADDRESS
EOF

    • mkdir -p $REGISTRY_DIRECTORY/certs
SSL_KEY_FILE=$REGISTRY_DIRECTORY/certs/domain.key
openssl genrsa 4096 > $SSL_KEY_FILE
chmod 400 $REGISTRY_DIRECTORY/certs/domain.key
openssl req -new -x509 -nodes -sha256 -days 365 -key $SSL_KEY_FILE -out $REGISTRY_DIRECTORY/certs/domain.crt --config $REGISTRY_DIRECTORY/san.conf
  2. start local registry 
    • mkdir -p $REGISTRY_DIRECTORY/storage
$CONTAINER_RUNTIME run --restart always \
    --name local-registry \
    -v $REGISTRY_DIRECTORY/certs:/certs \
    -v $REGISTRY_DIRECTORY/storage:/var/lib/registry \
    -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \
    -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
    -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
    -p 5443:5443 \
    -p 5000:5000 \
    -d registry:2.7.1
  3. load images and push them to local registry 
    • # there's fundamentally no difference between CER and CRT files
ln -s $REGISTRY_DIRECTORY/certs/domain.crt $REGISTRY_DIRECTORY/certs/domain.cert
while read IMAGE
do
    FILE_NAME="$(echo $IMAGE | sed s@"/"@"-"@g | sed s/":"/"-"/g)".dim
    IMAGE_FILE=$DOWNLOAD_IMAGES_DIRECTORY/$FILE_NAME
    $CONTAINER_RUNTIME image inspect $IMAGE > /dev/null || $CONTAINER_RUNTIME load -i $IMAGE_FILE
    IMAGE_ID=$($CONTAINER_RUNTIME image inspect --format '{{.Id}}' $IMAGE)
    $CONTAINER_RUNTIME tag $IMAGE_ID $IP_ADDRESS:5443/$IMAGE
    if [ "podman" == "$CONTAINER_RUNTIME" ]; then
        CERT_DIR_OPTION="--cert-dir=$REGISTRY_DIRECTORY/certs"
    else
        CERT_DIR_OPTION=""
        mkdir -p /etc/docker/certs.d
        sudo cp -r $REGISTRY_DIRECTORY/certs /etc/docker/certs.d/$IP_ADDRESS:5443
    fi
    $CONTAINER_RUNTIME push $CERT_DIR_OPTION $IP_ADDRESS:5443/$IMAGE
done < $INVENTORY_DIRECTORY/single-master/images.list
  4. the storage($REGISTRY_DIRECTORY/storage) can be migrated any where with the docker registry

create local yum repository

  1. create YUM_REPO_CONF 
    • YUM_REPO_CONF_DIRECTORY=$HOME/kubespray-offline-resource/fedora-38
mkdir -p $YUM_REPO_CONF_DIRECTORY
cat << 'EOF' > $YUM_REPO_CONF_DIRECTORY/fedora.repo
[fedora]
name=Fedora $releasever - $basearch
#baseurl=http://download.example/pub/fedora/linux/releases/$releasever/Everything/$basearch/os/
metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch
enabled=1
countme=1
metadata_expire=7d
repo_gpgcheck=0
type=rpm
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False
EOF
  2. create local yum repository 
    • YUM_REPOSITORY_DIRECTORY=$HOME/kubespray-offline-resource/yum-repository
mkdir -p $YUM_REPOSITORY_DIRECTORY
$CONTAINER_RUNTIME run --rm \
    -v $YUM_REPOSITORY_DIRECTORY:/yum-repository \
    -v $YUM_REPO_CONF_DIRECTORY:/etc/yum.repos.d:ro \
    -it fedora:38 \
    bash -c 'dnf install -y createrepo reposync && reposync -p /yum-repository --repo=fedora --download-metadata'

configure offline settings(NOT FINISHED)

* ```shell
  cat << EOF >> $INVENTORY_DIRECTORY/group_vars/all/offline.yml
  files_repo: "http://$IP_ADDRESS"
  registry_host: "$IP_ADDRESS:5443"
  kube_image_repo: ""
  gcr_image_repo: ""
  docker_image_repo: ""
  quay_image_repo: ""
  github_image_repo: ""
  kubeadm_download_url: "/release//bin/linux//kubeadm"
  kubectl_download_url: "/release//bin/linux//kubectl"
  kubelet_download_url: "/release//bin/linux//kubelet"
  # etcd is optional if you **DON'T** use etcd_deployment=host
  etcd_download_url: "/etcd-io/etcd/releases/download//etcd--linux-.tar.gz"
  cni_download_url: "/containernetworking/plugins/releases/download//cni-plugins-linux--.tgz"
  crictl_download_url: "/kubernetes-sigs/cri-tools/releases/download//crictl--0-.tar.gz"
  # If using Calico
  calicoctl_download_url: "/projectcalico/calico/releases/download//calicoctl-linux-"
  # If using Calico with kdd
  calico_crds_download_url: "/projectcalico/calico/archive/.tar.gz"
  # Containerd
  containerd_download_url: "/containerd/containerd/releases/download/v/containerd--linux-.tar.gz"
  runc_download_url: "/opencontainers/runc/releases/download//runc."
  nerdctl_download_url: "/containerd/nerdctl/releases/download/v/nerdctl--0-.tar.gz"
  EOF
  ```
* ```shell
  cat << EOF >>
  # unfortunately, containerd_registries_mirrors will be ignored if `container_manager=crio`
  registry_addr: $IP_ADDRESS:5443
  containerd_registries_mirrors:
    - prefix: ""
      mirrors:
        - host: "$IP_ADDRESS:5443"
          capabilities: ["pull", "resolve"]
          skip_verify: true
  EOF
  ```
Default footer
Copyright © 2024 ben.wangz