From 4bd08c4a73d8802cab6c416b8de56f50b12b85e9 Mon Sep 17 00:00:00 2001 From: Victor Morales Date: Wed, 21 Feb 2024 22:05:56 -0800 Subject: [PATCH] Initial commit --- .github/dependabot.yml | 15 ++++ .github/workflows/linter.yml | 32 ++++++++ .gitignore | 1 + .yamlfmt | 13 ++++ Makefile | 28 +++++++ README.md | 15 ++++ Vagrantfile | 102 +++++++++++++++++++++++++ scripts/_common.sh | 31 ++++++++ scripts/_utils.sh | 43 +++++++++++ scripts/configure.sh | 120 ++++++++++++++++++++++++++++++ scripts/defaults.env | 12 +++ scripts/demo.sh | 140 +++++++++++++++++++++++++++++++++++ scripts/infra.yaml | 49 ++++++++++++ scripts/install.sh | 34 +++++++++ scripts/main.sh | 24 ++++++ 15 files changed, 659 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/linter.yml create mode 100644 .gitignore create mode 100644 .yamlfmt create mode 100644 Makefile create mode 100644 README.md create mode 100644 Vagrantfile create mode 100755 scripts/_common.sh create mode 100755 scripts/_utils.sh create mode 100755 scripts/configure.sh create mode 100755 scripts/defaults.env create mode 100755 scripts/demo.sh create mode 100644 scripts/infra.yaml create mode 100755 scripts/install.sh create mode 100755 scripts/main.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..cccca6e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..881e4f1 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,32 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +name: Lint Code Base +# yamllint disable-line rule:truthy +on: + push: + pull_request: +jobs: + check-broken-links: + name: Check documentation external links + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - name: Check broken links + uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 + check-super-linter: + name: Check syntax (super-linter) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + - name: Run super-linter validation + uses: github/super-linter@v5.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LINTER_RULES_PATH: / diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8000dd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/.yamlfmt b/.yamlfmt new file mode 100644 index 0000000..99455a0 --- /dev/null +++ b/.yamlfmt @@ -0,0 +1,13 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +formatter: + type: basic + include_document_start: true + pad_line_comments: 2 + max_line_length: 160 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..585ec14 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +DOCKER_CMD ?= $(shell which docker 2> /dev/null || which podman 2> /dev/null || echo docker) + +.PHONY: lint +lint: + sudo -E $(DOCKER_CMD) run --rm -v $$(pwd):/tmp/lint \ + -e RUN_LOCAL=true \ + -e LINTER_RULES_PATH=/ \ + -e VALIDATE_KUBERNETES_KUBEVAL=false \ + -e KUBERNETES_KUBECONFORM_OPTIONS='-ignore-missing-schemas' \ + github/super-linter + tox -e lint + +.PHONY: fmt +fmt: + sudo -E $(DOCKER_CMD) run --rm -u "$$(id -u):$$(id -g)" \ + -v "$$(pwd):/mnt" \ + -w /mnt mvdan/shfmt -l -w -i 4 -s . + command -v yamlfmt > /dev/null || curl -s "https://i.jpillora.com/google/yamlfmt!!" | bash + yamlfmt -dstar **/*.{yaml,yml} diff --git a/README.md b/README.md new file mode 100644 index 0000000..53c34f9 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# Nephio PoCs + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![GitHub Super-Linter](https://github.com/electrocucaracha/nephio-poc/workflows/Lint%20Code%20Base/badge.svg)](https://github.com/marketplace/actions/super-linter) +[![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop/rubocop) + +![visitors](https://visitor-badge.laobi.icu/badge?page_id=electrocucaracha.nephio-poc) + +The goal of this project is to provision a [Nephio Management cluster][1] for testing different use cases and scenarios. + + +* Software Package hosting URL - + + +[1]: https://nephio.org/ diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..83a36c0 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +# -*- mode: ruby -*- +# vi: set ft=ruby : +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +host = RbConfig::CONFIG['host_os'] + +no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || '127.0.0.1,localhost' +(1..254).each do |i| + no_proxy += ",10.0.2.#{i}" +end + +case host +when /darwin/ + mem = `sysctl -n hw.memsize`.to_i / 1024 +when /linux/ + mem = `grep 'MemTotal' /proc/meminfo | sed -e 's/MemTotal://' -e 's/ kB//'`.to_i +when /mswin|mingw|cygwin/ + mem = `wmic computersystem Get TotalPhysicalMemory`.split[1].to_i / 1024 +end + +# rubocop:disable Metrics/BlockLength +Vagrant.configure('2') do |config| + # rubocop:enable Metrics/BlockLength + config.vm.provider :libvirt + config.vm.provider :virtualbox + + config.vm.box = 'generic/ubuntu2204' + config.vm.box_check_update = false + config.vm.synced_folder './', '/vagrant' + config.vm.network 'forwarded_port', guest: 3000, guest_ip: '172.18.0.200', host: 3000 + config.vm.network 'forwarded_port', guest: 80, guest_ip: '172.18.0.202', host: 8080 + + # Initial setup + config.vm.provision 'shell', privileged: false, inline: <<-SHELL + if [ -f /etc/netplan/01-netcfg.yaml ] && ! grep -q '1.1.1.1, 8.8.8.8, 8.8.4.4' /etc/netplan/01-netcfg.yaml; then + sudo sed -i "s/addresses: .*/addresses: [1.1.1.1, 8.8.8.8, 8.8.4.4]/g" /etc/netplan/01-netcfg.yaml + sudo netplan apply + fi + # Create .bash_aliases + echo 'cd /vagrant/' >> /home/vagrant/.bash_aliases + chown vagrant:vagrant /home/vagrant/.bash_aliases + SHELL + + # Install dependencies + config.vm.provision 'shell', privileged: false do |sh| + sh.env = { + DEBUG: ENV.fetch('DEBUG', true), + ENABLE_CLUSTER_API: 'true', + ENABLE_NETWORK_CONFIG: 'true' + } + sh.inline = <<-SHELL + set -o errexit + set -o pipefail + + cd /vagrant/scripts + ./main.sh | tee ~/main.log + SHELL + end + + %i[virtualbox libvirt].each do |provider| + config.vm.provider provider do |p| + p.cpus = ENV['CPUS'] || 8 + p.memory = ENV['MEMORY'] || mem / 1024 / 4 + end + end + + config.vm.provider 'virtualbox' do |v| + v.gui = false + v.customize ['modifyvm', :id, '--nictype1', 'virtio', '--cableconnected1', 'on'] + # Enable nested paging for memory management in hardware + v.customize ['modifyvm', :id, '--nestedpaging', 'on'] + # Use large pages to reduce Translation Lookaside Buffers usage + v.customize ['modifyvm', :id, '--largepages', 'on'] + # Use virtual processor identifiers to accelerate context switching + v.customize ['modifyvm', :id, '--vtxvpid', 'on'] + end + + config.vm.provider :libvirt do |v, override| + override.vm.synced_folder './', '/vagrant', type: 'nfs', + nfs_version: ENV.fetch('VAGRANT_NFS_VERSION', 3) + v.memorybacking :access, mode: 'shared' + v.random_hostname = true + v.management_network_address = '10.0.2.0/24' + v.management_network_name = 'administration' + v.cpu_mode = 'host-passthrough' + end + + if !ENV['http_proxy'].nil? && !ENV['https_proxy'].nil? && Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || '' + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || '' + config.proxy.no_proxy = no_proxy + config.proxy.enabled = { docker: false } + end +end diff --git a/scripts/_common.sh b/scripts/_common.sh new file mode 100755 index 0000000..ace1367 --- /dev/null +++ b/scripts/_common.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail +set -o errexit +set -o nounset +[[ ${DEBUG:-false} != "true" ]] || set -o xtrace + +# get_status() - Print the current status of the cluster +function get_status { + set +o xtrace + printf "CPU usage: " + grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage " %"}' + printf "Memory free(Kb):" + awk -v low="$(grep low /proc/zoneinfo | awk '{k+=$2}END{print k}')" '{a[$1]=$2} END{ print a["MemFree:"]+a["Active(file):"]+a["Inactive(file):"]+a["SReclaimable:"]-(12*low);}' /proc/meminfo + if command -v kubectl >/dev/null; then + echo "Kubernetes Events:" + kubectl get events --sort-by='.lastTimestamp' -A --field-selector type!=Normal + echo "Kubernetes Resources:" + kubectl get all -A -o wide + echo "Kubernetes Nodes:" + kubectl describe nodes + fi +} diff --git a/scripts/_utils.sh b/scripts/_utils.sh new file mode 100755 index 0000000..e4aeec2 --- /dev/null +++ b/scripts/_utils.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2023 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail +set -o errexit +set -o nounset +[[ ${DEBUG:-false} != "true" ]] || set -o xtrace + +# debug() - This function prints a debug message in the standard output +function debug { + _print_msg "DEBUG" "$1" + echo "::debug::$1" +} + +# info() - This function prints an information message in the standard output +function info { + _print_msg "INFO" "$1" + echo "::notice::$1" +} + +# warn() - This function prints a warning message in the standard output +function warn { + _print_msg "WARN" "$1" + echo "::warning::$1" +} + +# error() - This function prints an error message in the standard output +function error { + _print_msg "ERROR" "$1" + echo "::error::$1" + exit 1 +} + +function _print_msg { + echo "$(date +%H:%M:%S) - $1: $2" +} diff --git a/scripts/configure.sh b/scripts/configure.sh new file mode 100755 index 0000000..29dd070 --- /dev/null +++ b/scripts/configure.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail +set -o errexit +set -o nounset +[[ ${DEBUG:-false} != "true" ]] || set -o xtrace + +# shellcheck source=scripts/_common.sh +source _common.sh +# shellcheck source=./scripts/_utils.sh +source _utils.sh + +trap get_status ERR + +function _setup_sysctl { + local key="$1" + local value="$2" + + if [ "$(sysctl -n "$key")" != "$value" ]; then + if [ -d /etc/sysctl.d ]; then + echo "$key=$value" | sudo tee "/etc/sysctl.d/99-$key.conf" + elif [ -f /etc/sysctl.conf ]; then + echo "$key=$value" | sudo tee --append /etc/sysctl.conf + fi + + sudo sysctl "$key=$value" + fi +} + +function _deploy_kpt_pkg { + local pkg=$1 + local dest=${2:-${pkg##*/}} + local revision=${3:-main} + + [[ ! $dest =~ "/" ]] || mkdir -p "${dest%/*}" + kpt pkg get "https://github.com/nephio-project/catalog.git/${pkg}@${revision}" "$dest" --for-deployment "${4:-false}" + newgrp docker </dev/null || exit +pkgs="" +[ "${ENABLE_METALLB:-true}" == "true" ] && pkgs+="distros/sandbox/metallb distros/sandbox/metallb-sandbox-config " +[ "${ENABLE_GITEA:-true}" == "true" ] && pkgs+="distros/sandbox/gitea " +[ "${ENABLE_CLUSTER_API:-false}" == "true" ] && pkgs+="distros/sandbox/cert-manager infra/capi/cluster-capi infra/capi/cluster-capi-infrastructure-docker infra/capi/cluster-capi-kind-docker-templates " +[ "${ENABLE_PORCH:-true}" == "true" ] && pkgs+="nephio/core/porch " +[ "${ENABLE_NEPHIO_OPERATOR:-true}" == "true" ] && pkgs+="nephio/core/nephio-operator " +[ "${ENABLE_CONFIGSYNC:-true}" == "true" ] && pkgs+="nephio/core/configsync " # Required for access tokens to connect to gitea services +[ "${ENABLE_NETWORK_CONFIG:-false}" == "true" ] && pkgs+="nephio/optional/network-config " # Required for workload cluster provisioning process + +for pkg in $pkgs; do + _deploy_kpt_pkg "$pkg" +done +popd >/dev/null + +# Rootsync objects configure ConfigSync to watch the specified source and apply objects from that source to the cluster. +_deploy_kpt_pkg "nephio/optional/rootsync" "/tmp/optional/mgmt" "main" "true" + +# Manage the contents of the Management clusters +_deploy_kpt_pkg "distros/sandbox/repository" "/tmp/repository/mgmt" "main" "true" + +# Used internally during the cluster bootstrapping process +_deploy_kpt_pkg "distros/sandbox/repository" "/tmp/repository/mgmt-staging" "main" "true" + +# Register repositories required for Workload Nephio cluster package operation +for repo in "-infra-capi" "-nephio-core" "-distros-sandbox" "-nephio-optional"; do + cat </dev/null 2>&1 + kubectl --kubeconfig "$HOME/.kube/config" get secret "${cluster}-kubeconfig" -o jsonpath='{.data.value}' | base64 -d >"$file" + fi + echo "$file" +} + +# k8s_wait_ready_replicas() - Waits for the readiness of a minimum number of replicas +function k8s_wait_ready_replicas { + local resource_type=$1 + local resource_name=$2 + local kubeconfig=${3:-"$HOME/.kube/config"} + local resource_namespace=${4:-default} + + timeout=600 + min_ready=1 + status_field=readyReplicas + [ "$resource_type" != "daemonset" ] || status_field=numberReady + + # should validate the params... + [ -f "$kubeconfig" ] || error "Kubeconfig file doesn't exist" + + k8s_wait_exists "$resource_type" "$resource_name" "$kubeconfig" "$resource_namespace" "$timeout" + + info "checking readiness of $resource_type $resource_namespace/$resource_name using $kubeconfig" + local ready="" + while [[ $timeout -gt 0 ]]; do + ready=$(kubectl --kubeconfig "$kubeconfig" -n "$resource_namespace" get "$resource_type" "$resource_name" -o jsonpath="{.status.$status_field}" || echo) + if [[ $ready -ge $min_ready ]]; then + return + fi + timeout=$((timeout - 5)) + sleep 5 + done + + kubectl --kubeconfig "$kubeconfig" -n "$resource_namespace" describe "$resource_type" "$resource_name" + error "Timed out waiting for $resource_type $resource_namespace/$resource_name to be ready" +} + +# capi_cluster_ready() - Wait for Cluster API cluster service readiness +function capi_cluster_ready { + local cluster=$1 + + k8s_wait_ready "cl" "$cluster" + for machineset in $(kubectl get machineset -l cluster.x-k8s.io/cluster-name="$cluster" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + k8s_wait_ready "machineset" "$machineset" + done + + # Wait for package variants + for pv in cluster configsync kindnet local-path-provisioner multus repo rootsync vlanindex; do + k8s_wait_exists "packagevariants" "${cluster}-$pv" + done + + # Wait for deployments and daemonsets readiness + kubeconfig=$(k8s_get_capi_kubeconfig "$cluster") + k8s_wait_ready_replicas "deployment" "otel-collector" "$kubeconfig" "config-management-monitoring" + for deploy in config-management-operator reconciler-manager "root-reconciler-$cluster"; do + k8s_wait_ready_replicas "deployment" "$deploy" "$kubeconfig" "config-management-system" + done + k8s_wait_ready_replicas "deployment" "local-path-provisioner" "$kubeconfig" "local-path-storage" + k8s_wait_ready_replicas "daemonset" "kindnet" "$kubeconfig" "kube-system" + k8s_wait_ready_replicas "daemonset" "kube-multus-ds" "$kubeconfig" "kube-system" +} + +kpt alpha repo get oai-core-packages || kpt alpha repo reg https://github.com/OPENAIRINTERFACE/oai-packages.git --name oai-core-packages --branch r2 --namespace default +kubectl apply -f infra.yaml + +# Wait for cluster resources creation +k8s_wait_exists "workloadcluster" "core01" +k8s_wait_exists "packagevariant" "kcd-clusters-mgmt-core01" +k8s_wait_exists "cl" "core01" + +# Wait for cluster readiness +capi_cluster_ready "core01" diff --git a/scripts/infra.yaml b/scripts/infra.yaml new file mode 100644 index 0000000..795ec7c --- /dev/null +++ b/scripts/infra.yaml @@ -0,0 +1,49 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +apiVersion: config.porch.kpt.dev/v1alpha2 +kind: PackageVariantSet +metadata: + name: kcd-clusters +spec: + upstream: + repo: catalog-infra-capi + package: nephio-workload-cluster + revision: v2.0.0 + targets: + - repositories: + - name: mgmt + packageNames: + - core01 + template: + annotations: + approval.nephio.org/policy: initial + pipeline: + mutators: + - image: gcr.io/kpt-fn/set-labels:v0.2.0 + configMap: + nephio.org/site-type: core + nephio.org/region: us-west1 +--- +apiVersion: config.porch.kpt.dev/v1alpha1 +kind: PackageVariant +metadata: + name: oai-cp-operators +spec: + upstream: + repo: oai-core-packages + package: oai-cp-operators + revision: r2 + downstream: + repo: core01 + package: oai-cp-operators + annotations: + approval.nephio.org/policy: initial + injectors: + - name: core01 diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..1bdf2f5 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail +set -o errexit +set -o nounset +if [[ ${DEBUG:-false} == "true" ]]; then + set -o xtrace + export PKG_DEBUG=true +fi + +# shellcheck source=./scripts/_utils.sh +source _utils.sh +# shellcheck source=./scripts/defaults.env +source defaults.env + +export PKG_KREW_PLUGINS_LIST=" " +export PKG_CNI_PLUGINS_FOLDER="/opt/cni/bin/" + +# Install dependencies +# NOTE: Shorten link -> https://github.com/electrocucaracha/pkg-mgr_scripts +curl -fsSL http://bit.ly/install_pkg | PKG_COMMANDS_LIST="docker,kubectl,kind" PKG="cni-plugins" bash + +if ! command -v kpt >/dev/null; then + curl -s "https://i.jpillora.com/GoogleContainerTools/kpt@v${KPT_VERSION}!" | bash + kpt completion bash | sudo tee /etc/bash_completion.d/kpt >/dev/null +fi diff --git a/scripts/main.sh b/scripts/main.sh new file mode 100755 index 0000000..3e7a438 --- /dev/null +++ b/scripts/main.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2024 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail +set -o errexit +set -o nounset +set -o xtrace + +# shellcheck source=./scripts/_utils.sh +source _utils.sh + +export DEBUG=true + +for step in install configure; do + info "Running $step process" + bash "./$step.sh" +done