diff --git a/.github/workflows/build_push_docker_hub.yaml b/.github/workflows/build_push_docker_hub.yaml index 4a8aa56..a360027 100644 --- a/.github/workflows/build_push_docker_hub.yaml +++ b/.github/workflows/build_push_docker_hub.yaml @@ -31,7 +31,8 @@ jobs: { name: data-producer, context: ./services/data-producer, file: ./services/data-producer/Dockerfile }, { name: mlflow, context: ./services/mlflow, file: ./services/mlflow/Dockerfile }, { name: airflow-spark, context: ./services/airflow, file: ./services/airflow/Dockerfile }, - { name: ray, context: ./services/ray, file: ./services/ray/Dockerfile } + { name: ray, context: ./services/ray, file: ./services/ray/Dockerfile }, + { name: forecast-service, context: ./services/forecast-service, file: ./services/forecast-service/Dockerfile } ] steps: - name: Download repository artifact diff --git a/README.md b/README.md index ab1fa6c..6572be9 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ From doc: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-com ## How to run Kubernetes/Helm 1. `cd sfmlops-helm` and `helm dependency build` to fetch all dependencies -2. Both install and upgrade the main chart: `helm upgrade --install --create-namespace -n mlops sfmlops-helm ./ -f values.yaml` +2. Both install and upgrade the main chart: `helm upgrade --install --create-namespace -n mlops sfmlops-helm ./ -f values.yaml -f values-ray.yaml` 3. Deploy Kafka: `helm -n kafka upgrade --install kafka-release oci://registry-1.docker.io/bitnamicharts/kafka --create-namespace --version 23.0.7 -f values-kafka.yaml` 4. Deploy Airflow: 1. `helm repo add apache-airflow https://airflow.apache.org` diff --git a/sfmlops-helm/Chart.lock b/sfmlops-helm/Chart.lock new file mode 100644 index 0000000..0d84f67 --- /dev/null +++ b/sfmlops-helm/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: ingress-nginx + repository: https://kubernetes.github.io/ingress-nginx + version: 4.10.0 +- name: kuberay-operator + repository: https://ray-project.github.io/kuberay-helm/ + version: 1.1.0-rc.0 +- name: ray-cluster + repository: https://ray-project.github.io/kuberay-helm/ + version: 1.1.0-rc.0 +digest: sha256:8be42ea3b59b9b55ccd9d8ed173a5072eae637e988900dac1cdb28ef126f3724 +generated: "2024-03-31T14:18:24.709723+07:00" diff --git a/sfmlops-helm/Chart.yaml b/sfmlops-helm/Chart.yaml index c792b5c..bbc06b5 100644 --- a/sfmlops-helm/Chart.yaml +++ b/sfmlops-helm/Chart.yaml @@ -27,6 +27,12 @@ dependencies: - name: ingress-nginx version: 4.10.0 # Specify the version you want repository: https://kubernetes.github.io/ingress-nginx +- name: kuberay-operator + version: 1.1.0-rc.0 + repository: https://ray-project.github.io/kuberay-helm/ +- name: ray-cluster + version: 1.1.0-rc.0 + repository: https://ray-project.github.io/kuberay-helm/ # - name: kafka # version: 23.0.7 # repository: oci://registry-1.docker.io/bitnamicharts diff --git a/sfmlops-helm/charts/kuberay-operator-1.1.0-rc.0.tgz b/sfmlops-helm/charts/kuberay-operator-1.1.0-rc.0.tgz new file mode 100644 index 0000000..5c2ace8 Binary files /dev/null and b/sfmlops-helm/charts/kuberay-operator-1.1.0-rc.0.tgz differ diff --git a/sfmlops-helm/charts/ray-cluster-1.1.0-rc.0.tgz b/sfmlops-helm/charts/ray-cluster-1.1.0-rc.0.tgz new file mode 100644 index 0000000..d024a9e Binary files /dev/null and b/sfmlops-helm/charts/ray-cluster-1.1.0-rc.0.tgz differ diff --git a/sfmlops-helm/templates/forecast-service-deployment-service.yaml b/sfmlops-helm/templates/forecast-service-deployment-service.yaml index d6cf9e3..2f984dd 100644 --- a/sfmlops-helm/templates/forecast-service-deployment-service.yaml +++ b/sfmlops-helm/templates/forecast-service-deployment-service.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: forecast-service - image: ariya23156/sfmlops-training-service + image: ariya23156/sfmlops-forecast-service ports: - containerPort: {{ .Values.forecastServicePort }} envFrom: diff --git a/sfmlops-helm/templates/ray-cluster.yaml b/sfmlops-helm/templates/ray-cluster.yaml index ca10279..2ab5bfa 100644 --- a/sfmlops-helm/templates/ray-cluster.yaml +++ b/sfmlops-helm/templates/ray-cluster.yaml @@ -49,9 +49,6 @@ spec: envFrom: - configMapRef: name: global-configmap - containerEnv: - - name: MLFLOW_TRACKING_URI - value: "http://mlflow-service:{{ .Values.mlflowPort }}" volumeMounts: - mountPath: /tmp/ray name: ray-logs diff --git a/sfmlops-helm/values-ray.yaml b/sfmlops-helm/values-ray.yaml new file mode 100644 index 0000000..7244068 --- /dev/null +++ b/sfmlops-helm/values-ray.yaml @@ -0,0 +1,12 @@ +image: + tag: 2.9.3-py39-cpu-aarch64 + +head: + containerEnv: + - name: MLFLOW_TRACKING_URI + value: "http://mlflow-service:5050" + +worker: + containerEnv: + - name: MLFLOW_TRACKING_URI + value: "http://mlflow-service:5050"