diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..b9ccaa7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,25 @@ +--- +name: Check the container can be built + +on: + push: + branches: + - main + pull_request: + +jobs: + builder: + name: Build images + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build + uses: docker/build-push-action@v3 + with: + push: false diff --git a/Dockerfile b/Dockerfile index 20d8ef6..2f30a47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,14 @@ FROM python:3.10 -# Get supervisor and cron -# Do not get picky on exact cron & supervisor versions +# Get cron +# Do not get picky on exact cron version so ignore DL3008 # hadolint ignore=DL3008 RUN apt-get update \ - && apt-get install --no-install-recommends -y supervisor cron \ + && apt-get install --no-install-recommends -y cron \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && mkdir -p /var/log/supervisor \ && rm /etc/cron.daily/* -COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf - COPY cron-update-endpoints /etc/cron.d/endpoints COPY requirements.txt /fedcloud-dashboard/requirements.txt @@ -22,6 +19,4 @@ COPY ./dashboard /fedcloud-dashboard/dashboard WORKDIR /fedcloud-dashboard -EXPOSE 8000 - -CMD ["/usr/bin/supervisord"] +CMD ["cron", "-f"] diff --git a/README.md b/README.md index ce63b1d..3309d17 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,46 @@ # FedCloud dashboard -Proof of concept to gather all OpenStack Horizon endpoints published in the EGI GOCDB. +A web dashboard which shows all OpenStack Horizon endpoints published in +[EGI GOCDB](https://goc.egi.eu/) ## Installation -Clone this repository: +This code relies on docker-compose to run 3 containers: + +- [traefik](https://traefik.io/traefik/) to provide HTTP proxy and cert + management +- [dashy](https://dashy.to/) for generating the dashboard +- some python code to generate the list of endpoints + +The existing docker-compose file assumes you will run the code on a publicly +accessible host with a valid name. You can create a `.env` file with the +`DASHBOARD_HOSTNAME` variable defined with the hostname of your server and just +start the service: + +```shell +cd /path/to/working/directory +git clone https://github.com/EGI-Federation/fedcloud-dashboard.git +echo "DASHBOARD_HOSTNAME="" +docker-compose up --build +``` + +This will build the container that generates the list of endpoints and start all +the process to make the dashboard available. + +## Running locally + +If you don't have a publicly accessible host, the easiest is to manually run +dashy and the code to generate the list of endpoints: + +First clone the repository: + ```shell cd /path/to/working/directory git clone https://github.com/EGI-Federation/fedcloud-dashboard.git ``` Create a conda environment with requirements: + ```shell # Download and install conda cd /path/to/conda/install/folder @@ -24,44 +54,21 @@ conda env create -f environment.yml conda activate horizon-aggregator ``` -## Preview Test whether the query script works: -```shell -cd /path/to/working/directory/fedcloud-dashboard/dashboard/ -python find_endpoints.py -``` -Test whether the flask app works: ```shell cd /path/to/working/directory/fedcloud-dashboard/dashboard/ -export FLASK_APP=main -flask run --host=0.0.0.0 -``` - -## Use docker - -First things first: make sure port 8000 is open on the target system! - -### Build image - -Here are the steps: -```shell -git clone https://github.com/EGI-Federation/fedcloud-dashboard.git -cd fedcloud-dashboard -sudo docker build --no-cache -t dashboard:1.0.0 . +python dashy_endpoints.py > conf.yml ``` -### Run container +Use the generated `conf.yml` with dashy: -Here are the steps: ```shell -git clone https://github.com/EGI-Federation/fedcloud-dashboard.git -cd fedcloud-dashboard/dashboard -sudo docker run \ - --name dashboard \ - --detach \ - --publish 8000:8000 \ - dashboard:1.0.0 +cd /path/to/working/directory/fedcloud-dashboard/dashboard/ +docker run \ + -p 8080:80 \ + -v $PWD/conf.yml:/app/public/conf.yml \ + lissy93/dashy:latest ``` -The app should now return the list OpenStack Horizon endpoints published in the EGI GOCDB. +And point your browser to `http://localhost:8080` to see your dashboard running diff --git a/dashboard/find_endpoints.py b/dashboard/find_endpoints.py deleted file mode 100644 index 1dc18ad..0000000 --- a/dashboard/find_endpoints.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python - -""" -This code has been copied from https://github.com/tdviet/fedcloudclient -""" - -from urllib import parse - -import defusedxml.ElementTree as ElementTree -import requests - -GOCDB_PUBLICURL = "https://goc.egi.eu/gocdbpi/public/" -TIMEOUT = 10 - - -def get_sites(): - """ - Get list of sites (using GOCDB instead of site configuration) - :return: list of site IDs - """ - q = {"method": "get_site_list", "certification_status": "Certified"} - url = "?".join([GOCDB_PUBLICURL, parse.urlencode(q)]) - r = requests.get(url) - sites = [] - if r.status_code == 200: - root = ElementTree.fromstring(r.text) - for s in root: - sites.append(s.attrib.get("NAME")) - else: - print("Something went wrong...") - print(r.status_code) - print(r.text) - return sites - - -def find_endpoints(service_type, production=True, monitored=True, site=None): - """ - Searching GOCDB for endpoints according to service types and status - :param service_type: - :param production: - :param monitored: - :param site: list of sites, None for searching all sites - :return: list of endpoints - """ - q = {"method": "get_service_endpoint", "service_type": service_type} - if monitored: - q["monitored"] = "Y" - if site: - q["sitename"] = site - sites = [site] - else: - sites = get_sites() - url = "?".join([GOCDB_PUBLICURL, parse.urlencode(q)]) - r = requests.get(url) - endpoints = [] - if r.status_code == 200: - root = ElementTree.fromstring(r.text) - for sp in root: - if production: - prod = sp.find("IN_PRODUCTION").text.upper() - if prod != "Y": - continue - os_url = sp.find("URL").text - ep_site = sp.find("SITENAME").text - if ep_site not in sites: - continue - # os_url = urlparse.urlparse(sp.find('URL').text) - # sites[sp.find('SITENAME').text] = urlparse.urlunparse( - # (os_url[0], os_url[1], os_url[2], '', '', '')) - endpoints.append([sp.find("SITENAME").text, service_type, os_url]) - else: - print("Something went wrong...") - print(r.status_code) - print(r.text) - return endpoints - - -def main(): - endpoints = find_endpoints("org.openstack.horizon") - for site, service_type, endpoint in endpoints: - print(site, endpoint) - - -if __name__ == "__main__": - main() diff --git a/dashboard/horizonaggregator.wsgi b/dashboard/horizonaggregator.wsgi deleted file mode 100644 index 89a7f6f..0000000 --- a/dashboard/horizonaggregator.wsgi +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/python3 -import sys -sys.path.insert(0, '/var/www/html') -from wsgi import app as application diff --git a/dashboard/main.py b/dashboard/main.py deleted file mode 100644 index 9cae551..0000000 --- a/dashboard/main.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import sys - -from flask import Flask, render_template - -script_dir = os.path.abspath(os.path.dirname(__file__)) -sys.path.insert(0, script_dir) -from update_endpoints import read_endpoints - -app = Flask(__name__) - - -@app.route("/") -def main(): - return render_template("dashboard.html", endpoints=read_endpoints()) - - -if __name__ == "__main__": - main() diff --git a/dashboard/static/css/style.css b/dashboard/static/css/style.css deleted file mode 100644 index 905e0f4..0000000 --- a/dashboard/static/css/style.css +++ /dev/null @@ -1,12 +0,0 @@ -body { - color: #3d3d3d; - background-color: #c6d2eb; - font-family: sans-serif; - font-size: 16px; - font-weight: normal; -} - -.container { - width: 40%; - margin: auto; -} diff --git a/dashboard/static/images/dashboard-login-check-in-1.png b/dashboard/static/images/dashboard-login-check-in-1.png deleted file mode 100644 index e4cbaa0..0000000 Binary files a/dashboard/static/images/dashboard-login-check-in-1.png and /dev/null differ diff --git a/dashboard/static/images/dashboard-login-check-in-2.png b/dashboard/static/images/dashboard-login-check-in-2.png deleted file mode 100644 index 4b337da..0000000 Binary files a/dashboard/static/images/dashboard-login-check-in-2.png and /dev/null differ diff --git a/dashboard/static/images/dashboard-login-check-in-3.png b/dashboard/static/images/dashboard-login-check-in-3.png deleted file mode 100644 index 84bb48d..0000000 Binary files a/dashboard/static/images/dashboard-login-check-in-3.png and /dev/null differ diff --git a/dashboard/static/images/dashboard-login-check-partb.png b/dashboard/static/images/dashboard-login-check-partb.png deleted file mode 100644 index cbceb62..0000000 Binary files a/dashboard/static/images/dashboard-login-check-partb.png and /dev/null differ diff --git a/dashboard/static/images/egi.png b/dashboard/static/images/egi.png deleted file mode 100644 index bf684e1..0000000 Binary files a/dashboard/static/images/egi.png and /dev/null differ diff --git a/dashboard/templates/dashboard.html b/dashboard/templates/dashboard.html deleted file mode 100644 index dabc04e..0000000 --- a/dashboard/templates/dashboard.html +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - EGI Cloud Compute Dashboard - - -
- EGI logo -

OpenStack sites in the EGI Federation

- {% for site, endpoint in endpoints %} -

{{ site }}: {{ endpoint }}

- {% endfor %} -

Note: This list is automatically taken from the - - Configuration Database in EGI.

-

Contact EGI Support to add your site to the list.

-

Further instructions

-

Select EGI Check-In - to log into the OpenStack Horizon dashboards.

-

In the dropdown menu of Horizon, you may use EGI Check-In:

- Check-In Option 1 -

Or OpenID Connect:

- Check-In Option 2 -

Or egi.eu:

- Check-In Option 3 -

Additionally you may need to select aai.egi.eu/oidc as well:

- Check-In OIDC -
- - diff --git a/dashboard/update_endpoints.py b/dashboard/update_endpoints.py deleted file mode 100644 index eeabf50..0000000 --- a/dashboard/update_endpoints.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python - -import time -from pathlib import Path - -from find_endpoints import find_endpoints - -# filename for the sentinel file -SENTINEL_FILE_NAME = "/tmp/horizonaggregator-endpoints" - -# number of seconds to wait until -# we query again GOCDB -SENTINEL_FILE_CHANGE_SECONDS = 600 - -# sentinel file descriptor -sentinel_file = Path(SENTINEL_FILE_NAME) - - -def update_endpoints(): - """ - Update the file containing the list of GOCDB endpoints - of type "org.openstack.horizon" - """ - # Recreate sentinel file - sentinel_file.unlink(missing_ok=True) - - with open(sentinel_file, "w") as f: - endpoints = find_endpoints("org.openstack.horizon") - for site, service_type, endpoint in endpoints: - f.write(f"{site},{endpoint}\n") - - -def read_endpoints(): - """ - Read the list of GOCDB endpoints of type "org.openstack.horizon". - Instead of querying GOCDB every time, we save the result of - the query in a local file with SENTINEL_FILE_NAME. The file is - updated with GOCDB data only after SENTINEL_FILE_CHANGE_SECONDS. - """ - if not sentinel_file.exists(): - update_endpoints() - else: - # When was the sentinel file last modified? - time_sentinel_modified = round(sentinel_file.stat().st_ctime) - # What's the time now? - time_now_in_seconds = round(time.time()) - # How long has it passed? - difference = time_now_in_seconds - time_sentinel_modified - - if difference > SENTINEL_FILE_CHANGE_SECONDS: - update_endpoints() - - endpoints = [] - # see ideas from - # https://stackoverflow.com/questions/3277503/how-to-read-a-file-line-by-line-into-a-list - with open(sentinel_file) as f: - for line in f: - endpoints.append(line.rstrip().split(",")) - - return endpoints - - -def main(): - endpoints = read_endpoints() - for site, endpoint in endpoints: - print(site, endpoint) - - -if __name__ == "__main__": - main() diff --git a/environment.yml b/environment.yml index b2bc489..e15be37 100644 --- a/environment.yml +++ b/environment.yml @@ -9,4 +9,4 @@ dependencies: - python 3.10.* - defusedxml - requests - - flask + - pyyaml diff --git a/requirements.txt b/requirements.txt index 39a9fe7..50f6b02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -gunicorn -flask requests defusedxml pyyaml diff --git a/supervisord.conf b/supervisord.conf deleted file mode 100644 index 453fe9f..0000000 --- a/supervisord.conf +++ /dev/null @@ -1,9 +0,0 @@ -[supervisord] -nodaemon=true - -[program:gunicorn] -directory=/fedcloud-dashboard -command=gunicorn --bind 0.0.0.0:8000 dashboard.main:app - -[program:cron] -command=cron -f