Merge pull request #77 from Roblox/allow_privileged

Plugin configuration level privileged mode

README.md

@@ -85,7 +85,8 @@ More detailed instructions are in the [`example README.md`](https://github.com/R
 | :---: | :---: | :---: | :---: | :--- |
 | **enabled** | bool | no | true | Enable/Disable task driver. |
 | **containerd_runtime** | string | yes | N/A | Runtime for containerd e.g. `io.containerd.runc.v1` or `io.containerd.runc.v2`. |
-| **stats_interval** | string | no | 1s | Interval for collecting `TaskStats` |
+| **stats_interval** | string | no | 1s | Interval for collecting `TaskStats`. |
+| **allow_privileged** | bool | no | true | If set to `false`, driver will deny running privileged jobs. |
 
 **Task Config**
 

containerd/containerd.go

@@ -95,6 +95,10 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 
 	opts = append(opts, oci.WithImageConfigArgs(containerConfig.Image, args))
 
+	if !d.config.AllowPrivileged && config.Privileged {
+		return nil, fmt.Errorf("Running privileged jobs are not allowed. Set allow_privileged to true in plugin config to allow running privileged jobs.")
+	}
+
 	// Enable privileged mode.
 	if config.Privileged {
 		opts = append(opts, oci.WithPrivileged)

containerd/driver.go

@@ -79,6 +79,10 @@ var (
 		),
 		"containerd_runtime": hclspec.NewAttr("containerd_runtime", "string", true),
 		"stats_interval":     hclspec.NewAttr("stats_interval", "string", false),
+		"allow_privileged": hclspec.NewDefault(
+			hclspec.NewAttr("allow_privileged", "bool", false),
+			hclspec.NewLiteral("true"),
+		),
 	})
 
 	// taskConfigSpec is the specification of the plugin's configuration for
@@ -130,6 +134,7 @@ type Config struct {
 	Enabled           bool   `codec:"enabled"`
 	ContainerdRuntime string `codec:"containerd_runtime"`
 	StatsInterval     string `codec:"stats_interval"`
+	AllowPrivileged   bool   `codec:"allow_privileged"`
 }
 
 // Volume, bind, and tmpfs type mounts are supported.

example/privileged_not_allowed.nomad

@@ -0,0 +1,21 @@
+job "privileged-not-allowed" {
+  datacenters = ["dc1"]
+
+  group "privileged-not-allowed-group" {
+    task "privileged-not-allowed-task" {
+      driver = "containerd-driver"
+
+      config {
+        image           = "ubuntu:16.04"
+        command         = "sleep"
+        args            = ["600s"]
+        privileged      = true
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

tests/008-test-allow-privileged.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+source $SRCDIR/utils.sh
+job_name=privileged-not-allowed
+
+# allow_privileged=false set in the plugin config, should deny all privileged jobs.
+test_allow_privileged() {
+    pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
+
+    cp agent.hcl agent.hcl.bkp
+
+    sed -i '8 i \    allow_privileged = false' agent.hcl
+    sudo systemctl restart nomad
+    is_systemd_service_active "nomad.service" true
+
+    echo "INFO: Starting nomad ${job_name} job using nomad-driver-containerd."
+    nomad job run privileged_not_allowed.nomad
+    # Sleep for 5 seconds, to allow ${alloc_id} to get populated.
+    sleep 5s
+
+    echo "INFO: Checking status of ${job_name} job."
+    alloc_id=$(nomad job status ${job_name}|grep failed|awk 'NR==1'|cut -d ' ' -f 1)
+    output=$(nomad alloc status $alloc_id)
+    echo -e "$output" |grep "Running privileged jobs are not allowed" &>/dev/null
+    if [ $? -ne 0 ];then
+       echo "ERROR: ${job_name} should have failed to run."
+       return 1
+    fi
+
+    echo "INFO: purge nomad ${job_name} job."
+    nomad job stop -purge ${job_name}
+
+    mv agent.hcl.bkp agent.hcl
+    popd
+}
+
+cleanup() {
+    if [ -f agent.hcl.bkp ]; then
+       mv agent.hcl.bkp agent.hcl
+    fi
+    sudo systemctl restart nomad
+    is_systemd_service_active "nomad.service" false 
+}
+
+trap cleanup EXIT
+
+test_allow_privileged

tests/run_tests.sh

@@ -132,7 +132,7 @@ EOF
         sudo systemctl unmask containerd
         echo "INFO: Starting containerd daemon."
         sudo systemctl start containerd
-	is_systemd_service_active "containerd.service"
+	is_systemd_service_active "containerd.service" false
 
 	# Remove default golang (1.7.3) and install a custom version (1.14.3) of golang.
 	# This is required for supporting go mod, and to be able to compile nomad-driver-containerd.
@@ -184,7 +184,7 @@ EOF
 
 	echo "INFO: Starting nomad server and nomad-driver-containerd."
 	sudo systemctl start nomad
-	is_systemd_service_active "nomad.service"
+	is_systemd_service_active "nomad.service" false
 	popd
 }
 
@@ -216,20 +216,4 @@ is_containerd_driver_active() {
 	fi
 }
 
-is_systemd_service_active() {
-	local service_name=$1
-	i="0"
-	while test $i -lt 5 && !(systemctl -q is-active "$service_name"); do
-		printf "INFO: %s is down, sleep for 4 seconds.\n" $service_name
-		sleep 4s
-		i=$[$i+1]
-	done
-
-	if [ $i -ge 5 ]; then
-		printf "ERROR: %s didn't come up. exit 1.\n" $service_name
-		exit 1
-	fi
-	printf "INFO: %s is up and running\n" $service_name
-}
-
 main "$@"

tests/utils.sh

@@ -25,3 +25,25 @@ is_container_active() {
                 exit 1
         fi
 }
+
+is_systemd_service_active() {
+	local service_name=$1
+	local is_sleep=$2
+
+	i="0"
+	while test $i -lt 5 && !(systemctl -q is-active "$service_name"); do
+		printf "INFO: %s is down, sleep for 4 seconds.\n" $service_name
+		sleep 4s
+		i=$[$i+1]
+		done
+
+	if [ $i -ge 5 ]; then
+	printf "ERROR: %s didn't come up. exit 1.\n" $service_name
+		exit 1
+	fi
+
+	if [ "$is_sleep" = true ]; then
+		sleep 7s
+	fi
+	printf "INFO: %s is up and running\n" $service_name
+}