Merge pull request #2261 from gravitl/NET-39/nm-quick-refactor

NET-39 nm-quick.sh refactor (SSL certificates)

compose/docker-compose-emqx.yml

@@ -50,6 +50,7 @@ services:
     restart: unless-stopped
     volumes:
       - /root/Caddyfile:/etc/caddy/Caddyfile
+      - /root/certs:/root/certs
       - caddy_data:/data
       - caddy_conf:/config
     ports:

compose/docker-compose.ee.yml

@@ -57,6 +57,7 @@ services:
     restart: unless-stopped
     volumes:
       - /root/Caddyfile:/etc/caddy/Caddyfile
+      - /root/certs:/root/certs
       - caddy_data:/data
       - caddy_conf:/config
     ports:
@@ -118,7 +119,9 @@ services:
     depends_on:
       - netmaker
     environment:
-      SERVER_BROKER_ENDPOINT: "ws://mq:1883"
+      MQ_PASSWORD: "REPLACE_MQ_PASSWORD"
+      MQ_USERNAME: "REPLACE_MQ_USERNAME"
+      MQ_URL: "ws://mq:1883"
       BROKER_ENDPOINT: "wss://broker.NETMAKER_BASE_DOMAIN"
       PROMETHEUS: "on"
       VERBOSITY: "1"

compose/docker-compose.reference.yml

@@ -64,6 +64,7 @@ services:
     restart: unless-stopped
     volumes:
       - /root/Caddyfile:/etc/caddy/Caddyfile # Config file for Caddy
+      - /root/certs:/root/certs
       - caddy_data:/data
       - caddy_conf:/config
     ports:

compose/docker-compose.yml

@@ -56,6 +56,7 @@ services:
       - "host.docker.internal:host-gateway"
     volumes:
       - /root/Caddyfile:/etc/caddy/Caddyfile
+      - /root/certs:/root/certs
       - caddy_data:/data
       - caddy_conf:/config
     ports:

docker/Caddyfile

@@ -1,51 +1,51 @@
-{
-        # ZeroSSL account
-        # acme_ca https://acme.zerossl.com/v2/DV90
-        email YOUR_EMAIL
-}
-
 # Dashboard
 https://dashboard.NETMAKER_BASE_DOMAIN {
-        # Apply basic security headers
-        header {
-                # Enable cross origin access to *.NETMAKER_BASE_DOMAIN
-                Access-Control-Allow-Origin *.NETMAKER_BASE_DOMAIN
-                # Enable HTTP Strict Transport Security (HSTS)
-                Strict-Transport-Security "max-age=31536000;"
-                # Enable cross-site filter (XSS) and tell browser to block detected attacks
-                X-XSS-Protection "1; mode=block"
-                # Disallow the site to be rendered within a frame on a foreign domain (clickjacking protection)
-                X-Frame-Options "SAMEORIGIN"
-                # Prevent search engines from indexing
-                X-Robots-Tag "none"
-                # Remove the server name
-                -Server
-        }
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	# Apply basic security headers
+	header {
+		# Enable cross origin access to *.NETMAKER_BASE_DOMAIN
+		Access-Control-Allow-Origin *.NETMAKER_BASE_DOMAIN
+		# Enable HTTP Strict Transport Security (HSTS)
+		Strict-Transport-Security "max-age=31536000;"
+		# Enable cross-site filter (XSS) and tell browser to block detected attacks
+		X-XSS-Protection "1; mode=block"
+		# Disallow the site to be rendered within a frame on a foreign domain (clickjacking protection)
+		X-Frame-Options "SAMEORIGIN"
+		# Prevent search engines from indexing
+		X-Robots-Tag "none"
+		# Remove the server name
+		-Server
+	}
 
-        reverse_proxy http://netmaker-ui
+	reverse_proxy http://netmaker-ui
 }
 
 # API
 https://api.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://netmaker:8081
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy http://netmaker:8081
 }
 
 # STUN
 https://stun.NETMAKER_BASE_DOMAIN {
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
 	reverse_proxy netmaker:3478
 }
 
 # TURN
 https://turn.NETMAKER_BASE_DOMAIN {
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
 	reverse_proxy host.docker.internal:3479
 }
 
-#TURN API
+# TURN API
 https://turnapi.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://host.docker.internal:8089
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+    reverse_proxy http://host.docker.internal:8089
 }
 
 # MQ
 wss://broker.NETMAKER_BASE_DOMAIN {
-        reverse_proxy ws://mq:8883 # For EMQX websockets use `reverse_proxy ws://mq:8083`
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy ws://mq:8883 # For EMQX websockets use `reverse_proxy ws://mq:8083`
 }

docker/Caddyfile-EE

@@ -1,66 +1,69 @@
-{
-        # ZeroSSL account
-        acme_ca https://acme.zerossl.com/v2/DV90
-        email YOUR_EMAIL
-}
-
 # Dashboard
 https://dashboard.NETMAKER_BASE_DOMAIN {
-        # Apply basic security headers
-        header {
-                # Enable cross origin access to *.NETMAKER_BASE_DOMAIN
-                Access-Control-Allow-Origin *.NETMAKER_BASE_DOMAIN
-                # Enable HTTP Strict Transport Security (HSTS)
-                Strict-Transport-Security "max-age=31536000;"
-                # Enable cross-site filter (XSS) and tell browser to block detected attacks
-                X-XSS-Protection "1; mode=block"
-                # Disallow the site to be rendered within a frame on a foreign domain (clickjacking protection)
-                X-Frame-Options "SAMEORIGIN"
-                # Prevent search engines from indexing
-                X-Robots-Tag "none"
-                # Remove the server name
-                -Server
-        }
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	# Apply basic security headers
+	header {
+		# Enable cross origin access to *.NETMAKER_BASE_DOMAIN
+		Access-Control-Allow-Origin *.NETMAKER_BASE_DOMAIN
+		# Enable HTTP Strict Transport Security (HSTS)
+		Strict-Transport-Security "max-age=31536000;"
+		# Enable cross-site filter (XSS) and tell browser to block detected attacks
+		X-XSS-Protection "1; mode=block"
+		# Disallow the site to be rendered within a frame on a foreign domain (clickjacking protection)
+		X-Frame-Options "SAMEORIGIN"
+		# Prevent search engines from indexing
+		X-Robots-Tag "none"
+		# Remove the server name
+		-Server
+	}
 
-        reverse_proxy http://netmaker-ui
+	reverse_proxy http://netmaker-ui
 }
 
 # Netmaker Exporter
 https://netmaker-exporter.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://netmaker-exporter:8085
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy http://netmaker-exporter:8085
 }
 
 # Prometheus
 https://prometheus.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://prometheus:9090
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy http://prometheus:9090
 }
 
 # Grafana
 https://grafana.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://grafana:3000
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy http://grafana:3000
 }
 
 # API
 https://api.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://netmaker:8081
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy http://netmaker:8081
 }
 
 # STUN
 https://stun.NETMAKER_BASE_DOMAIN {
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
 	reverse_proxy netmaker:3478
 }
 
 # TURN
 https://turn.NETMAKER_BASE_DOMAIN {
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
 	reverse_proxy host.docker.internal:3479
 }
 
-#TURN API
+# TURN API
 https://turnapi.NETMAKER_BASE_DOMAIN {
-        reverse_proxy http://host.docker.internal:8089
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy http://host.docker.internal:8089
 }
 
 # MQ
 wss://broker.NETMAKER_BASE_DOMAIN {
-        reverse_proxy ws://mq:8883
+	tls /root/certs/fullchain.pem /root/certs/privkey.pem
+	reverse_proxy ws://mq:8883
 }

scripts/nm-certs.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+CONFIG_FILE=netmaker.env
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+
+# get and check the config
+if [ ! -f "$SCRIPT_DIR/$CONFIG_FILE" ]; then
+	echo "Config file missing"
+	exit 1
+fi
+source "$SCRIPT_DIR/$CONFIG_FILE"
+if [ -z "$NM_DOMAIN" ] || [ -z "$NM_EMAIL" ]; then
+	echo "Config not valid"
+	exit 1
+fi
+
+# TODO make sure this doesnt break, parse `certbot certificates` if yes
+CERT_DIR="$SCRIPT_DIR/letsencrypt/live/stun.$NM_DOMAIN"
+
+echo "Setting up SSL certificates..."
+
+# preserve the env state
+RESTART_CADDY=false
+if [ -n "$(docker ps | grep caddy)" ]; then
+	echo "Caddy is running, stopping for now..."
+	RESTART_CADDY=true
+	docker-compose -f /root/docker-compose.yml stop caddy
+fi
+
+CERTBOT_PARAMS=$(cat <<EOF
+certonly --standalone \
+	--non-interactive --agree-tos \
+	-m "$NM_EMAIL" \
+	-d "stun.$NM_DOMAIN" \
+	-d "api.$NM_DOMAIN" \
+	-d "broker.$NM_DOMAIN" \
+	-d "dashboard.$NM_DOMAIN" \
+	-d "turn.$NM_DOMAIN" \
+	-d "turnapi.$NM_DOMAIN" \
+	-d "netmaker-exporter.$NM_DOMAIN" \
+	-d "grafana.$NM_DOMAIN" \
+	-d "prometheus.$NM_DOMAIN"
+EOF
+)
+
+# generate an entrypoint for zerossl-certbot
+cat <<EOF >"$SCRIPT_DIR/certbot-entry.sh"
+#!/bin/sh
+# deps
+apk add bash curl
+# zerossl
+wget -qO zerossl-bot.sh "https://github.com/zerossl/zerossl-bot/raw/master/zerossl-bot.sh"
+chmod +x zerossl-bot.sh
+# request the certs
+./zerossl-bot.sh "$CERTBOT_PARAMS"
+EOF
+chmod +x certbot-entry.sh
+
+# request certs
+sudo docker run -it --rm --name certbot \
+	-p 80:80 -p 443:443 \
+	-v "$SCRIPT_DIR/certbot-entry.sh:/opt/certbot/certbot-entry.sh" \
+	-v "$SCRIPT_DIR/letsencrypt:/etc/letsencrypt" \
+	--entrypoint "/opt/certbot/certbot-entry.sh" \
+	certbot/certbot
+
+# clean up TODO enable
+#rm "$SCRIPT_DIR/certbot-entry.sh"
+
+# check if successful
+if [ ! -f "$CERT_DIR"/fullchain.pem ]; then
+	# fallback to letsencrypt-certbot
+	sudo docker run -it --rm --name certbot \
+		-p 80:80 -p 443:443 \
+		-v "$SCRIPT_DIR/letsencrypt:/etc/letsencrypt" \
+		--entrypoint "/opt/certbot/certbot-entry.sh" \
+		certbot/certbot "$CERTBOT_PARAMS"
+	if [ ! -f "$CERT_DIR"/fullchain.pem ]; then
+		echo "Missing file: $CERT_DIR/fullchain.pem"
+		echo "SSL certificates failed"
+		exit 1
+	fi
+fi
+
+# copy for mounting
+mkdir -p certs
+cp -L "$CERT_DIR/fullchain.pem" /root/certs/fullchain.pem
+cp -L "$CERT_DIR/privkey.pem" /root/certs/privkey.pem
+
+echo "SSL certificates ready"
+
+# preserve the env state
+if [ "$RESTART_CADDY" = true ]; then
+	echo "Starting Caddy..."
+	docker-compose -f /root/docker-compose.yml start caddy
+fi
+
+# install crontab
+ln -sfn "$SCRIPT_DIR"/nm-certs.sh /etc/cron.monthly/nm-certs.sh

scripts/nm-quick.sh

@@ -1,31 +1,22 @@
 #!/bin/bash
 
+CONFIG_FILE=netmaker.env
+# location of nm-quick.sh (usually `/root`)
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+CONFIG_PATH="$SCRIPT_DIR/$CONFIG_FILE"
 LATEST=$(curl -s https://api.github.com/repos/gravitl/netmaker/releases/latest | grep "tag_name" | cut -d : -f 2,3 | tr -d [:space:],\")
 
-print_logo() { (
-	cat <<"EOF"
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-                                                                                         
- __   __     ______     ______   __    __     ______     __  __     ______     ______    
-/\ "-.\ \   /\  ___\   /\__  _\ /\ "-./  \   /\  __ \   /\ \/ /    /\  ___\   /\  == \   
-\ \ \-.  \  \ \  __\   \/_/\ \/ \ \ \-./\ \  \ \  __ \  \ \  _"-.  \ \  __\   \ \  __<   
- \ \_\\"\_\  \ \_____\    \ \_\  \ \_\ \ \_\  \ \_\ \_\  \ \_\ \_\  \ \_____\  \ \_\ \_\ 
-  \/_/ \/_/   \/_____/     \/_/   \/_/  \/_/   \/_/\/_/   \/_/\/_/   \/_____/   \/_/ /_/ 
-                                                                                                                                                                                                 
-
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-EOF
-); }
-
 if [ $(id -u) -ne 0 ]; then
 	echo "This script must be run as root"
 	exit 1
 fi
 
+# read the config file
+if [ -f "$CONFIG_PATH" ]; then
+	echo "Reading config from $CONFIG_PATH"
+	source "$CONFIG_PATH"
+fi
+
 unset INSTALL_TYPE
 unset BUILD_TYPE
 unset BUILD_TAG
@@ -246,22 +237,42 @@ confirm() { (
 		[Nn]*)
 			echo "exiting..."
 			exit 1
+			# TODO start from the beginning instead
 			;;
 		*) echo "Please answer yes or no." ;;
 		esac
 	done
 ) }
 
+save_config() { (
+	echo "Saving the config to $CONFIG_PATH"
+	touch "$CONFIG_PATH"
+	# email
+	if grep -q "^NM_EMAIL=" "$CONFIG_PATH"; then
+		sed -i "s/NM_EMAIL=.*/NM_EMAIL=$EMAIL/" "$CONFIG_PATH"
+	else
+		echo "NM_EMAIL=$EMAIL" >>"$CONFIG_PATH"
+	fi
+	# domain
+	if grep -q "^NM_DOMAIN=" "$CONFIG_PATH"; then
+		sed -i "s/NM_DOMAIN=.*/NM_DOMAIN=$NETMAKER_BASE_DOMAIN/" "$CONFIG_PATH"
+	else
+		echo "NM_DOMAIN=$NETMAKER_BASE_DOMAIN" >>"$CONFIG_PATH"
+	fi
+); }
+
 # local_install_setup - builds artifacts based on specified branch locally to use in install
 local_install_setup() { (
 	rm -rf netmaker-tmp
 	mkdir netmaker-tmp
 	cd netmaker-tmp
-	git clone https://www.github.com/gravitl/netmaker
+	git clone --single-branch --depth=1 --branch=$BUILD_TAG https://www.github.com/gravitl/netmaker
 	cd netmaker
-	git checkout $BUILD_TAG
-	git pull origin $BUILD_TAG
-	docker build --no-cache --build-arg version=$IMAGE_TAG -t gravitl/netmaker:$IMAGE_TAG .
+	if test -z "$NM_SKIP_BUILD"; then
+		docker build --no-cache --build-arg version=$IMAGE_TAG -t gravitl/netmaker:$IMAGE_TAG .
+	else
+		echo "Skipping build on NM_SKIP_BUILD"
+	fi
 	if [ "$INSTALL_TYPE" = "ee" ]; then
 		cp compose/docker-compose.ee.yml /root/docker-compose.yml
 		cp docker/Caddyfile-EE /root/Caddyfile
@@ -269,6 +280,7 @@ local_install_setup() { (
 		cp compose/docker-compose.yml /root/docker-compose.yml
 		cp docker/Caddyfile /root/Caddyfile
 	fi
+	cp scripts/nm-certs.sh /root/nm-certs.sh
 	cp docker/mosquitto.conf /root/mosquitto.conf
 	cp docker/wait.sh /root/wait.sh
 	cd ../../
@@ -308,22 +320,14 @@ install_dependencies() {
 		dependencies="git wireguard wget jq docker.io docker-compose"
 		update_cmd='pkg update'
 		install_cmd='pkg install -y'
-	elif [ -f /etc/turris-version ]; then
-		dependencies="git wireguard-tools bash jq docker.io docker-compose"
-		OS="TurrisOS"
-		update_cmd='opkg update'
-		install_cmd='opkg install'
-	elif [ -f /etc/openwrt_release ]; then
-		dependencies="git wireguard-tools bash jq docker.io docker-compose"
-		OS="OpenWRT"
-		update_cmd='opkg update'
-		install_cmd='opkg install'
 	else
 		install_cmd=''
 	fi
 
 	if [ -z "${install_cmd}" ]; then
 		echo "OS unsupported for automatic dependency install"
+		# TODO shouldnt exit, check if deps available, if not
+		#  ask the user to install manually and continue when ready
 		exit 1
 	fi
 
@@ -395,7 +399,8 @@ set_install_vars() {
 	fi
 
 	NETMAKER_BASE_DOMAIN=nm.$(echo $IP_ADDR | tr . -).nip.io
-	COREDNS_IP=$(ip route get 1 | sed -n 's/^.*src \([0-9.]*\) .*$/\1/p')
+	# TODO dead code?
+	# COREDNS_IP=$(ip route get 1 | sed -n 's/^.*src \([0-9.]*\) .*$/\1/p')
 	SERVER_PUBLIC_IP=$IP_ADDR
 	MASTER_KEY=$(
 		tr -dc A-Za-z0-9 </dev/urandom | head -c 30
@@ -440,7 +445,7 @@ set_install_vars() {
 	echo "             broker.$NETMAKER_BASE_DOMAIN"
 	echo "               stun.$NETMAKER_BASE_DOMAIN"
 	echo "               turn.$NETMAKER_BASE_DOMAIN"
-	echo "               turnapi.$NETMAKER_BASE_DOMAIN"
+	echo "            turnapi.$NETMAKER_BASE_DOMAIN"
 
 	if [ "$INSTALL_TYPE" = "ee" ]; then
 		echo "         prometheus.$NETMAKER_BASE_DOMAIN"
@@ -479,12 +484,18 @@ set_install_vars() {
 	unset GET_EMAIL
 	unset RAND_EMAIL
 	RAND_EMAIL="$(echo $RANDOM | md5sum | head -c 16)@email.com"
+	# suggest the prev email or a random one
+	EMAIL_SUGGESTED=${NM_EMAIL:-$RAND_EMAIL}
 	if [ -z $AUTO_BUILD ]; then
-		read -p "Email Address for Domain Registration (click 'enter' to use $RAND_EMAIL): " GET_EMAIL
+		read -p "Email Address for Domain Registration (click 'enter' to use $EMAIL_SUGGESTED): " GET_EMAIL
 	fi
 	if [ -z "$GET_EMAIL" ]; then
-		echo "using rand email"
-		EMAIL="$RAND_EMAIL"
+		EMAIL="$EMAIL_SUGGESTED"
+		if [ "$EMAIL" = "$NM_EMAIL" ]; then
+			echo "using config email"
+		else
+			echo "using rand email"
+		fi
 	else
 		EMAIL="$GET_EMAIL"
 	fi
@@ -546,7 +557,7 @@ set_install_vars() {
 		read -p "TURN Username (click 'enter' to use 'netmaker'): " GET_TURN_USERNAME
 	fi
 	if [ -z "$GET_TURN_USERNAME" ]; then
-		echo "using default username for mq"
+		echo "using default username for TURN"
 		TURN_USERNAME="netmaker"
 	else
 		TURN_USERNAME="$GET_TURN_USERNAME"
@@ -603,6 +614,7 @@ set_install_vars() {
 
 	confirm
 
+	save_config
 }
 
 # install_netmaker - sets the config files and starts docker-compose
@@ -614,6 +626,7 @@ install_netmaker() {
 
 	wait_seconds 3
 
+	# TODO extract wgets to setup(), mirror local_setup()
 	echo "Pulling config files..."
 
 	COMPOSE_URL="https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/compose/docker-compose.yml"
@@ -622,9 +635,13 @@ install_netmaker() {
 		COMPOSE_URL="https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/compose/docker-compose.ee.yml"
 		CADDY_URL="https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/docker/Caddyfile-EE"
 	fi
+
 	if [ ! "$BUILD_TYPE" = "local" ]; then
-		wget -O /root/docker-compose.yml $COMPOSE_URL && wget -O /root/mosquitto.conf https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/docker/mosquitto.conf && wget -O /root/Caddyfile $CADDY_URL
-		wget -O /root/wait.sh https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/docker/wait.sh
+		wget -qO /root/docker-compose.yml $COMPOSE_URL
+		wget -qO /root/Caddyfile $CADDY_URL
+		wget -qO /root/mosquitto.conf "https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/docker/mosquitto.conf"
+		wget -qO /root/nm-certs.sh "https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/scripts/nm-certs.sh"
+		wget -qO /root/wait.sh "https://raw.githubusercontent.com/gravitl/netmaker/$BUILD_TAG/docker/wait.sh"
 	fi
 
 	chmod +x /root/wait.sh
@@ -661,7 +678,12 @@ install_netmaker() {
 
 	echo "Starting containers..."
 
-	docker-compose -f /root/docker-compose.yml up -d
+	# increase the timeouts
+	export DOCKER_CLIENT_TIMEOUT=120
+	export COMPOSE_HTTP_TIMEOUT=120
+
+	# start docker and rebuild containers / networks
+	docker-compose -f /root/docker-compose.yml up -d --force-recreate
 
 	wait_seconds 2
 
@@ -747,6 +769,16 @@ set -e
 # 6. get user input for variables
 set_install_vars
 
+# stop
+for name in "mq" "netmaker-ui" "coredns" "turn" "caddy" "netmaker"; do
+	if test -n "$(docker ps | grep name)"; then
+		docker stop $name
+	fi
+done
+
+# Fetch / update certs using certbot
+"$SCRIPT_DIR"/nm-certs.sh
+
 # 7. get and set config files, startup docker-compose
 install_netmaker
 

scripts/nm-upgrade-0-17-1-to-0-19-0.sh

@@ -27,6 +27,7 @@ backup_v17_files() {
 }
 
 backup_volumes() {
+  # TODO backup to /root/nm-backup
   cp -r /var/lib/docker/volumes/root_caddy_conf/ /var/lib/docker/volumes/root_caddy_conf-backup/
   cp -r /var/lib/docker/volumes/root_caddy_data/ /var/lib/docker/volumes/root_caddy_data-backup/
   cp -r /var/lib/docker/volumes/root_dnsconfig/ /var/lib/docker/volumes/root_dnsconfig-backup/