initial commit on new k8s templates

k8s/netclient/netclient-daemonset.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: netclient
+  labels:
+    app: netclient
+spec:
+  selector:
+    matchLabels:
+      app: netclient
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: netclient
+    spec:
+      hostNetwork: true
+      containers:
+      - name: netclient
+        image: gravitl/netclient:v0.7.2
+        command: ['bash', '-c', "netclient checkin -n $NETWORK; sleep $SLEEP"]
+        env:
+        - name: ACCESS_TOKEN
+          value: "XXXX"
+        - name: NETWORK
+          value: "YYYY"
+        - name: SLEEP
+          value: 30
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        - mountPath: /usr/bin/wg
+          name: wg
+        securityContext:
+          privileged: true
+      initContainers:
+      - name: netclient-join
+        image: gravitl/netclient:v0.7.2
+        command: ['bash', '-c', "netclient join -t $ACCESS_TOKEN --daemon off"]
+        env:
+        - name: ACCESS_TOKEN
+          value: "XXXX"
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        - mountPath: /usr/bin/wg
+          name: wg
+        securityContext:
+          privileged: true
+  volumes:
+  - hostPath:
+      path: /etc/netclient
+      type: DirectoryOrCreate
+    name: etc-netclient
+  - hostPath:
+      path: /usr/bin/wg
+      type: File
+    name: wg

k8s/netclient/netclient-template-doks-uspace.yaml

@@ -0,0 +1,43 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: netclient-1
+  labels:
+    app: netclient-1
+spec:
+  selector:
+    matchLabels:
+      app: netclient-1
+  template:
+    metadata:
+      labels:
+        app: netclient-1
+    spec:
+      hostNetwork: true
+      containers:
+      - name: netclient-1
+        image: gravitl/netclient:0.9.2-doks-uspace
+        env:
+        - name: NETCLIENT_ROAMING
+          value: "no"
+        - name: NETCLIENT_PORT
+          value: "51821"
+        - name: NETCLIENT_IS_STATIC
+          value: "yes"
+        - name: NETCLIENT_ENDPOINT
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: TOKEN
+          value: "<token>"
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        securityContext:
+          privileged: true
+      volumes:
+      - hostPath:
+          path: /etc/netclient
+          type: DirectoryOrCreate
+        name: etc-netclient
+

k8s/netclient/netclient-template-doks.yaml

@@ -0,0 +1,93 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: netclient-1
+  labels:
+    app: netclient-1
+spec:
+  selector:
+    matchLabels:
+      app: netclient-1
+  template:
+    metadata:
+      labels:
+        app: netclient-1
+    spec:
+      hostNetwork: true
+      containers:
+      - name: netclient-1
+        image: gravitl/netclient:0.9.2-doks
+        env:
+        - name: NETCLIENT_ROAMING
+          value: "no"
+        - name: NETCLIENT_PORT
+          value: "51821"
+        - name: NETCLIENT_IS_STATIC
+          value: "yes"
+        - name: NETCLIENT_ENDPOINT
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: TOKEN
+          value: "<token>"
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        - mountPath: /usr/bin/wg
+          name: wg
+        securityContext:
+          privileged: true
+      volumes:
+      - hostPath:
+          path: /etc/netclient
+          type: DirectoryOrCreate
+        name: etc-netclient
+      - hostPath:
+          path: /usr/bin/wg
+          type: File
+        name: wg
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: wireguard-controller
+  labels:
+    app: wireguard-controller
+spec:
+  selector:
+    matchLabels:
+      app: wireguard-controller
+  template:
+    metadata:
+      labels:
+        app: wireguard-controller
+    spec:
+      hostNetwork: true
+      containers:
+      - image: gravitl/netclient:0.9.2-doks
+        imagePullPolicy: IfNotPresent
+        name: wg-installer
+        command: ['bash', '-c']
+        args:
+          - while [ 1 ];
+            do if ! command -v wg &> /dev/null;
+            then echo "wireguard not installed, installing";
+            echo 'deb http://deb.debian.org/debian buster-backports main contrib non-free' > /etc/apt/sources.list.d/buster-backports.list;
+            apt update;
+            sudo apt -y install linux-headers-$(uname --kernel-release);
+            apt -y install wireguard wireguard-tools;
+            else echo "wireguard installed";
+            sleep 300;
+            fi;
+            done
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: rootfolder
+          mountPath: /
+      volumes:
+      - hostPath:
+          path: /
+          type: ""
+        name: rootfolder
+

k8s/netclient/netclient-template.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: netclient
+  labels:
+    app: netclient
+spec:
+  selector:
+    matchLabels:
+      app: netclient
+  template:
+    metadata:
+      labels:
+        app: netclient
+    spec:
+      hostNetwork: true
+      containers:
+      - name: netclient
+        image: gravitl/netclient:v0.5.11
+        command: ['bash', '-c', "/root/netclient join -t $ACCESS_TOKEN --daemon off --name $(echo $NODE_NAME| sed -e s/.$NETWORK//); while true; do /root/netclient checkin --dns on -n $NETWORK; sleep $SLEEP; done"]
+        env:
+        - name: ACCESS_TOKEN
+          value: "ACCESS_TOKEN_VALUE"
+        - name: NETWORK
+          value: "microk8s"
+        - name: SLEEP
+          value: "30"
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        - mountPath: /usr/bin/wg
+          name: wg
+        - mountPath: /var/run/dbus/system_bus_socket
+          name: systemd-bus-socket
+        securityContext:
+          privileged: true
+      volumes:
+      - hostPath:
+          path: /etc/netclient
+          type: DirectoryOrCreate
+        name: etc-netclient
+      - hostPath:
+          path: /usr/bin/wg
+          type: File
+        name: wg 
+      - hostPath:
+          path: /usr/bin/resolvectl
+          type: File
+        name: resolvectl
+      - hostPath:
+          path: /var/run/dbus/system_bus_socket
+          type: ""
+        name: systemd-bus-socket

k8s/netmaker-ha/kustomization.yml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: netmaker
+resources:
+  - database/
+  - server/
+  - mq/
+  - ui/
+  - netmaker-deploy.yml
+  - netmaker-ing.yml
+  - netmaker-pvc.yml
+  - netmaker-sa.yml
+  - netmaker-ns.yml

k8s/netmaker-ha/netmaker-ing.yml

@@ -0,0 +1,76 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: &app "netmaker"
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    nginx.ingress.kubernetes.io/secure-backends: "true"
+    kubernetes.io/ingress.class: "nginx"
+spec:
+  rules:
+  - host: netmaker.<path:apps-kv/data/general#DN>
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: *app
+            port:
+              number: 80
+  tls:
+  - hosts:
+    - netmaker.<path:apps-kv/data/general#DN>
+    secretName: netmaker-tls
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: netmaker-api
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    kubernetes.io/ingress.class: "nginx"
+spec:
+  rules:
+  - host: api.netmaker.<path:apps-kv/data/general#DN>
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: netmaker-api
+            port:
+              number: 8081
+  tls:
+  - hosts:
+    - api.netmaker.<path:apps-kv/data/general#DN>
+    secretName: netmaker-tls
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: netmaker-grpc
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+spec:
+  rules:
+  - host: grpc.netmaker.<path:apps-kv/data/general#DN>
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: netmaker-grpc
+            port:
+              number: 443
+  tls:
+  - hosts:
+    - grpc.netmaker.<path:apps-kv/data/general#DN>
+    secretName: netmaker-tls

k8s/netmaker-ha/netmaker-ns.yml

@@ -0,0 +1,4 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: netmaker

k8s/netmaker-ha/netmaker-pvc.yml

@@ -0,0 +1,26 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: netmaker
+spec:
+  storageClassName: managed-nfs-storage
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: netmaker-rqlite
+  annotations:
+    nfs.io/storage-path: "data01/netmaker/rqlite"
+spec:
+  storageClassName: managed-nfs-storage
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi

k8s/netmaker-ha/netmaker-sa.yml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netmaker

k8s/netmaker-ha/server/kustomization.yml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: netmaker
+resources:
+  - netmaker-deploy.yml
+  - netmaker-svc.yml

k8s/netmaker-ha/server/netmaker-backend-deploy.yml

@@ -0,0 +1,115 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: netmaker
+  name: netmaker
+spec:
+  replicas: 1
+  serviceName: netmaker-headless
+  selector:
+    matchLabels:
+      app: netmaker
+  template:
+    metadata:
+      labels:
+        app: netmaker
+    spec:
+      initContainers:
+      - name: init-sysctl
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/sh", "-c"]
+        args: ["sysctl -w net.ipv4.ip_forward=1 && sysctl -w net.ipv4.conf.all.src_valid_mark=1 && sysctl -w net.ipv6.conf.all.disable_ipv6=0 && sysctl -w net.ipv6.conf.all.forwarding=1"]
+        securityContext:
+          privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - netmaker
+            topologyKey: "kubernetes.io/hostname"
+      containers:
+      - env:
+        - name: SERVER_NAME
+          value: broker.nm.k8s.clustercat.com
+        - name: SERVER_API_CONN_STRING
+          value: api.nm.k8s.clustercat.com:443
+        - name: SERVER_HTTP_HOST
+          value: api.nm.k8s.clustercat.com
+        - name: API_PORT
+          value: "8081"
+        - name: WG_QUICK_USERSPACE_IMPLEMENTATION
+          value: wireguard-go
+        - name: DNS_MODE
+          value: "off"
+        - name: CLIENT_MODE
+          value: "on"
+        - name: DISPLAY_KEYS
+          value: "on"
+        - name: DATABASE
+          value: sqlite
+        - name: MASTER_KEY
+          value: cqYXwQGWiLKj
+        - name: PLATFORM
+          value: Kubernetes
+        - name: CORS_ALLOWED_ORIGIN
+          value: '*'
+        - name: MQ_HOST
+          value: "mq"
+        - name: MQ_PORT
+          value: "31883"
+        - name: PLATFORM
+          value: "Kubernetes"
+        - name: VERBOSITY
+          value: "3"
+        image: gravitl/netmaker:v0.14.3
+        imagePullPolicy: Always
+        name: netmaker
+        ports:
+        - containerPort: 8081
+          protocol: TCP
+        - containerPort: 31821
+          protocol: UDP
+        - containerPort: 31822
+          protocol: UDP
+        - containerPort: 31823
+          protocol: UDP
+        - containerPort: 31824
+          protocol: UDP
+        - containerPort: 31825
+          protocol: UDP
+        - containerPort: 31826
+          protocol: UDP
+        - containerPort: 31827
+          protocol: UDP
+        - containerPort: 31828
+          protocol: UDP
+        - containerPort: 31829
+          protocol: UDP
+        - containerPort: 31830
+          protocol: UDP
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_MODULE
+        volumeMounts:
+        - mountPath: /etc/netmaker/
+          name: shared-certs
+        - mountPath: /root/data
+          name: netmaker-data
+      volumes:
+      - name: shared-certs
+        persistentVolumeClaim:
+          claimName: shared-certs-pvc
+      - name: netmaker-data
+        persistentVolumeClaim:
+          claimName: netmaker-data-pvc

k8s/netmaker-ha/server/netmaker-backend-svc.yml

@@ -0,0 +1,32 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: &labels
+    app: netmaker-backend
+    part-of: netmaker
+  name: netmaker-api
+spec:
+  ports:
+  - port: 8081
+    protocol: TCP
+    targetPort: backend
+  selector: *labels
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: &labels
+    app: netmaker-backend
+    part-of: netmaker
+  name: netmaker-grpc
+spec:
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: backend-grpc
+  selector: *labels
+  sessionAffinity: None
+  type: ClusterIP

k8s/netmaker-singlenode/01-namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: netmaker

k8s/netmaker-singlenode/02-pvc.yaml

@@ -0,0 +1,26 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: nm-sqldata
+  namespace: netmaker
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: STORAGE_CLASS_RWO
+  resources:
+    requests:
+      storage: 500Mi
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: nm-mq-certs
+  namespace: netmaker
+spec:
+  accessModes:
+  - ReadWriteMany
+  storageClassName: STORAGE_CLASS_RWX
+  resources:
+    requests:
+      storage: 50Mi

k8s/netmaker-singlenode/03-svc.yaml

@@ -0,0 +1,107 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+  name: 'netmaker-wireguard'
+spec:
+  externalTrafficPolicy: Local
+  type: NodePort
+  ports:
+  - port: 31821
+    nodePort: 31821
+    protocol: UDP
+    targetPort: 31821
+    name: wg-iface-31821
+  - port: 31822
+    nodePort: 31822
+    protocol: UDP
+    targetPort: 31822
+    name: wg-iface-31822
+  - port: 31823
+    nodePort: 31823
+    protocol: UDP
+    targetPort: 31823
+    name: wg-iface-31823
+  - port: 31824
+    nodePort: 31824
+    protocol: UDP
+    targetPort: 31824
+    name: wg-iface-31824
+  - port: 31825
+    nodePort: 31825
+    protocol: UDP
+    targetPort: 31825
+    name: wg-iface-31825
+  selector:
+    app: 'netmaker'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: 'netmaker-rest'
+spec:
+  ports:
+  - name: rest
+    port: 8081
+    protocol: TCP
+    targetPort: 8081
+  selector:
+    app: 'netmaker'
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: netmaker
+  labels:
+    app: netmaker
+  name: netmaker-ui
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: netmaker
+  sessionAffinity: None
+  type: ClusterIP
+# ---
+# apiVersion: v1
+# kind: Service
+# metadata:
+#   name: mq
+#   namespace: netmaker
+# spec:
+#   ports:
+#   - name: mqtt
+#     port: 1883
+#     protocol: TCP
+#     targetPort: mqtt
+#   - name: mqtt2
+#     port: 8883
+#     protocol: TCP
+#     targetPort: mqtt2    
+#   selector:
+#     app.kubernetes.io/instance: mosquitto
+#     app.kubernetes.io/name: mosquitto
+#   sessionAffinity: None
+# ---
+# apiVersion: v1
+# kind: Service
+# metadata:
+#   labels:
+#   name: 'netmaker-mqtt'
+# spec:
+#   externalTrafficPolicy: Local
+#   type: NodePort
+#   selector:
+#     app.kubernetes.io/instance: mosquitto
+#     app.kubernetes.io/name: mosquitto
+#   ports:
+#   - port: 31883
+#     nodePort: 31883
+#     protocol: TCP
+#     targetPort: 8883
+#     name: nm-mqtt

k8s/netmaker-singlenode/04-configmap.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: v1
+data:
+  mosquitto.conf: |
+    per_listener_settings true
+    listener 8883
+    allow_anonymous false
+    require_certificate true
+    use_identity_as_username true
+    cafile /mosquitto/certs/root.pem
+    certfile /mosquitto/certs/server.pem
+    keyfile /mosquitto/certs/server.key
+    listener 1883 
+    allow_anonymous true  
+kind: ConfigMap
+metadata:
+  name: mosquitto-config
+  namespace: netmaker

k8s/netmaker-singlenode/05-ingress-nginx.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nm-api-ingress
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    cert-manager.io/cluster-issuer: "letsencrypt-nginx"
+    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - api.BASE_DOMAIN
+    secretName: nm-api-tls
+  rules:
+  - host: api.BASE_DOMAIN
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: netmaker-rest
+            port:
+              number: 8081
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nm-ui-ingress
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    cert-manager.io/cluster-issuer: "letsencrypt-nginx"
+    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - dashboard.BASE_DOMAIN
+    secretName: nm-ui-tls
+  rules:
+  - host: dashboard.BASE_DOMAIN
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: netmaker-ui
+            port:
+              number: 80

k8s/netmaker-singlenode/05-ingress-traefik.yaml

@@ -0,0 +1,75 @@
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: secheaders
+  namespace: netmaker
+spec:
+  headers:
+    stsIncludeSubdomains: true
+    stsPreload: true
+    stsSeconds: 31536000
+    forceSTSHeader: true
+    sslRedirect: true
+    referrerPolicy: "same-origin"
+    frameDeny: true
+    contentTypeNosniff: true
+    browserXssFilter: true
+    accessControlAllowMethods: ["GET", "OPTIONS", "PUT"]
+    accessControlMaxAge: 100
+    customFrameOptionsValue: SAMEORIGIN
+    contentSecurityPolicy: frame-ancestors 'self'
+    permissionsPolicy: geolocation=(), microphone=()
+    referrerPolicy: no-referrer
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: nm-api-ingress-tls
+  namespace: netmaker
+spec:
+  entryPoints:
+    - websecure
+  routes:
+  - match: Host(`api.BASE_DOMAIN`)
+    kind: Rule
+    services:
+    - name: netmaker-api
+      port: 8081
+  tls:
+    certResolver: CERT_PROVIDER
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: nm-ui-ingress-tls
+  namespace: netmaker
+spec:
+  entryPoints:
+    - websecure
+  routes:
+  - match: Host(`dashboard.BASE_DOMAIN`)
+    kind: Rule
+    services:
+    - name: netmaker-ui
+      port: 80
+    middlewares:
+    - name: secheaders
+  tls:
+    certResolver:  CERT_PROVIDER
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRouteTCP
+metadata:
+  namespace: netmaker
+  name: nm-mq-ingress-tls
+spec:
+  entryPoints:
+    - websecure
+  routes:
+  - match: HostSNI(`broker.BASE_DOMAIN`)
+    services:
+      - name: netmaker-mq
+        port: 8883
+  tls:
+    passthrough: true

k8s/netmaker-singlenode/06-deployment.yaml

@@ -0,0 +1,156 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netmaker
+  namespace: netmaker
+  labels:
+    app: netmaker
+spec:
+  selector:
+    matchLabels:
+      app: netmaker
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: netmaker
+    spec:
+      initContainers:
+      - name: init-sysctl
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/sh", "-c"]
+        args: ["sysctl -w net.ipv4.ip_forward=1 && sysctl -w net.ipv4.conf.all.src_valid_mark=1 && sysctl -w net.ipv6.conf.all.disable_ipv6=0 && sysctl -w net.ipv6.conf.all.forwarding=1"]
+        securityContext:
+          privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - netmaker
+            topologyKey: "kubernetes.io/hostname"
+      containers:
+      - env:
+        - name: SERVER_NAME
+          value: broker.BASE_DOMAIN
+        - name: SERVER_API_CONN_STRING
+          value: api.BASE_DOMAIN:443
+        - name: SERVER_HTTP_HOST
+          value: api.BASE_DOMAIN
+        - name: API_PORT
+          value: "8081"
+        - name: WG_QUICK_USERSPACE_IMPLEMENTATION
+          value: wireguard-go
+        - name: DNS_MODE
+          value: "off"
+        - name: CLIENT_MODE
+          value: "on"
+        - name: DISPLAY_KEYS
+          value: "on"
+        - name: DATABASE
+          value: sqlite
+        - name: MASTER_KEY
+          value: REPLACE_MASTER_KEY
+        - name: PLATFORM
+          value: Kubernetes
+        - name: CORS_ALLOWED_ORIGIN
+          value: '*'
+        - name: MQ_HOST
+          value: "127.0.0.1"          
+#        Uncomment this section if using the Nginx config
+#        - name: MQ_PORT
+#          value: "31883"
+        - name: PLATFORM
+          value: "Kubernetes"
+        - name: VERBOSITY
+          value: "3"
+        image: gravitl/netmaker:v0.14.3
+        imagePullPolicy: Always
+        name: netmaker
+        ports:
+        - containerPort: 8081
+          protocol: TCP
+        - containerPort: 31821
+          protocol: UDP
+        - containerPort: 31822
+          protocol: UDP
+        - containerPort: 31823
+          protocol: UDP
+        - containerPort: 31824
+          protocol: UDP
+        - containerPort: 31825
+          protocol: d
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_MODULE
+        volumeMounts:
+        - mountPath: /etc/netmaker/
+          name: shared-certs
+        - mountPath: /root/data
+          name: netmaker-data
+      - name: netmaker-ui
+        image: gravitl/netmaker-ui:v0.14.3
+        ports:
+        - containerPort: 80
+        env:
+        - name: BACKEND_URL
+          value: "https://api.netmaker.NETMAKER_BASE_DOMAIN"
+      - image: eclipse-mosquitto:2.0.11-openssl
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          periodSeconds: 10
+          successThreshold: 1
+          tcpSocket:
+            port: 8883
+          timeoutSeconds: 1
+        name: mosquitto
+        ports:
+        - containerPort: 1883        
+          name: mqtt
+          protocol: TCP
+        - containerPort: 8883        
+          name: mqtt2
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          periodSeconds: 10
+          successThreshold: 1
+          tcpSocket:
+            port: 8883
+          timeoutSeconds: 1
+        resources: {}
+        startupProbe:
+          failureThreshold: 30
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 8883
+          timeoutSeconds: 1
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /mosquitto/config/mosquitto.conf
+          name: mosquitto-config
+          subPath: mosquitto.conf
+        - mountPath: /mosquitto/certs
+          name: shared-certs
+      volumes:
+      - name: nm-sqldata
+        persistentVolumeClaim:
+          claimName: nm-pvc-sqldata
+      - name: netmaker-mq-configmap
+        configMap:
+          name: netmaker-mq-configmap