External Secrets setup

Setup the external secrets helm charts
Convert hard coded secrets to external secrets
Add BitWarden CLI for serving external secrets

.gitignore

@@ -6,3 +6,4 @@ result
 # where I am storing my repo
 nix.conf
 manifests/postgres/charts
+manifests/external-secrets/charts

.gitlab-ci.yml

@@ -21,3 +21,16 @@ default:
         job: "Evaluate for builds"
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+
+"Build image":
+  stage: build
+  parallel:
+    matrix:
+      - IMG:
+          - img-bitwarden
+  script:
+    - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+    - nix build ".#${IMG}"
+    - podman load -i result
+    - podman tag "localhost/${IMG}:latest" "$CI_REGISTRY/greg/ci-images/${IMG}:latest"
+    - podman push "$CI_REGISTRY/greg/ci-images/${IMG}:latest"

manifests/README.md

@@ -1,5 +1,23 @@
 Stands up my personal infrastructure in a Kubernetes environment.
 
+# First Thing First
+
+In order to properly get things up and going, you will need to create a secret to allow
+the external secrets to log into BitWarden Password Manager. For obvious reasont this
+cannot be safely added to this repository. As such, it is suggested you create this
+manually.
+
+```bash
+kubectl create namespace bitwarden
+kubectl create secret generic bitwarden-cli --namespace bitwarden --from-literal=BW_USERNAME=my_username --from-literal=BW_PASSWORD=my_password
+# Alternative to the preceding line if you don't want the data in your shell history
+kubectl create secret generic bitwarden-cli  --namespace bitwarden --from-file=./BW_USERNAME.txt --from-file=./BW_PASSWORD.txt
+# And yet an entirely other option, if I'm logged into my own systems
+sudo cat /run/agenix/bw_secret | kubectl apply -f -
+```
+
+# Now Configure Cluster Services
+
 To apply this you need to install kubectl, kustomize, and helm. It can then by applied
 by simply invoking the command:
 

manifests/apply.sh

@@ -7,5 +7,10 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 cd "$SCRIPT_DIR"
 
 kubectl apply -k namespaces
+kustomize build external-secrets --enable-helm | kubectl apply -f -
+echo -e "\nPausing for a moment to allow the External Secrets stuff to get going\n"
+sleep 5
 kustomize build postgres --enable-helm | kubectl apply -f - --server-side --force-conflicts
+echo -e "\nPausing for a moment to allow Postgres stuff to get going\n"
+sleep 5
 kubectl apply -k .

manifests/bitwarden/cluster-stores.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: external-secrets.io/v1
+kind: ClusterSecretStore
+metadata:
+  name: bitwarden-login
+spec:
+  provider:
+    webhook:
+      url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/item/{{ .remoteRef.key }}"
+      headers:
+        Content-Type: application/json
+      result:
+        jsonPath: "$.data.login.{{ .remoteRef.property }}"
+---
+apiVersion: external-secrets.io/v1
+kind: ClusterSecretStore
+metadata:
+  name: bitwarden-fields
+spec:
+  provider:
+    webhook:
+      url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/item/{{ .remoteRef.key }}"
+      result:
+        jsonPath: '$.data.fields[?@.name=="{{ .remoteRef.property }}"].value'
+---
+apiVersion: external-secrets.io/v1
+kind: ClusterSecretStore
+metadata:
+  name: bitwarden-notes
+spec:
+  provider:
+    webhook:
+      url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/item/{{ .remoteRef.key }}"
+      result:
+        jsonPath: "$.data.notes"
+---
+apiVersion: external-secrets.io/v1
+kind: ClusterSecretStore
+metadata:
+  name: bitwarden-attachments
+spec:
+  provider:
+    webhook:
+      url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/attachment/{{ .remoteRef.property }}?itemid={{ .remoteRef.key }}"
+      result: {}

manifests/bitwarden/deployment.yaml

@@ -0,0 +1,70 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bitwarden-cli
+  labels:
+    app.kubernetes.io/instance: bitwarden-cli
+    app.kubernetes.io/name: bitwarden-cli
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: bitwarden-cli
+      app.kubernetes.io/instance: bitwarden-cli
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: bitwarden-cli
+        app.kubernetes.io/instance: bitwarden-cli
+    spec:
+      containers:
+        - name: bitwarden-cli
+          image: "registry.thehellings.com/greg/nixos-config/img-bitwarden:latest"
+          imagePullPolicy: Always
+          env:
+            - name: BW_CLIENTID
+              valueFrom:
+                secretKeyRef:
+                  name: bitwarden-cli
+                  key: BW_CLIENTID
+            - name: BW_CLIENTSECRET
+              valueFrom:
+                secretKeyRef:
+                  name: bitwarden-cli
+                  key: BW_CLIENTSECRET
+            - name: BW_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: bitwarden-cli
+                  key: BW_PASSWORD
+          ports:
+            - name: http
+              containerPort: 8087
+              protocol: TCP
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - -q
+                - http://127.0.0.1:8087/sync?force=true
+                - --post-data=''
+            initialDelaySeconds: 20
+            failureThreshold: 3
+            timeoutSeconds: 10
+            periodSeconds: 120
+          readinessProbe:
+            tcpSocket:
+              port: 8087
+            initialDelaySeconds: 20
+            failureThreshold: 3
+            timeoutSeconds: 1
+            periodSeconds: 10
+          startupProbe:
+            tcpSocket:
+              port: 8087
+            initialDelaySeconds: 10
+            failureThreshold: 30
+            timeoutSeconds: 1
+            periodSeconds: 5

manifests/bitwarden/kustomization.yaml

@@ -0,0 +1,6 @@
+namespace: bitwarden
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - cluster-stores.yaml

manifests/bitwarden/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: bitwarden-cli
+  labels:
+    app.kubernetes.io/instance: bitwarden-cli
+    app.kubernetes.io/name: bitwarden-cli
+  annotations:
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8087
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: bitwarden-cli
+    app.kubernetes.io/instance: bitwarden-cli

manifests/databases/postgres-gitlab.yaml

@@ -1,15 +1,3 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: postgres-user-gitlab
-  labels:
-    cnpg.io/reload: "true"
-type: kubernetes.io/basic-auth
-stringData:
-  # Currently this is set to "gitlab"
-  username: gitlab
-  password: gitlab
----
 apiVersion: postgresql.cnpg.io/v1
 kind: Database
 metadata:

manifests/databases/postgres-pgadmin.yaml

@@ -1,15 +1,4 @@
 apiVersion: v1
-kind: Secret
-metadata:
-  name: postgres-user-pgadmin
-  labels:
-    cnpg.io/reload: "true"
-type: kubernetes.io/basic-auth
-stringData:
-  username: pgadmin
-  password: pgadmin
----
-apiVersion: v1
 kind: ConfigMap
 metadata:
   name: config-pgadmin

manifests/external-secrets/kustomization.yaml

@@ -0,0 +1,13 @@
+namespace: external-secrets
+
+helmCharts:
+  # Install the operator first (with CRDs)
+  - name: external-secrets
+    repo: "https://charts.external-secrets.io/"
+    version: "0.17.0"
+    namespace: external-secrets
+    releaseName: external-secrets
+    includeCRDs: true
+    valuesInline:
+      crds:
+        create: true

manifests/kustomization.yaml

@@ -1,3 +1,5 @@
 resources:
   - namespaces
+  - bitwarden
+  - secrets
   - databases

manifests/namespaces/bitwarden.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: bitwarden

manifests/namespaces/external-secrets.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: external-secrets

manifests/namespaces/kustomization.yaml

@@ -1,2 +1,4 @@
 resources:
+  - external-secrets.yaml
+  - bitwarden.yaml
   - db.yaml

manifests/secrets/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - postgres-user-gitlab.yaml
+  - postgres-user-pgadmin.yaml

manifests/secrets/postgres-user-gitlab.yaml

@@ -0,0 +1,33 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: postgres-user-gitlab
+  namespace: db
+spec:
+  target:
+    name: postgres-user-gitlab
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5282ad16-c2dc-49d3-8fb3-b2e9012bab57
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5282ad16-c2dc-49d3-8fb3-b2e9012bab57
+        property: password

manifests/secrets/postgres-user-pgadmin.yaml

@@ -0,0 +1,33 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: postgres-user-pgadmin
+  namespace: db
+spec:
+  target:
+    name: postgres-user-pgadmin
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: f333d637-1667-499d-b9a0-b2e9012bd8b7
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: f333d637-1667-499d-b9a0-b2e9012bd8b7
+        property: password

modules/nixos/kubernetes.nix

@@ -20,7 +20,10 @@ in
   };
 
   config = lib.mkIf cfg.enable {
-    age.secrets.kubernetesToken.file = ../../secrets/kubernetes/kubernetesToken.age;
+    age.secrets = {
+      kubernetesToken.file = ../../secrets/kubernetes/kubernetesToken.age;
+      bw_secret.file = ../../secrets/kubernetes/bw_secret.age;
+    };
 
     environment.systemPackages = [
       pkgs.kubectl-cnpg

pkgs/default.nix

@@ -14,6 +14,7 @@ in
   hms = c ./hms { };
   inject-darwin = c ./inject-darwin.nix { };
   inject = c ./inject.nix { };
+  img-bitwarden = c ./img-bitwarden.nix { };
   qemu-hook = c ./qemu-hook.nix { };
   setup-ssh = c ./setup-ssh { };
   upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };

pkgs/img-bitwarden.nix

@@ -0,0 +1,44 @@
+{
+  bitwarden-cli,
+  cacert,
+  dockerTools,
+  lib,
+  writeShellApplication,
+  ...
+}:
+dockerTools.buildLayeredImage {
+  name = "img-bitwarden";
+  tag = "latest";
+  contents = [
+    dockerTools.binSh
+    dockerTools.caCertificates
+  ];
+  config = {
+    Cmd = [
+      (lib.getExe (writeShellApplication {
+        name = "bitwarden-cli-entrypoint.sh";
+        runtimeInputs = [ bitwarden-cli ];
+        text = ''
+          set -ex
+
+          # Uncomment if you need to hit a custom host
+          #bw config server ''${BW_HOST}
+
+          echo "Using apikey to log in"
+          bw login --apikey --raw
+          BW_SESSION="$(bw unlock --passwordenv BW_PASSWORD --raw)"
+          export BW_SESSION
+
+          echo 'Running "bw serve" on port 8087'
+          bw serve --hostname all --port 8087
+        '';
+      }))
+    ];
+    Env = [
+      "CURL_CA_BUNDLE=${cacert}/etc/ssl/certs/ca-bundle.crt"
+    ];
+    ExposedPorts = {
+      "8087/tcp" = { };
+    };
+  };
+}

secrets/secrets.nix

@@ -104,6 +104,7 @@ in
   "minio_secret_access_key.age".publicKeys = everyone;
   "minio_access_key_id.age".publicKeys = everyone;
 
+  "kubernetes/bw_secret.age".publicKeys = everyone;
   "kubernetes/kubernetesToken.age".publicKeys = [
     isaiah
     jeremiah