From ce5f60463621cb97f021188dcf3f0fb90051b1ac Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Mon, 26 May 2025 17:10:01 -0500 Subject: [PATCH] External Secrets setup Setup the external secrets helm charts Convert hard coded secrets to external secrets Add BitWarden CLI for serving external secrets --- .gitignore | 1 + .gitlab-ci.yml | 13 ++++ manifests/README.md | 18 +++++ manifests/apply.sh | 5 ++ manifests/bitwarden/cluster-stores.yaml | 45 +++++++++++ manifests/bitwarden/deployment.yaml | 70 ++++++++++++++++++ manifests/bitwarden/kustomization.yaml | 6 ++ manifests/bitwarden/service.yaml | 18 +++++ manifests/databases/postgres-gitlab.yaml | 12 --- manifests/databases/postgres-pgadmin.yaml | 11 --- manifests/external-secrets/kustomization.yaml | 13 ++++ manifests/kustomization.yaml | 2 + manifests/namespaces/bitwarden.yaml | 4 + manifests/namespaces/external-secrets.yaml | 4 + manifests/namespaces/kustomization.yaml | 2 + manifests/secrets/kustomization.yaml | 3 + manifests/secrets/postgres-user-gitlab.yaml | 33 +++++++++ manifests/secrets/postgres-user-pgadmin.yaml | 33 +++++++++ modules/nixos/kubernetes.nix | 5 +- pkgs/default.nix | 1 + pkgs/img-bitwarden.nix | 44 +++++++++++ secrets/kubernetes/bw_secret.age | Bin 0 -> 2365 bytes secrets/secrets.nix | 1 + 23 files changed, 320 insertions(+), 24 deletions(-) create mode 100644 manifests/bitwarden/cluster-stores.yaml create mode 100644 manifests/bitwarden/deployment.yaml create mode 100644 manifests/bitwarden/kustomization.yaml create mode 100644 manifests/bitwarden/service.yaml create mode 100644 manifests/external-secrets/kustomization.yaml create mode 100644 manifests/namespaces/bitwarden.yaml create mode 100644 manifests/namespaces/external-secrets.yaml create mode 100644 manifests/secrets/kustomization.yaml create mode 100644 manifests/secrets/postgres-user-gitlab.yaml create mode 100644 manifests/secrets/postgres-user-pgadmin.yaml create mode 100644 pkgs/img-bitwarden.nix create mode 100644 secrets/kubernetes/bw_secret.age diff --git a/.gitignore b/.gitignore index fe79c61..12a76b9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ result # where I am storing my repo nix.conf manifests/postgres/charts +manifests/external-secrets/charts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3b6eae..3d9e09b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,3 +21,16 @@ default: job: "Evaluate for builds" variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID + +"Build image": + stage: build + parallel: + matrix: + - IMG: + - img-bitwarden + script: + - podman login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + - nix build ".#${IMG}" + - podman load -i result + - podman tag "localhost/${IMG}:latest" "$CI_REGISTRY/greg/ci-images/${IMG}:latest" + - podman push "$CI_REGISTRY/greg/ci-images/${IMG}:latest" diff --git a/manifests/README.md b/manifests/README.md index 4f086a2..e0aafec 100644 --- a/manifests/README.md +++ b/manifests/README.md @@ -1,5 +1,23 @@ Stands up my personal infrastructure in a Kubernetes environment. +# First Thing First + +In order to properly get things up and going, you will need to create a secret to allow +the external secrets to log into BitWarden Password Manager. For obvious reasont this +cannot be safely added to this repository. As such, it is suggested you create this +manually. + +```bash +kubectl create namespace bitwarden +kubectl create secret generic bitwarden-cli --namespace bitwarden --from-literal=BW_USERNAME=my_username --from-literal=BW_PASSWORD=my_password +# Alternative to the preceding line if you don't want the data in your shell history +kubectl create secret generic bitwarden-cli --namespace bitwarden --from-file=./BW_USERNAME.txt --from-file=./BW_PASSWORD.txt +# And yet an entirely other option, if I'm logged into my own systems +sudo cat /run/agenix/bw_secret | kubectl apply -f - +``` + +# Now Configure Cluster Services + To apply this you need to install kubectl, kustomize, and helm. It can then by applied by simply invoking the command: diff --git a/manifests/apply.sh b/manifests/apply.sh index 546b1e9..1419a56 100755 --- a/manifests/apply.sh +++ b/manifests/apply.sh @@ -7,5 +7,10 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" cd "$SCRIPT_DIR" kubectl apply -k namespaces +kustomize build external-secrets --enable-helm | kubectl apply -f - +echo -e "\nPausing for a moment to allow the External Secrets stuff to get going\n" +sleep 5 kustomize build postgres --enable-helm | kubectl apply -f - --server-side --force-conflicts +echo -e "\nPausing for a moment to allow Postgres stuff to get going\n" +sleep 5 kubectl apply -k . diff --git a/manifests/bitwarden/cluster-stores.yaml b/manifests/bitwarden/cluster-stores.yaml new file mode 100644 index 0000000..32e1a93 --- /dev/null +++ b/manifests/bitwarden/cluster-stores.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: bitwarden-login +spec: + provider: + webhook: + url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/item/{{ .remoteRef.key }}" + headers: + Content-Type: application/json + result: + jsonPath: "$.data.login.{{ .remoteRef.property }}" +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: bitwarden-fields +spec: + provider: + webhook: + url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/item/{{ .remoteRef.key }}" + result: + jsonPath: '$.data.fields[?@.name=="{{ .remoteRef.property }}"].value' +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: bitwarden-notes +spec: + provider: + webhook: + url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/item/{{ .remoteRef.key }}" + result: + jsonPath: "$.data.notes" +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: bitwarden-attachments +spec: + provider: + webhook: + url: "http://bitwarden-cli.bitwarden.svc.cluster.local:8087/object/attachment/{{ .remoteRef.property }}?itemid={{ .remoteRef.key }}" + result: {} diff --git a/manifests/bitwarden/deployment.yaml b/manifests/bitwarden/deployment.yaml new file mode 100644 index 0000000..5c48996 --- /dev/null +++ b/manifests/bitwarden/deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bitwarden-cli + labels: + app.kubernetes.io/instance: bitwarden-cli + app.kubernetes.io/name: bitwarden-cli +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: bitwarden-cli + app.kubernetes.io/instance: bitwarden-cli + template: + metadata: + labels: + app.kubernetes.io/name: bitwarden-cli + app.kubernetes.io/instance: bitwarden-cli + spec: + containers: + - name: bitwarden-cli + image: "registry.thehellings.com/greg/nixos-config/img-bitwarden:latest" + imagePullPolicy: Always + env: + - name: BW_CLIENTID + valueFrom: + secretKeyRef: + name: bitwarden-cli + key: BW_CLIENTID + - name: BW_CLIENTSECRET + valueFrom: + secretKeyRef: + name: bitwarden-cli + key: BW_CLIENTSECRET + - name: BW_PASSWORD + valueFrom: + secretKeyRef: + name: bitwarden-cli + key: BW_PASSWORD + ports: + - name: http + containerPort: 8087 + protocol: TCP + livenessProbe: + exec: + command: + - wget + - -q + - http://127.0.0.1:8087/sync?force=true + - --post-data='' + initialDelaySeconds: 20 + failureThreshold: 3 + timeoutSeconds: 10 + periodSeconds: 120 + readinessProbe: + tcpSocket: + port: 8087 + initialDelaySeconds: 20 + failureThreshold: 3 + timeoutSeconds: 1 + periodSeconds: 10 + startupProbe: + tcpSocket: + port: 8087 + initialDelaySeconds: 10 + failureThreshold: 30 + timeoutSeconds: 1 + periodSeconds: 5 diff --git a/manifests/bitwarden/kustomization.yaml b/manifests/bitwarden/kustomization.yaml new file mode 100644 index 0000000..6154c52 --- /dev/null +++ b/manifests/bitwarden/kustomization.yaml @@ -0,0 +1,6 @@ +namespace: bitwarden + +resources: + - deployment.yaml + - service.yaml + - cluster-stores.yaml diff --git a/manifests/bitwarden/service.yaml b/manifests/bitwarden/service.yaml new file mode 100644 index 0000000..ff3fd87 --- /dev/null +++ b/manifests/bitwarden/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: bitwarden-cli + labels: + app.kubernetes.io/instance: bitwarden-cli + app.kubernetes.io/name: bitwarden-cli + annotations: +spec: + type: ClusterIP + ports: + - port: 8087 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: bitwarden-cli + app.kubernetes.io/instance: bitwarden-cli diff --git a/manifests/databases/postgres-gitlab.yaml b/manifests/databases/postgres-gitlab.yaml index 6490dff..f41796f 100644 --- a/manifests/databases/postgres-gitlab.yaml +++ b/manifests/databases/postgres-gitlab.yaml @@ -1,15 +1,3 @@ -apiVersion: v1 -kind: Secret -metadata: - name: postgres-user-gitlab - labels: - cnpg.io/reload: "true" -type: kubernetes.io/basic-auth -stringData: - # Currently this is set to "gitlab" - username: gitlab - password: gitlab ---- apiVersion: postgresql.cnpg.io/v1 kind: Database metadata: diff --git a/manifests/databases/postgres-pgadmin.yaml b/manifests/databases/postgres-pgadmin.yaml index 3ba206a..ab82d78 100644 --- a/manifests/databases/postgres-pgadmin.yaml +++ b/manifests/databases/postgres-pgadmin.yaml @@ -1,15 +1,4 @@ apiVersion: v1 -kind: Secret -metadata: - name: postgres-user-pgadmin - labels: - cnpg.io/reload: "true" -type: kubernetes.io/basic-auth -stringData: - username: pgadmin - password: pgadmin ---- -apiVersion: v1 kind: ConfigMap metadata: name: config-pgadmin diff --git a/manifests/external-secrets/kustomization.yaml b/manifests/external-secrets/kustomization.yaml new file mode 100644 index 0000000..917c23a --- /dev/null +++ b/manifests/external-secrets/kustomization.yaml @@ -0,0 +1,13 @@ +namespace: external-secrets + +helmCharts: + # Install the operator first (with CRDs) + - name: external-secrets + repo: "https://charts.external-secrets.io/" + version: "0.17.0" + namespace: external-secrets + releaseName: external-secrets + includeCRDs: true + valuesInline: + crds: + create: true diff --git a/manifests/kustomization.yaml b/manifests/kustomization.yaml index cf57688..0f54c5c 100644 --- a/manifests/kustomization.yaml +++ b/manifests/kustomization.yaml @@ -1,3 +1,5 @@ resources: - namespaces + - bitwarden + - secrets - databases diff --git a/manifests/namespaces/bitwarden.yaml b/manifests/namespaces/bitwarden.yaml new file mode 100644 index 0000000..04cf575 --- /dev/null +++ b/manifests/namespaces/bitwarden.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bitwarden diff --git a/manifests/namespaces/external-secrets.yaml b/manifests/namespaces/external-secrets.yaml new file mode 100644 index 0000000..4ef398e --- /dev/null +++ b/manifests/namespaces/external-secrets.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-secrets diff --git a/manifests/namespaces/kustomization.yaml b/manifests/namespaces/kustomization.yaml index 46ae31c..4a33a16 100644 --- a/manifests/namespaces/kustomization.yaml +++ b/manifests/namespaces/kustomization.yaml @@ -1,2 +1,4 @@ resources: + - external-secrets.yaml + - bitwarden.yaml - db.yaml diff --git a/manifests/secrets/kustomization.yaml b/manifests/secrets/kustomization.yaml new file mode 100644 index 0000000..84079cc --- /dev/null +++ b/manifests/secrets/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - postgres-user-gitlab.yaml + - postgres-user-pgadmin.yaml diff --git a/manifests/secrets/postgres-user-gitlab.yaml b/manifests/secrets/postgres-user-gitlab.yaml new file mode 100644 index 0000000..fde4202 --- /dev/null +++ b/manifests/secrets/postgres-user-gitlab.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgres-user-gitlab + namespace: db +spec: + target: + name: postgres-user-gitlab + deletionPolicy: Delete + template: + type: Opaque + data: + username: |- + {{ .username }} + password: |- + {{ .password }} + data: + - secretKey: username + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: 5282ad16-c2dc-49d3-8fb3-b2e9012bab57 + property: username + - secretKey: password + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: 5282ad16-c2dc-49d3-8fb3-b2e9012bab57 + property: password diff --git a/manifests/secrets/postgres-user-pgadmin.yaml b/manifests/secrets/postgres-user-pgadmin.yaml new file mode 100644 index 0000000..0f2500a --- /dev/null +++ b/manifests/secrets/postgres-user-pgadmin.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgres-user-pgadmin + namespace: db +spec: + target: + name: postgres-user-pgadmin + deletionPolicy: Delete + template: + type: Opaque + data: + username: |- + {{ .username }} + password: |- + {{ .password }} + data: + - secretKey: username + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: f333d637-1667-499d-b9a0-b2e9012bd8b7 + property: username + - secretKey: password + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: f333d637-1667-499d-b9a0-b2e9012bd8b7 + property: password diff --git a/modules/nixos/kubernetes.nix b/modules/nixos/kubernetes.nix index a10c08a..15ac302 100644 --- a/modules/nixos/kubernetes.nix +++ b/modules/nixos/kubernetes.nix @@ -20,7 +20,10 @@ in }; config = lib.mkIf cfg.enable { - age.secrets.kubernetesToken.file = ../../secrets/kubernetes/kubernetesToken.age; + age.secrets = { + kubernetesToken.file = ../../secrets/kubernetes/kubernetesToken.age; + bw_secret.file = ../../secrets/kubernetes/bw_secret.age; + }; environment.systemPackages = [ pkgs.kubectl-cnpg diff --git a/pkgs/default.nix b/pkgs/default.nix index 2e2bf19..75f2738 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -14,6 +14,7 @@ in hms = c ./hms { }; inject-darwin = c ./inject-darwin.nix { }; inject = c ./inject.nix { }; + img-bitwarden = c ./img-bitwarden.nix { }; qemu-hook = c ./qemu-hook.nix { }; setup-ssh = c ./setup-ssh { }; upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { }; diff --git a/pkgs/img-bitwarden.nix b/pkgs/img-bitwarden.nix new file mode 100644 index 0000000..d57e7d6 --- /dev/null +++ b/pkgs/img-bitwarden.nix @@ -0,0 +1,44 @@ +{ + bitwarden-cli, + cacert, + dockerTools, + lib, + writeShellApplication, + ... +}: +dockerTools.buildLayeredImage { + name = "img-bitwarden"; + tag = "latest"; + contents = [ + dockerTools.binSh + dockerTools.caCertificates + ]; + config = { + Cmd = [ + (lib.getExe (writeShellApplication { + name = "bitwarden-cli-entrypoint.sh"; + runtimeInputs = [ bitwarden-cli ]; + text = '' + set -ex + + # Uncomment if you need to hit a custom host + #bw config server ''${BW_HOST} + + echo "Using apikey to log in" + bw login --apikey --raw + BW_SESSION="$(bw unlock --passwordenv BW_PASSWORD --raw)" + export BW_SESSION + + echo 'Running "bw serve" on port 8087' + bw serve --hostname all --port 8087 + ''; + })) + ]; + Env = [ + "CURL_CA_BUNDLE=${cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + ExposedPorts = { + "8087/tcp" = { }; + }; + }; +} diff --git a/secrets/kubernetes/bw_secret.age b/secrets/kubernetes/bw_secret.age new file mode 100644 index 0000000000000000000000000000000000000000..bfc8982de181d1e81ba93fefdd56bf1441ecca63 GIT binary patch literal 2365 zcmZY9>F?wO8NhKtgK_X!h{3Re8KW4a`O#zAX^Y6(W7_F8op#zzXB3lO({`rQnNIIw zRH7h66Eqv6xWofi6Y(GcP0$w@5iz0(N-)t7)-wbpM0M@&*Ox_ zQIL9vc3;H1^yb|m1S`0AbXz~qqocq_FbpR9L$q|(b$?PybOf6?86)gGwnuV9iqJ|9 z*xa3M9BHnOh8@oNdYEcGMlL``;A@_j=8H)@sR@1+9hMj+Wf9#>MO#?!T!|ZE6m?wN zwWF<3_vN5P46k7f%K;|qEQ4DyBxXWBEO8}+lr+(FeG;3}JP~?T=}x>Q9V-=WEZba~ zyQ&M1`K3t#Vzyk=0s%C<5qN?m9M`t$cDf4tGmH+fcxG+zd8rwChpHDi%~~2Q0945YTx&=26m$2~#mv8XY+(BqPEK+Di^}Tw92-?$&*Z z#YJYe-`SF8$5kq!plV3M7%WL{5<3~X5HoDe<~S89J$R&&MO*;IkS-?mMkMM1m=7|r zBJwV`>x4)a2FM#uBJ7T9&H|xqs_*b(WjZ?K1J1^4QG{9R<8(XO!G5`%gAro&q9G2GIL7Nh5wxUsZffbcyS&MEPbnqD8X6A9NnH1@c>L13b0m4Nf zrSW0!+Zk)m!Zife^+*!R#Fw^+G2>Bf33X`XU%xGO#B(h@A;b9u9Jo>pgL>n zeYsGmN#ip`1NDQd0e4s?E@q$F_0o`F*B!%x_7@>tCG1)Cy`QnjfQ6a4&Yi>^x$eVF= z8(-2!nCTqT`V3Tv;nb!a)picSqT(%%FxLxqho^K>)2q&dbkS;cCGmO$(KT#iwh0)o zN9jQyuOuUgH60%ytTEgus>bZrG_x-l<#-ze&T}d3@7TatlEESHT~|;9m?0%m3Xr8a zM}-zBt&(UDk)GjQYW6b8^!0X^d-*J;^rFfzP{Hak!|=+$z^P70M%*x`#}$7`+av+8 zmg-N38?X`QQ*kDu3u7{9!35B>k-Z7`sXC)_4}zunpaBRD>an^BB(}a#@O(a8Y*qAt z4Z5*u`BXcW^C`Pc+?3$0YCo%wxADFlGFo2;*EA%PiM$A%GL$8Ct%z1fHG?_KSz04{ zgauIZ(P7W;dLGhFmA#5KzMcpL#s$L=E0e)wJv9jmhMW{g@_o$MGTz0{KQq#!rtbZkJqtV3sXlp+XO(y>EL^p1ijE)dON znGFCmg7XTUb?qv@IQ3&+ncH{OO6$|GXHN%`c5yDUXG#-f;MN(F$0-wIgqO5Dw6{G$ znEM;0H+98mXc9XR*&kvy2oz%=llx_;q(MxvOS7UmUQeOIB=h6i#-CQ*#WtHz#$&D0 zI397Kv`T%&v?&GKjxO#)#@h7k6{e!FYx_LMB?>M~r<69rlpJ56RiM*B zqwtt-!NI7wWG8p&*4~HxL>g8ShpDn0Ete2vmTPD^bpUw{>#pOMWYw`~z$zUwM|+7w zlUTOF8VlN%>5f^DChe{b%t(VJtQy|71c%|pliI0^<84eDu~69GVB^D5D;1>cQX8;V zX==OH$ZblO2eC#mQZM3(Mg{c{?UigOv9-AcW>Jj~E0bNe)pnR{2G~kdnGv$G1}=xO z^mKkn8yf5pljY`;lW4%O{Bu_`9t%WWolPKj}X45-5I_gU;CLX3*1~Jhx18Pq8HKYq zHe3|=Yz+kIIFTf?cW?du;?rL~`HN@U)9f4bqOuk1c?{_MA| z0$=*nCts|tyB@Y)x#!NS=k=@Ky+bx$KX?DBTiO#R-l05H6WQIRxcd3|C$D_;b1$Cx z$WKARj(VrBe>pjP{kp08;JfkZca0{Iq`>YQn#HVMpGKU=>hp8fEJ#~!i8f7}b- z{@0H^{M-YQeEFXr@4fu!eZT(DsTZm^x%GR`{p;-AzkS?%%On51@YAli@@#F8o&-{IxCf*NM?|Sv_%l`fJWqCo|4n6^+b{{a^x94`O> literal 0 HcmV?d00001 diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 7c41616..2d8392a 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -104,6 +104,7 @@ in "minio_secret_access_key.age".publicKeys = everyone; "minio_access_key_id.age".publicKeys = everyone; + "kubernetes/bw_secret.age".publicKeys = everyone; "kubernetes/kubernetesToken.age".publicKeys = [ isaiah jeremiah