Merge branch 'main' of https://git.procopius.dk/plasmagoat/homelab

hive.nix

@@ -45,4 +45,9 @@ inputs @ {
     imports = [./machines/${name}/definition.nix];
     deployment.tags = ["grafana" "prometheus"];
   };
+
+  auth = {name, ...}: {
+    imports = [./machines/${name}/definition.nix];
+    deployment.tags = ["zitadel" "sso" "ldap"];
+  };
 }

machines/_default/default.nix

@@ -18,7 +18,7 @@
 
   sops = {
     age.keyFile = "/etc/sops/age.key";
-    defaultSopsFile = ../../secrets/secrets.yml;
+    defaultSopsFile = ../../secrets/secrets.yaml;
   };
 
   # home-manager = {

machines/auth/authelia.nix

@@ -0,0 +1,151 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  authelia = "authelia-procopius";
+in {
+  networking.firewall.allowedTCPPorts = [
+    9091
+  ];
+
+  services = {
+    authelia.instances.procopius = {
+      enable = true;
+      settings = {
+        theme = "auto";
+        authentication_backend.ldap = {
+          address = "ldap://localhost:3890";
+          base_dn = "dc=procopius,dc=dk";
+          users_filter = "(&({username_attribute}={input})(objectClass=person))";
+          groups_filter = "(member={dn})";
+          user = "uid=authelia,ou=people,dc=procopius,dc=dk";
+        };
+        access_control = {
+          default_policy = "deny";
+          # We want this rule to be low priority so it doesn't override the others
+          rules = lib.mkAfter [
+            {
+              domain = "*.procopius.dk";
+              policy = "one_factor";
+            }
+          ];
+        };
+        storage.postgres = {
+          address = "unix:///run/postgresql";
+          database = authelia;
+          username = authelia;
+          # I'm using peer authentication, so this doesn't actually matter, but Authelia
+          # complains if I don't have it.
+          # https://github.com/authelia/authelia/discussions/7646
+          password = authelia;
+        };
+        session = {
+          redis.host = "/var/run/redis-procopius/redis.sock";
+          cookies = [
+            {
+              domain = "procopius.dk";
+              authelia_url = "https://authelia.procopius.dk";
+              # The period of time the user can be inactive for before the session is destroyed
+              inactivity = "1M";
+              # The period of time before the cookie expires and the session is destroyed
+              expiration = "3M";
+              # The period of time before the cookie expires and the session is destroyed
+              # when the remember me box is checked
+              remember_me = "1y";
+            }
+          ];
+        };
+        notifier.smtp = {
+          address = "smtp://mail.procopius.dk";
+          username = "admin@procopius.dk";
+          sender = "auth@procopius.dk";
+        };
+        log.level = "info";
+        # identity_providers.oidc = {
+        #   # https://www.authelia.com/integration/openid-connect/openid-connect-1.0-claims/#restore-functionality-prior-to-claims-parameter
+        #   claims_policies = {
+        #     # karakeep.id_token = ["email"];
+        #   };
+        #   cors = {
+        #     endpoints = ["token"];
+        #     allowed_origins_from_client_redirect_uris = true;
+        #   };
+        #   authorization_policies.default = {
+        #     default_policy = "one_factor";
+        #     rules = [
+        #       {
+        #         policy = "deny";
+        #         subject = "group:lldap_strict_readonly";
+        #       }
+        #     ];
+        #   };
+        # };
+        # Necessary for Traefik integration
+        # See https://www.authelia.com/integration/proxies/traefik/#implementation
+        server.endpoints.authz.forward-auth.implementation = "ForwardAuth";
+      };
+      # Templates don't work correctly when parsed from Nix, so our OIDC clients are defined here
+      # settingsFiles = [./oidc_clients.yaml];
+      secrets = with config.sops; {
+        jwtSecretFile = secrets."authelia/jwt_secret".path;
+        # oidcIssuerPrivateKeyFile = secrets."authelia/jwks".path;
+        # oidcHmacSecretFile = secrets."authelia/hmac_secret".path;
+        sessionSecretFile = secrets."authelia/session_secret".path;
+        storageEncryptionKeyFile = secrets."authelia/storage_encryption_key".path;
+      };
+      environmentVariables = with config.sops; {
+        AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD_FILE =
+          secrets."authelia/lldap_authelia_password".path;
+        AUTHELIA_NOTIFIER_SMTP_PASSWORD_FILE = secrets."authelia/smtp_authelia_password".path;
+      };
+    };
+    # caddy = {
+    #   virtualHosts."auth.procopius.cc".extraConfig = ''
+    #     reverse_proxy :9091
+    #   '';
+    #   # A Caddy snippet that can be imported to enable Authelia in front of a service
+    #   # Taken from https://www.authelia.com/integration/proxies/caddy/#subdomain
+    #   extraConfig = ''
+    #     (auth) {
+    #         forward_auth :9091 {
+    #             uri /api/authz/forward-auth
+    #             copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
+    #         }
+    #     }
+    #   '';
+    # };
+  };
+
+  # Give Authelia access to the Redis socket
+  users.users.${authelia}.extraGroups = ["redis-procopius"];
+
+  systemd.services.${authelia} = let
+    dependencies = [
+      "lldap.service"
+      "postgresql.service"
+      "redis-procopius.service"
+    ];
+  in {
+    # Authelia requires LLDAP, PostgreSQL, and Redis to be running
+    after = dependencies;
+    requires = dependencies;
+    # Required for templating
+    serviceConfig.Environment = "X_AUTHELIA_CONFIG_FILTERS=template";
+  };
+
+  sops.secrets = {
+    "authelia/hmac_secret".owner = authelia;
+    "authelia/jwks".owner = authelia;
+    "authelia/jwt_secret".owner = authelia;
+    "authelia/session_secret".owner = authelia;
+    "authelia/storage_encryption_key".owner = authelia;
+    # The password for the `authelia` LLDAP user
+    "authelia/lldap_authelia_password".owner = authelia;
+    "authelia/smtp_authelia_password".owner = authelia;
+    smtp-password_authelia = {
+      owner = authelia;
+      key = "service_accounts/authelia/password";
+    };
+  };
+}

machines/auth/bootstrap/bootstrap.sh

@@ -0,0 +1,762 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+echo "$LLDAP_SET_PASSWORD_PATH"
+
+LLDAP_ADMIN_PASSWORD="$(cat $LLDAP_ADMIN_PASSWORD_FILE)"
+
+LLDAP_URL="${LLDAP_URL:-http://localhost:17170}"
+LLDAP_ADMIN_USERNAME="${LLDAP_ADMIN_USERNAME:-admin}"
+LLDAP_ADMIN_PASSWORD="${LLDAP_ADMIN_PASSWORD:-password}"
+USER_SCHEMAS_DIR="${USER_SCHEMAS_DIR:-/bootstrap/user-schemas}"
+GROUP_SCHEMAS_DIR="${GROUP_SCHEMAS_DIR:-/bootstrap/group-schemas}"
+USER_CONFIGS_DIR="${USER_CONFIGS_DIR:-/bootstrap/user-configs}"
+GROUP_CONFIGS_DIR="${GROUP_CONFIGS_DIR:-/bootstrap/group-configs}"
+LLDAP_SET_PASSWORD_PATH="${LLDAP_SET_PASSWORD_PATH:-/app/lldap_set_password}"
+DO_CLEANUP="${DO_CLEANUP:-false}"
+
+# Fallback to support legacy defaults
+if [[ ! -d $USER_CONFIGS_DIR ]] && [[ -d "/user-configs" ]]; then
+  USER_CONFIGS_DIR="/user-configs"
+fi
+if [[ ! -d $GROUP_CONFIGS_DIR ]] && [[ -d "/group-configs" ]]; then
+  GROUP_CONFIGS_DIR="/group-configs"
+fi
+
+check_install_dependencies() {
+  local commands=('curl' 'jq' 'jo')
+  local commands_not_found='false'
+
+  if ! hash "${commands[@]}" 2>/dev/null; then
+    if hash 'apk' 2>/dev/null && [[ $EUID -eq 0 ]]; then
+      apk add "${commands[@]}"
+    elif hash 'apt' 2>/dev/null && [[ $EUID -eq 0 ]]; then
+      apt update -yqq
+      apt install -yqq "${commands[@]}"
+    else
+      local command=''
+      for command in "${commands[@]}"; do
+        if ! hash "$command" 2>/dev/null; then
+          printf 'Command not found "%s"\n' "$command"
+        fi
+      done
+      commands_not_found='true'
+    fi
+  fi
+
+  if [[ "$commands_not_found" == 'true' ]]; then
+    return 1
+  fi
+}
+
+check_required_env_vars() {
+  local env_var_not_specified='false'
+  local dual_env_vars_list=(
+    'LLDAP_URL'
+    'LLDAP_ADMIN_USERNAME'
+    'LLDAP_ADMIN_PASSWORD'
+  )
+
+  local dual_env_var_name=''
+  for dual_env_var_name in "${dual_env_vars_list[@]}"; do
+    local dual_env_var_file_name="${dual_env_var_name}_FILE"
+
+    if [[ -z "${!dual_env_var_name}" ]] && [[ -z "${!dual_env_var_file_name}" ]]; then
+      printf 'Please specify "%s" or "%s" variable!\n' "$dual_env_var_name" "$dual_env_var_file_name" >&2
+      env_var_not_specified='true'
+    else
+      if [[ -n "${!dual_env_var_file_name}" ]]; then
+        declare -g "$dual_env_var_name"="$(cat "${!dual_env_var_file_name}")"
+      fi
+    fi
+  done
+
+  if [[ "$env_var_not_specified" == 'true' ]]; then
+    return 1
+  fi
+}
+
+check_configs_validity() {
+  local config_file='' config_invalid='false'
+  for config_file in "$@"; do
+    local error=''
+    if ! error="$(jq '.' -- "$config_file" 2>&1 >/dev/null)"; then
+      printf '%s: %s\n' "$config_file" "$error"
+      config_invalid='true'
+    fi
+  done
+
+  if [[ "$config_invalid" == 'true' ]]; then
+    return 1
+  fi
+}
+
+auth() {
+  local url="$1" admin_username="$2" admin_password="$3"
+
+  local response
+  response="$(curl --silent --request POST \
+    --url "$url/auth/simple/login" \
+    --header 'Content-Type: application/json' \
+    --data "$(jo -- username="$admin_username" password="$admin_password")")"
+
+  TOKEN="$(printf '%s' "$response" | jq --raw-output .token)"
+}
+
+make_query() {
+  local query_file="$1" variables_file="$2"
+
+  curl --silent --request POST \
+    --url "$LLDAP_URL/api/graphql" \
+    --header "Authorization: Bearer $TOKEN" \
+    --header 'Content-Type: application/json' \
+    --data @<(jq --slurpfile variables "$variables_file" '. + {"variables": $variables[0]}' "$query_file")
+}
+
+get_group_list() {
+  local query='{"query":"query GetGroupList {groups {id displayName}}","operationName":"GetGroupList"}'
+  make_query <(printf '%s' "$query") <(printf '{}')
+}
+
+get_group_array() {
+  get_group_list | jq --raw-output '.data.groups[].displayName'
+}
+
+group_exists() {
+  if [[ "$(get_group_list | jq --raw-output --arg displayName "$1" '.data.groups | any(.[]; select(.displayName == $displayName))')" == 'true' ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+get_group_id() {
+  get_group_list | jq --raw-output --arg displayName "$1" '.data.groups[] | if .displayName == $displayName then .id else empty end'
+}
+
+create_group() {
+  local group_name="$1"
+
+  if group_exists "$group_name"; then
+    printf 'Group "%s" (%s) already exists\n' "$group_name" "$(get_group_id "$group_name")"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation CreateGroup($name: String!) {createGroup(name: $name) {id displayName}}","operationName":"CreateGroup"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- name="$group_name"))"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'Group "%s" (%s) successfully created\n' "$group_name" "$(printf '%s' "$response" | jq --raw-output '.data.createGroup.id')"
+  fi
+}
+
+delete_group() {
+  local group_name="$1" id=''
+
+  if ! group_exists "$group_name"; then
+    printf '[WARNING] Group "%s" does not exist\n' "$group_name"
+    return
+  fi
+
+  id="$(get_group_id "$group_name")"
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation DeleteGroupQuery($groupId: Int!) {deleteGroup(groupId: $groupId) {ok}}","operationName":"DeleteGroupQuery"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- groupId="$id"))"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'Group "%s" (%s) successfully deleted\n' "$group_name" "$id"
+  fi
+}
+
+get_user_details() {
+  local id="$1"
+
+  # shellcheck disable=SC2016
+  local query='{"query":"query GetUserDetails($id: String!) {user(userId: $id) {id email displayName firstName lastName creationDate uuid groups {id displayName} attributes {name value}}}","operationName":"GetUserDetails"}'
+  make_query <(printf '%s' "$query") <(jo -- id="$id")
+}
+
+user_in_group() {
+  local user_id="$1" group_name="$2"
+
+  if ! group_exists "$group_name"; then
+    printf '[WARNING] Group "%s" does not exist\n' "$group_name"
+    return
+  fi
+
+  if ! user_exists "$user_id"; then
+    printf 'User "%s" is not exists\n' "$user_id"
+    return
+  fi
+
+  if [[ "$(get_user_details "$user_id" | jq --raw-output --arg displayName "$group_name" '.data.user.groups | any(.[]; select(.displayName == $displayName))')" == 'true' ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+add_user_to_group() {
+  local user_id="$1" group_name="$2" group_id=''
+
+  if ! group_exists "$group_name"; then
+    printf '[WARNING] Group "%s" does not exist\n' "$group_name"
+    return
+  fi
+
+  group_id="$(get_group_id "$group_name")"
+
+  if user_in_group "$user_id" "$group_name"; then
+    printf 'User "%s" already in group "%s" (%s)\n' "$user_id" "$group_name" "$group_id"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation AddUserToGroup($user: String!, $group: Int!) {addUserToGroup(userId: $user, groupId: $group) {ok}}","operationName":"AddUserToGroup"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- user="$user_id" group="$group_id"))"
+  error="$(printf '%s' "$response" | jq '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'User "%s" successfully added to the group "%s" (%s)\n' "$user_id" "$group_name" "$group_id"
+  fi
+}
+
+remove_user_from_group() {
+  local user_id="$1" group_name="$2" group_id=''
+
+  if ! group_exists "$group_name"; then
+    printf '[WARNING] Group "%s" does not exist\n' "$group_name"
+    return
+  fi
+
+  group_id="$(get_group_id "$group_name")"
+
+  # shellcheck disable=SC2016
+  local query='{"operationName":"RemoveUserFromGroup","query":"mutation RemoveUserFromGroup($user: String!, $group: Int!) {removeUserFromGroup(userId: $user, groupId: $group) {ok}}"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- user="$user_id" group="$group_id"))"
+  error="$(printf '%s' "$response" | jq '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'User "%s" successfully removed from the group "%s" (%s)\n' "$user_id" "$group_name" "$group_id"
+  fi
+}
+
+get_users_list() {
+  # shellcheck disable=SC2016
+  local query='{"query": "query ListUsersQuery($filters: RequestFilter) {users(filters: $filters) {id email displayName firstName lastName creationDate}}","operationName": "ListUsersQuery"}'
+  make_query <(printf '%s' "$query") <(jo -- filters=null)
+}
+
+user_exists() {
+  if [[ "$(get_users_list | jq --raw-output --arg id "$1" '.data.users | any(.[]; .id == $id)')" == 'true' ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+delete_user() {
+  local id="$1"
+
+  if ! user_exists "$id"; then
+    printf 'User "%s" is not exists\n' "$id"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query": "mutation DeleteUserQuery($user: String!) {deleteUser(userId: $user) {ok}}","operationName": "DeleteUserQuery"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- user="$id"))"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'User "%s" successfully deleted\n' "$id"
+  fi
+}
+
+get_group_property_list() {
+  local query='{"query":"query GetGroupAttributesSchema { schema { groupSchema { attributes { name }}}}","operationName":"GetGroupAttributesSchema"}'
+  make_query <(printf '%s' "$query") <(printf '{}')
+}
+group_property_exists() {
+  if [[ "$(get_group_property_list | jq --raw-output --arg name "$1" '.data.schema.groupSchema.attributes | any(.[]; select(.name == $name))')" == 'true' ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+create_group_schema_property() {
+  local name="$1"
+  local attributeType="$2"
+  local isEditable="$3"
+  local isList="$4"
+  local isVisible="$5"
+
+  if group_property_exists "$name"; then
+    printf 'Group property "%s" already exists\n' "$name"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation CreateGroupAttribute($name: String!, $attributeType: AttributeType!, $isList: Boolean!, $isVisible: Boolean!, $isEditable: Boolean!) {addGroupAttribute(name: $name, attributeType: $attributeType, isList: $isList, isVisible: $isVisible, isEditable: $isEditable) {ok}}","operationName":"CreateGroupAttribute"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- name="$name" attributeType="$attributeType" isEditable="$isEditable" isList="$isList" isVisible="$isVisible"))"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'Group attribute "%s" successfully created\n' "$name"
+  fi
+}
+
+get_user_property_list() {
+  local query='{"query":"query GetUserAttributesSchema { schema { userSchema { attributes { name }}}}","operationName":"GetUserAttributesSchema"}'
+  make_query <(printf '%s' "$query") <(printf '{}')
+}
+user_property_exists() {
+  if [[ "$(get_user_property_list | jq --raw-output --arg name "$1" '.data.schema.userSchema.attributes | any(.[]; select(.name == $name))')" == 'true' ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+create_user_schema_property() {
+  local name="$1"
+  local attributeType="$2"
+  local isEditable="$3"
+  local isList="$4"
+  local isVisible="$5"
+
+  if user_property_exists "$name"; then
+    printf 'User property "%s" already exists\n' "$name"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation CreateUserAttribute($name: String!, $attributeType: AttributeType!, $isList: Boolean!, $isVisible: Boolean!, $isEditable: Boolean!) {addUserAttribute(name: $name, attributeType: $attributeType, isList: $isList, isVisible: $isVisible, isEditable: $isEditable) {ok}}","operationName":"CreateUserAttribute"}'
+
+  local response='' error=''
+  response="$(make_query <(printf '%s' "$query") <(jo -- name="$name" attributeType="$attributeType" isEditable="$isEditable" isList="$isList" isVisible="$isVisible"))"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'User attribute "%s" successfully created\n' "$name"
+  fi
+}
+
+update_group_attributes() {
+  local group_id="$1"
+  local attributes_json="$2"
+
+  # shellcheck disable=SC2016
+  local query
+  query=$(jq -n -c \
+    --argjson groupId "$group_id" \
+    --argjson attributes "$attributes_json" '
+    {
+      "query": "mutation UpdateGroup($group: UpdateGroupInput!) {updateGroup(group: $group) {ok}}",
+      "operationName": "UpdateGroup",
+      "variables": {
+        "group": {
+          "id": $groupId,
+          "insertAttributes": $attributes
+        }
+      }
+    }
+  ')
+
+  local response='' error=''
+  response="$(curl --silent --request POST \
+    --url "$LLDAP_URL/api/graphql" \
+    --header "Authorization: Bearer $TOKEN" \
+    --header 'Content-Type: application/json' \
+    --data "$query")"
+
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf 'Error updating attributes for group ID "%s": %s\n' "$group_id" "$error"
+  else
+    printf 'Custom attributes for group ID "%s" successfully updated\n' "$group_id"
+  fi
+}
+
+update_user_attributes() {
+  local user_id="$1"
+  local attributes_json="$2"
+
+  local query
+  query=$(jq -n -c \
+    --arg userId "$user_id" \
+    --argjson attributes "$attributes_json" '
+    {
+      "query": "mutation UpdateUser($user: UpdateUserInput!) {updateUser(user: $user) {ok}}",
+      "operationName": "UpdateUser",
+      "variables":{
+        "user": {
+          "id":$userId,
+          "insertAttributes":$attributes
+        }
+      }
+    }
+  ')
+
+  local response='' error=''
+  response="$(curl --silent --request POST \
+    --url "$LLDAP_URL/api/graphql" \
+    --header "Authorization: Bearer $TOKEN" \
+    --header 'Content-Type: application/json' \
+    --data "$query")"
+
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf 'Error updating attributes for user "%s": %s\n' "$user_id" "$error"
+  else
+    printf 'Custom attributes for user "%s" successfully updated\n' "$user_id"
+  fi
+}
+
+extract_custom_group_attributes() {
+  extract_custom_attributes "$1" '"name"'
+}
+
+extract_custom_user_attributes() {
+  extract_custom_attributes "$1" '"id","email","password","displayName","firstName","lastName","groups","avatar_file","avatar_url","gravatar_avatar","weserv_avatar"'
+}
+
+extract_custom_attributes() {
+  local json_config="$1"
+  local standard_fields="$2"
+
+  # Extract all keys from the user config
+  local all_keys=$(echo "$json_config" | jq 'keys | .[]')
+
+  # Filter out standard fields
+  local custom_keys=$(echo "$all_keys" | jq -c --arg std_fields "$standard_fields" 'select(. | inside($std_fields) | not)')
+
+  # Build attribute array for GraphQL
+  local attributes_array="["
+  local first=true
+
+  while read -r key; do
+    if $first; then
+      first=false
+    else
+      attributes_array+=","
+    fi
+
+    key=$(echo "$key" | tr -d '"')
+
+    # If key is empty - this condition traps configurations without custom attributes
+    if [ -z "$key" ]; then continue; fi
+
+    # Get the value
+    local value=$(echo "$json_config" | jq --arg key "$key" '.[$key]')
+
+    # If the value is null, skip it
+    if echo "$value" | jq -e 'type == "null"' > /dev/null; then
+      continue
+    # Check if value is a JSON array
+    elif echo "$value" | jq -e 'type == "array"' > /dev/null; then
+      # For array types, ensure each element is a string, use compact JSON
+      local array_values=$(echo "$value" | jq -c 'map(tostring)')
+      attributes_array+="{\"name\":\"$key\",\"value\":$array_values}"
+    else
+      # For single values, make sure it's a string
+      local string_value=$(echo "$value" | jq 'tostring')
+      attributes_array+="{\"name\":\"$key\",\"value\":[$string_value]}"
+    fi
+  done < <(echo "$custom_keys")
+
+  attributes_array+="]"
+  echo "$attributes_array"
+}
+
+__common_user_mutation_query() {
+  local \
+    query="$1" \
+    id="${2:-null}" \
+    email="${3:-null}" \
+    displayName="${4:-null}" \
+    firstName="${5:-null}" \
+    lastName="${6:-null}" \
+    avatar_file="${7:-null}" \
+    avatar_url="${8:-null}" \
+    gravatar_avatar="${9:-false}" \
+    weserv_avatar="${10:-false}"
+
+  local variables_arr=(
+    '-s' "id=$id"
+    '-s' "email=$email"
+    '-s' "displayName=$displayName"
+    '-s' "firstName=$firstName"
+    '-s' "lastName=$lastName"
+  )
+
+  local temp_avatar_file=''
+
+  if [[ "$gravatar_avatar" == 'true' ]]; then
+    avatar_url="https://gravatar.com/avatar/$(printf '%s' "$email" | sha256sum | cut -d ' ' -f 1)?size=512"
+  fi
+
+  if [[ "$avatar_url" != 'null' ]]; then
+    temp_avatar_file="${TMP_AVATAR_DIR}/$(printf '%s' "$avatar_url" | md5sum | cut -d ' ' -f 1)"
+
+    if ! [[ -f "$temp_avatar_file" ]]; then
+      if [[ "$weserv_avatar" == 'true' ]]; then
+        avatar_url="https://wsrv.nl/?url=$avatar_url&output=jpg"
+      fi
+      curl --silent --location --output "$temp_avatar_file" "$avatar_url"
+    fi
+
+    avatar_file="$temp_avatar_file"
+  fi
+
+  if [[ "$avatar_file" == 'null' ]]; then
+    variables_arr+=('-s' 'avatar=null')
+  else
+    variables_arr+=("avatar=%$avatar_file")
+  fi
+
+  make_query <(printf '%s' "$query") <(jo -- user=:<(jo -- "${variables_arr[@]}"))
+}
+
+create_user() {
+  local id="$1"
+
+  if user_exists "$id"; then
+    printf 'User "%s" already exists\n' "$id"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation CreateUser($user: CreateUserInput!) {createUser(user: $user) {id creationDate}}","operationName":"CreateUser"}'
+
+  local response='' error=''
+  response="$(__common_user_mutation_query "$query" "$@")"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'User "%s" successfully created\n' "$id"
+  fi
+}
+
+update_user() {
+  local id="$1"
+
+  if ! user_exists "$id"; then
+    printf 'User "%s" is not exists\n' "$id"
+    return
+  fi
+
+  # shellcheck disable=SC2016
+  local query='{"query":"mutation UpdateUser($user: UpdateUserInput!) {updateUser(user: $user) {ok}}","operationName":"UpdateUser"}'
+
+  local response='' error=''
+  response="$(__common_user_mutation_query "$query" "$@")"
+  error="$(printf '%s' "$response" | jq --raw-output '.errors | if . != null then .[].message else empty end')"
+  if [[ -n "$error" ]]; then
+    printf '%s\n' "$error"
+  else
+    printf 'User "%s" successfully updated\n' "$id"
+  fi
+}
+
+create_update_user() {
+  local id="$1"
+
+  if user_exists "$id"; then
+    update_user "$@"
+  else
+    create_user "$@"
+  fi
+}
+
+main() {
+  check_install_dependencies
+  check_required_env_vars
+
+  local user_config_files=("${USER_CONFIGS_DIR}"/*.json)
+  local group_config_files=("${GROUP_CONFIGS_DIR}"/*.json)
+  local user_schema_files=()
+  local group_schema_files=()
+
+  local file=''
+  [[ -d "$USER_SCHEMAS_DIR" ]] && for file in "${USER_SCHEMAS_DIR}"/*.json; do
+    user_schema_files+=("$file")
+  done
+  [[ -d "$GROUP_SCHEMAS_DIR" ]] && for file in "${GROUP_SCHEMAS_DIR}"/*.json; do
+    group_schema_files+=("$file")
+  done
+
+  if ! check_configs_validity "${group_config_files[@]}" "${user_config_files[@]}" "${group_schema_files[@]}" "${user_schema_files[@]}"; then
+    exit 1
+  fi
+
+  until curl --silent -o /dev/null "$LLDAP_URL"; do
+    printf 'Waiting lldap to start...\n'
+    sleep 10
+  done
+
+  auth "$LLDAP_URL" "$LLDAP_ADMIN_USERNAME" "$LLDAP_ADMIN_PASSWORD"
+
+  printf -- '\n--- group schemas ---\n'
+  local group_schema_config_row=''
+  [[ ${#group_schema_files[@]} -gt 0 ]] && while read -r group_schema_config_row; do
+    local field='' name='' attributeType='' isEditable='' isList='' isVisible=''
+    for field in 'name' 'attributeType' 'isEditable' 'isList' 'isVisible'; do
+      declare "$field"="$(printf '%s' "$group_schema_config_row" | jq --raw-output --arg field "$field" '.[$field]')"
+    done
+    create_group_schema_property "$name" "$attributeType" "$isEditable" "$isList" "$isVisible"
+  done < <(jq --compact-output '.[]' -- "${group_schema_files[@]}")
+  printf -- '--- group schemas ---\n'
+
+  printf -- '\n--- user schemas ---\n'
+  local user_schema_config_row=''
+  [[ ${#user_schema_files[@]} -gt 0 ]] && while read -r user_schema_config_row; do
+    local field='' name='' attributeType='' isEditable='' isList='' isVisible=''
+    for field in 'name' 'attributeType' 'isEditable' 'isList' 'isVisible'; do
+      declare "$field"="$(printf '%s' "$user_schema_config_row" | jq --raw-output --arg field "$field" '.[$field]')"
+    done
+    create_user_schema_property "$name" "$attributeType" "$isEditable" "$isList" "$isVisible"
+  done < <(jq --compact-output '.[]' -- "${user_schema_files[@]}")
+  printf -- '--- user schemas ---\n'
+
+  local redundant_groups=''
+  redundant_groups="$(get_group_list | jq '[ .data.groups[].displayName ]' | jq --compact-output '. - ["lldap_admin","lldap_password_manager","lldap_strict_readonly"]')"
+
+  printf -- '\n--- groups ---\n'
+  local group_config=''
+  while read -r group_config; do
+    local group_name=''
+    group_name="$(printf '%s' "$group_config" | jq --raw-output '.name')"
+    create_group "$group_name"
+    redundant_groups="$(printf '%s' "$redundant_groups" | jq --compact-output --arg name "$group_name" '. - [$name]')"
+    # Process custom attributes
+    printf -- '--- Processing custom attributes for group %s ---\n' "$group_name"
+    local attributes_json
+    attributes_json=$(extract_custom_group_attributes "$group_config")
+
+    if [[ "$attributes_json" != "[]" ]]; then
+      # Get the group ID
+      local group_id
+      group_id="$(get_group_id "$group_name")"
+
+      update_group_attributes "$group_id" "$attributes_json"
+    else
+      printf 'No custom attributes found for group "%s"\n' "$group_name"
+    fi
+  done < <(jq --compact-output '.' -- "${group_config_files[@]}")
+  printf -- '--- groups ---\n'
+
+  printf -- '\n--- redundant groups ---\n'
+  if [[ "$redundant_groups" == '[]' ]]; then
+    printf 'There are no redundant groups\n'
+  else
+    local group_name=''
+    while read -r group_name; do
+      if [[ "$DO_CLEANUP" == 'true' ]]; then
+        delete_group "$group_name"
+      else
+        printf '[WARNING] Group "%s" is not declared in config files\n' "$group_name"
+      fi
+    done < <(printf '%s' "$redundant_groups" | jq --raw-output '.[]')
+  fi
+  printf -- '--- redundant groups ---\n'
+
+  local redundant_users=''
+  redundant_users="$(get_users_list | jq '[ .data.users[].id ]' | jq --compact-output --arg admin_id "$LLDAP_ADMIN_USERNAME" '. - [$admin_id]')"
+
+  TMP_AVATAR_DIR="$(mktemp -d)"
+
+  local user_config=''
+  while read -r user_config; do
+    local field='' id='' email='' displayName='' firstName='' lastName='' avatar_file='' avatar_url='' gravatar_avatar='' weserv_avatar='' password=''
+    for field in 'id' 'email' 'displayName' 'firstName' 'lastName' 'avatar_file' 'avatar_url' 'gravatar_avatar' 'weserv_avatar' 'password'; do
+      declare "$field"="$(printf '%s' "$user_config" | jq --raw-output --arg field "$field" '.[$field]')"
+    done
+    printf -- '\n--- %s ---\n' "$id"
+
+    create_update_user "$id" "$email" "$displayName" "$firstName" "$lastName" "$avatar_file" "$avatar_url" "$gravatar_avatar" "$weserv_avatar"
+    redundant_users="$(printf '%s' "$redundant_users" | jq --compact-output --arg id "$id" '. - [$id]')"
+
+    if [[ "$password" != 'null' ]] && [[ "$password" != '""' ]]; then
+      "$LLDAP_SET_PASSWORD_PATH" --base-url "$LLDAP_URL" --token "$TOKEN" --username "$id" --password "$password"
+    fi
+
+    # Process custom attributes
+    printf -- '--- Processing custom attributes for user %s ---\n' "$id"
+    local attributes_json
+    attributes_json=$(extract_custom_user_attributes "$user_config")
+
+    if [[ "$attributes_json" != "[]" ]]; then
+      update_user_attributes "$id" "$attributes_json"
+    else
+      printf 'No custom attributes found for user "%s"\n' "$id"
+    fi
+
+    local redundant_user_groups=''
+    redundant_user_groups="$(get_user_details "$id" | jq '[ .data.user.groups[].displayName ]')"
+
+    local group=''
+    while read -r group; do
+      if [[ -n "$group" ]]; then
+        add_user_to_group "$id" "$group"
+        redundant_user_groups="$(printf '%s' "$redundant_user_groups" | jq --compact-output --arg group "$group" '. - [$group]')"
+      fi
+    done < <(printf '%s' "$user_config" | jq --raw-output '.groups | if . == null then "" else .[] end')
+
+    local user_group_name=''
+    while read -r user_group_name; do
+      if [[ "$DO_CLEANUP" == 'true' ]]; then
+        remove_user_from_group "$id" "$user_group_name"
+      else
+        printf '[WARNING] User "%s" is not declared as member of the "%s" group in the config files\n' "$id" "$user_group_name"
+      fi
+    done < <(printf '%s' "$redundant_user_groups" | jq --raw-output '.[]')
+    printf -- '--- %s ---\n' "$id"
+  done < <(jq --compact-output '.' -- "${user_config_files[@]}")
+
+  rm -r "$TMP_AVATAR_DIR"
+
+  printf -- '\n--- redundant users ---\n'
+  if [[ "$redundant_users" == '[]' ]]; then
+    printf 'There are no redundant users\n'
+  else
+    local id=''
+    while read -r id; do
+      if [[ "$DO_CLEANUP" == 'true' ]]; then
+        delete_user "$id"
+      else
+        printf '[WARNING] User "%s" is not declared in config files\n' "$id"
+      fi
+    done < <(printf '%s' "$redundant_users" | jq --raw-output '.[]')
+  fi
+  printf -- '--- redundant users ---\n'
+}
+
+main "$@"

machines/auth/bootstrap/default.nix

@@ -0,0 +1,39 @@
+{
+  pkgs,
+  config,
+  ...
+}: {
+  systemd.services.lldap-bootstrap = {
+    description = "Bootstraps LLDAP users";
+    requires = ["lldap.service"];
+    serviceConfig = {
+      DynamicUser = true;
+      Type = "oneshot";
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateUsers = true;
+      PrivateTmp = true;
+      LoadCredential = "inadyn.conf:${config.sops.templates."inadyn.conf".path}";
+      CacheDirectory = "inadyn";
+      ExecStart = ''
+        export LLDAP_URL=http://localhost:8080
+        export LLDAP_ADMIN_USERNAME=admin
+        export LLDAP_ADMIN_PASSWORD=changeme
+        export USER_CONFIGS_DIR="$(realpath ./configs/user)"
+        export GROUP_CONFIGS_DIR="$(realpath ./configs/group)"
+        export USER_SCHEMAS_DIR="$(realpath ./configs/user-schema)"
+        export GROUP_SCHEMAS_DIR="$(realpath ./configs/group-schema)"
+        export LLDAP_SET_PASSWORD_PATH="$(realpath ./lldap_set_password)"
+        export DO_CLEANUP=false
+        ./bootstrap.sh
+
+        ${pkgs.inadyn}/bin/inadyn \
+          --foreground \
+          --syslog \
+          --once \
+          --cache-dir ''${CACHE_DIRECTORY} \
+          --config ''${CREDENTIALS_DIRECTORY}/inadyn.conf
+      '';
+    };
+  };
+}

machines/auth/bootstrap/group-configs.nix

@@ -0,0 +1,14 @@
+{config, ...}: {
+  sops.templates."default-groups.json" = {
+    content = ''
+      {
+        "name": "git-user"
+      }
+      {
+        "name": "git-admin"
+      }
+    '';
+    path = "/bootstrap/group-configs/default-groups.json";
+    owner = "lldap";
+  };
+}

machines/auth/bootstrap/lldap-bootstrap.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  cfg = config.services.lldapBootstrap;
+in {
+  imports = [
+    ./user-configs.nix
+    ./group-configs.nix
+  ];
+
+  options.services.lldapBootstrap = {
+    enable = lib.mkEnableOption "LLDAP bootstrapping service.";
+
+    host = lib.mkOption {
+      type = lib.types.str;
+      default = "http://localhost:17170";
+      description = "The LLDAP host and port (e.g., 'localhost:17170').";
+    };
+
+    adminUsername = lib.mkOption {
+      type = lib.types.str;
+      default = "admin";
+      description = "The LLDAP admin username.";
+    };
+
+    adminPasswordFile = lib.mkOption {
+      type = lib.types.path;
+      description = "Path to the sops secret file containing the LLDAP admin password.";
+      default = "/run/secrets/lldap/admin_password";
+      example = "/run/secrets/lldap/admin_password";
+    };
+
+    # Add any other environment variables your bootstrap script might need
+    extraEnv = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = {};
+      description = "Additional environment variables to pass to the bootstrap script.";
+    };
+
+    # Option to control when the bootstrap service runs (e.g., OnUnitActive)
+    # Be careful with this, as you generally only want it to run once.
+    # We'll default to OneShot and disable unless specifically enabled and configured.
+    runOnce = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "If true, the service will run once and then disable itself on success.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.etc."bootstrap/bootstrap.sh" = {
+      source = ./bootstrap.sh;
+      user = "lldap";
+      group = "lldap";
+      mode = "0770";
+    };
+
+    environment.systemPackages = with pkgs; [
+      curl
+      jq
+      jo
+    ];
+
+    systemd.services.lldap-bootstrap = {
+      description = "LLDAP Bootstrap Service";
+      # type = "oneshot";
+      after = ["network.target" "lldap.service"]; # Assuming your LLDAP service is called 'lldap.service'
+      wantedBy = ["multi-user.target"];
+
+      # Environment variables. Secrets will be read directly from the sops-nix managed paths.
+      environment =
+        {
+          LLDAP_URL = cfg.host;
+          LLDAP_ADMIN_USERNAME = cfg.adminUsername;
+          LLDAP_ADMIN_PASSWORD_FILE = cfg.adminPasswordFile;
+          LLDAP_SET_PASSWORD_PATH = "${pkgs.lldap}/bin/lldap_set_password";
+        }
+        // cfg.extraEnv; # Merge with any extra environment variables
+
+      # The command to execute. Ensure your script is executable.
+      # We use pkgs.writeScriptBin to embed the script directly into the Nix store
+      # This makes the service self-contained and ensures the script path is valid.
+      # script = ''
+      #   /etc/bootstrap/bootstrap.sh
+      # '';
+
+      path = [pkgs.bash pkgs.curl pkgs.jq pkgs.jo];
+      # Optional: Control service behavior after successful run.
+      # If runOnce is true, disable the service after it successfully completes.
+      # This prevents it from running on every reboot if the bootstrap is a one-time operation.
+      serviceConfig = lib.mkIf cfg.runOnce {
+        Type = "oneshot";
+        User = "lldap";
+        Group = "lldap";
+        DynamicUser = false;
+        ExecStart = "/etc/bootstrap/bootstrap.sh";
+      };
+    };
+  };
+}

machines/auth/bootstrap/user-configs.nix

@@ -0,0 +1,28 @@
+{config, ...}: {
+  sops.secrets."service_accounts/authelia/password" = {};
+  sops.secrets."service_accounts/forgejo/password" = {};
+  sops.templates."service-accounts.json" = {
+    content = ''
+      {
+        "id": "authelia",
+        "email": "authelia@procopius.dk",
+        "password": "${config.sops.placeholder."service_accounts/authelia/password"}",
+        "displayName": "Authelia",
+        "groups": [
+          "lldap_password_manager"
+        ]
+      }
+      {
+        "id": "forgejo",
+        "email": "forgejo@procopius.dk",
+        "password": "${config.sops.placeholder."service_accounts/forgejo/password"}",
+        "displayName": "Forgejo",
+        "groups": [
+          "lldap_password_manager"
+        ]
+      }
+    '';
+    path = "/bootstrap/user-configs/service-accounts.json";
+    owner = "lldap";
+  };
+}

machines/auth/definition.nix

@@ -0,0 +1,10 @@
+{
+  imports = [
+    ./lldap.nix
+    ./authelia.nix
+    ./postgres.nix
+    ./redis.nix
+  ];
+
+  system.stateVersion = "25.05";
+}

machines/auth/lldap.nix

@@ -0,0 +1,61 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  cfg = config.services.lldap;
+in {
+  imports = [
+    ./bootstrap/lldap-bootstrap.nix
+  ];
+
+  sops.secrets = {
+    "lldap/jwt_secret".owner = "lldap";
+    "lldap/key_seed".owner = "lldap";
+    "lldap/admin_password".owner = "lldap";
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    cfg.settings.http_port
+    cfg.settings.ldap_port
+  ];
+
+  services.lldapBootstrap.enable = true;
+
+  services.lldap = {
+    enable = true;
+    settings = {
+      ldap_base_dn = "dc=procopius,dc=dk";
+      ldap_user_email = "admin@procopius.dk";
+
+      database_url = "postgresql://lldap@localhost/lldap?host=/run/postgresql";
+    };
+    environment = {
+      LLDAP_JWT_SECRET_FILE = config.sops.secrets."lldap/jwt_secret".path;
+      LLDAP_KEY_SEED_FILE = config.sops.secrets."lldap/key_seed".path;
+      LLDAP_LDAP_USER_PASS_FILE = config.sops.secrets."lldap/admin_password".path;
+    };
+  };
+
+  systemd.services.lldap = let
+    dependencies = [
+      "postgresql.service"
+    ];
+  in {
+    # LLDAP requires PostgreSQL to be running
+    after = dependencies;
+    requires = dependencies;
+    # DynamicUser screws up sops-nix ownership because
+    # the user doesn't exist outside of runtime.
+    serviceConfig.DynamicUser = lib.mkForce false;
+  };
+
+  # Setup a user and group for LLDAP
+  users = {
+    users.lldap = {
+      group = "lldap";
+      isSystemUser = true;
+    };
+    groups.lldap = {};
+  };
+}

machines/auth/postgres.nix

@@ -0,0 +1,23 @@
+{lib, ...}: {
+  services.postgresql = {
+    enable = true;
+    ensureDatabases = [
+      "authelia-procopius"
+      "lldap"
+    ];
+    ensureUsers = [
+      {
+        name = "authelia-procopius";
+        ensureDBOwnership = true;
+      }
+      {
+        name = "lldap";
+        ensureDBOwnership = true;
+      }
+    ];
+    authentication = lib.mkForce ''
+      # TYPE  DATABASE        USER            ADDRESS                 METHOD
+      local   all             all                                     trust
+    '';
+  };
+}

machines/auth/redis.nix

@@ -0,0 +1,3 @@
+{
+  services.redis.servers.procopius.enable = true;
+}

machines/monitor/dashboards/gatus.json

@@ -0,0 +1,580 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 3,
+  "links": [],
+  "panels": [
+    {
+      "cacheTimeout": null,
+      "datasource": null,
+      "description": "Number of successful results compared to the total number of results during the current interval",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 9,
+      "links": [],
+      "options": {
+        "fieldOptions": {
+          "calcs": ["mean"],
+          "defaults": {
+            "mappings": [
+              {
+                "id": 0,
+                "op": "=",
+                "text": "N/A",
+                "type": 1,
+                "value": "null"
+              }
+            ],
+            "max": 1,
+            "min": 0,
+            "nullValueMode": "connected",
+            "thresholds": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "semi-dark-orange",
+                "value": 0.6
+              },
+              {
+                "color": "yellow",
+                "value": 0.8
+              },
+              {
+                "color": "dark-green",
+                "value": 0.95
+              }
+            ],
+            "unit": "percentunit"
+          },
+          "override": {},
+          "values": false
+        },
+        "orientation": "horizontal",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": false
+      },
+      "pluginVersion": "6.4.4",
+      "targets": [
+        {
+          "expr": "sum(rate(gatus_results_total{success=\"true\"}[30s])) by (key) / sum(rate(gatus_results_total[30s])) by (key)",
+          "hide": false,
+          "legendFormat": "{{key}}",
+          "refId": "B"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Success rate",
+      "type": "gauge"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 11,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pluginVersion": "6.4.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "gatus_results_duration_seconds",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{key}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Response time",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 10,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pluginVersion": "6.4.4",
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(gatus_results_total{success=\"true\"}[30s])) by (key) / sum(rate(gatus_results_total[30s])) by (key)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{key}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Success rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "description": "Number of results per minute",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 2,
+      "interval": "",
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(gatus_results_total[5m])*60) by (key)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "30s",
+          "intervalFactor": 1,
+          "legendFormat": "{{key}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Total results per minute",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(gatus_results_total{success=\"true\"}[5m])*60) by (key)",
+          "instant": false,
+          "interval": "30s",
+          "legendFormat": "{{key}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Successful results per minute",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 3,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(gatus_results_total{success=\"false\"}[5m])*60) by (key)",
+          "interval": "30s",
+          "legendFormat": "{{key}} ",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Unsuccessful results per minute",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 20,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Gatus",
+  "uid": "KPI7Qj1Wk",
+  "version": 2
+}

machines/monitor/definition.nix

@@ -5,6 +5,7 @@
     ./influxdb.nix
     ./loki.nix
     ./grafana.nix
+    ./gatus.nix
 
     ./jellyfin-exporter.nix
   ];

machines/monitor/gatus.nix

@@ -0,0 +1,32 @@
+{
+  services.gatus = {
+    enable = true;
+    openFirewall = true;
+    settings = {
+      web.port = 8080;
+      metrics = true;
+      endpoints = [
+        {
+          name = "jellyfin";
+          url = "https://jellyfin.procopius.dk/health";
+          interval = "5m";
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY] == Healthy"
+            "[RESPONSE_TIME] < 300"
+          ];
+        }
+        {
+          name = "sonarr";
+          url = "https://sonarr.procopius.dk/health";
+          interval = "5m";
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY] == Healthy"
+            "[RESPONSE_TIME] < 300"
+          ];
+        }
+      ];
+    };
+  };
+}

machines/monitor/grafana.nix

@@ -5,6 +5,8 @@
   lib,
   ...
 }: {
+  # Add grafana user to the inlfuxdb2 group (for secret)
+  users.users.grafana.extraGroups = ["influxdb2"];
   services.grafana.enable = true;
   services.grafana.settings = {
     server = {
@@ -57,7 +59,7 @@
             httpHeaderName1 = "Authorization";
           };
           secureJsonData = {
-            httpHeaderValue1 = "Token iY4MTuqUAVJbBkDUiMde";
+            httpHeaderValue1 = "$__file{${config.sops.secrets."influxdb/token".path}}";
           };
         }
       ];
@@ -123,4 +125,11 @@
     group = "grafana";
     mode = "0644";
   };
+
+  environment.etc."grafana-dashboards/gatus.json" = {
+    source = ./dashboards/gatus.json;
+    user = "grafana";
+    group = "grafana";
+    mode = "0644";
+  };
 }

machines/monitor/influxdb.nix

@@ -12,7 +12,8 @@ in {
   };
   sops.secrets."influxdb/token" = {
     sopsFile = ../../secrets/secrets.yaml;
-    owner = "influxdb2";
+    group = "influxdb2";
+    mode = "0440";
   };
 
   networking.firewall.allowedTCPPorts = [8086];

machines/monitor/prometheus.nix

@@ -75,6 +75,13 @@
     relabel_configs = instance_relabel_config;
   };
 
+  gatus_target = "${monitor_hostname}:8080";
+  gatus_job = {
+    job_name = "gatus";
+    static_configs = [{targets = [gatus_target];}];
+    relabel_configs = instance_relabel_config;
+  };
+
   traefik_monitor_port = 8082;
   traefik_job = {
     job_name = "traefik";
@@ -110,11 +117,11 @@
       {
         targets = [
           "${media_hostname}:9707" # sonarr
-          "${media_hostname}:9708" # readarr
-          "${media_hostname}:9709" # radarr
-          "${media_hostname}:9710" # prowlarr
-          "${media_hostname}:9711" # lidarr
-          "${media_hostname}:9712" # bazarr
+          "${media_hostname}:9708" # radarr
+          "${media_hostname}:9709" # lidarr
+          "${media_hostname}:9710" # readarr
+          "${media_hostname}:9711" # prowlarr
+          # "${media_hostname}:9712" # bazarr
         ];
       }
     ];
@@ -158,6 +165,7 @@ in {
       prometheus_job
       alertmanager_job
       grafana_job
+      gatus_job
       traefik_job
       forgejo_job
       postgres_job

machines/monitor/promtail.nix

@@ -0,0 +1,80 @@
+{
+  config,
+  pkgs,
+  ...
+}: let
+  promtail_port = 9080;
+in {
+  networking.firewall.allowedTCPPorts = [promtail_port];
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/promtail 0755 promtail promtail -"
+  ];
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = promtail_port;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/var/lib/promtail/positions.yaml";
+      };
+      clients = [
+        {
+          url = "http://monitor.lab:3100/loki/api/v1/push";
+        }
+      ];
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            path = "/var/log/journal";
+            labels = {
+              job = "promtail";
+              host = config.networking.hostName;
+              env = "proxmox";
+              instance = "${config.networking.hostName}.lab";
+            };
+          };
+          relabel_configs = [
+            {
+              source_labels = ["__journal__systemd_unit"];
+              target_label = "unit";
+            }
+            {
+              source_labels = ["__journal__hostname"];
+              target_label = "host";
+            }
+            {
+              source_labels = ["__journal__systemd_user_unit"];
+              target_label = "user_unit";
+            }
+            {
+              source_labels = ["__journal__transport"];
+              target_label = "transport";
+            }
+            {
+              source_labels = ["__journal_priority_keyword"];
+              target_label = "severity";
+            }
+          ];
+        }
+        # {
+        #   job_name = "secure";
+        #   static_configs = {
+        #     targets = ["localhost"];
+        #     labels = {
+        #       job = "secure";
+        #       host = config.networking.hostName;
+        #       env = "proxmox";
+        #       instance = "${config.networking.hostName}.lab";
+        #       __path__ = "/var/log/secure";
+        #     };
+        #   };
+        # }
+      ];
+    };
+  };
+}

nixos/hosts/forgejo-runner/runner.nix

@@ -13,7 +13,7 @@
       # tokenFile should be in format TOKEN=<secret>, since it's EnvironmentFile for systemd
       tokenFile = config.sops.secrets."forgejo-runner-registration-token".path;
       labels = [
-        "ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-22.04"
+        "ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-latest"
         "node-22:docker://node:22-bookworm"
         "nixos-latest:docker://nixos/nix"
         ## optionally provide native execution on the host:
@@ -23,6 +23,10 @@
         log = {
           level = "debug";
         };
+
+        container = {
+          docker_host = "automount";
+        };
       };
     };
   };

nixos/hosts/forgejo/forgejo.nix

@@ -56,6 +56,22 @@ in {
       actions = {
         ZOMBIE_TASK_TIMEOUT = "30m";
       };
+      ldap = {
+        AUTHORIZATION_NAME = "My LDAP";
+        HOST = "ldap.example.com";
+        PORT = 389;
+        ENABLE_TLS = false;
+        USER_SEARCH_BASE = "ou=users,dc=example,dc=com";
+        USER_FILTER = "(&(objectClass=user)(sAMAccountName=%[1]s))";
+        USERNAME_ATTRIBUTE = "sAMAccountName";
+        EMAIL_ATTRIBUTE = "mail";
+        FIRST_NAME_ATTRIBUTE = "givenName";
+        SURNAME_ATTRIBUTE = "sn";
+        ADMIN_FILTER = "(&(objectClass=user)(memberOf=cn=admins,ou=groups,dc=example,dc=com))";
+        SKIP_LOCAL_2FA = false;
+        ALLOW_DEACTIVATE_ALL = false;
+      };
+
       oauth2 = {
       };
       oauth2_client = {

nixos/hosts/media/exportarr.nix

@@ -1,44 +0,0 @@
-{config, ...}: {
-  services.prometheus.exporters.exportarr-sonarr = {
-    enable = true;
-    url = "http://media.lab:8989";
-    port = 9707;
-    openFirewall = true;
-    apiKeyFile = config.sops.secrets.sonarr-api-key.path;
-  };
-  services.prometheus.exporters.exportarr-readarr = {
-    enable = true;
-    url = "http://media.lab:8787";
-    port = 9708;
-    openFirewall = true;
-    apiKeyFile = config.sops.secrets.readarr-api-key.path;
-  };
-  services.prometheus.exporters.exportarr-radarr = {
-    enable = true;
-    url = "http://media.lab:7878";
-    port = 9709;
-    openFirewall = true;
-    apiKeyFile = config.sops.secrets.radarr-api-key.path;
-  };
-  services.prometheus.exporters.exportarr-prowlarr = {
-    enable = true;
-    url = "http://media.lab:9696";
-    port = 9710;
-    openFirewall = true;
-    apiKeyFile = config.sops.secrets.prowlarr-api-key.path;
-  };
-  services.prometheus.exporters.exportarr-lidarr = {
-    enable = true;
-    url = "http://media.lab:8686";
-    port = 9711;
-    openFirewall = true;
-    apiKeyFile = config.sops.secrets.lidarr-api-key.path;
-  };
-  services.prometheus.exporters.exportarr-bazarr = {
-    enable = true;
-    url = "http://media.lab:6767";
-    port = 9712;
-    openFirewall = true;
-    apiKeyFile = config.sops.secrets.bazarr-api-key.path;
-  };
-}

nixos/hosts/media/host.nix

@@ -5,8 +5,7 @@
     ./networking.nix
     ./storage.nix
     ./nixarr.nix
-    ./exportarr.nix
-    ./jellyfin-exporter.nix
     ./sops.nix
+    ./modules/monitoring.nix
   ];
 }

nixos/hosts/media/jellyfin-exporter.nix

@@ -1,8 +0,0 @@
-{config, ...}: {
-  services.prometheus.exporters.json = {
-    enable = true;
-    configFile = config.sops.secrets.jellyfin-exporter-config.path;
-    openFirewall = true;
-    user = "jellyfin";
-  };
-}

nixos/hosts/media/lib/api-keys.nix

@@ -0,0 +1,98 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib; let
+  cfg = config.nixarr;
+
+  # Helper to create API key extraction for a service
+  mkApiKeyExtractor = serviceName: serviceConfig: {
+    description = "Extract ${serviceName} API key";
+    after = ["${serviceName}.service"];
+    requires = ["${serviceName}.service"];
+
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      # Use DynamicUser if the parent service does
+      DynamicUser = serviceConfig.serviceConfig.DynamicUser or false;
+      # Only set User if not using DynamicUser
+      ${
+        if !(serviceConfig.serviceConfig.DynamicUser or false)
+        then "User"
+        else null
+      } =
+        serviceConfig.user or null;
+      Group = "${serviceName}-api";
+      UMask = "0027"; # Results in 0640 permissions
+
+      ExecStartPre = [
+        "${pkgs.coreutils}/bin/mkdir -p ${cfg.stateDir}/api-keys"
+        "${pkgs.coreutils}/bin/chown root:${serviceName}-api ${cfg.stateDir}/api-keys"
+        "${pkgs.coreutils}/bin/chmod 750 ${cfg.stateDir}/api-keys"
+        # Wait for config file to exist
+        "${pkgs.bash}/bin/bash -c 'while [ ! -f ${serviceConfig.stateDir}/config.xml ]; do sleep 1; done'"
+      ];
+
+      # Bazarr api key is located a different place...
+      ExecStart = pkgs.writeShellScript "extract-${serviceName}-api-key" ''
+        ${pkgs.dasel}/bin/dasel -f "${serviceConfig.stateDir}/config.xml" \
+          -s ".Config.ApiKey" | tr -d '\n\r' > "${cfg.stateDir}/api-keys/${serviceName}.key"
+        chown $USER:${serviceName}-api "${cfg.stateDir}/api-keys/${serviceName}.key"
+      '';
+    };
+  };
+in {
+  config = mkIf cfg.enable {
+    # Create per-service API key groups
+    users.groups = mkMerge [
+      (mkIf cfg.sonarr.enable {sonarr-api = {};})
+      (mkIf cfg.radarr.enable {radarr-api = {};})
+      (mkIf cfg.lidarr.enable {lidarr-api = {};})
+      (mkIf cfg.readarr.enable {readarr-api = {};})
+      (mkIf cfg.prowlarr.enable {prowlarr-api = {};})
+      # (mkIf cfg.bazarr.enable {bazarr-api = {};})
+    ];
+
+    # Add services that need API keys to their respective groups
+    users.users = mkMerge [
+      # Static users
+      (mkIf cfg.transmission.enable {
+        transmission.extraGroups = optional cfg.prowlarr.enable "prowlarr-api";
+      })
+      (mkIf cfg.transmission.privateTrackers.cross-seed.enable {
+        cross-seed.extraGroups = optional cfg.prowlarr.enable "prowlarr-api";
+      })
+    ];
+
+    # Add api groups to services with DynamicUser
+    systemd.services = mkMerge [
+      (mkIf cfg.sonarr.enable {sonarr.serviceConfig.SupplementaryGroups = ["sonarr-api"];})
+      (mkIf cfg.radarr.enable {radarr.serviceConfig.SupplementaryGroups = ["radarr-api"];})
+      (mkIf cfg.lidarr.enable {lidarr.serviceConfig.SupplementaryGroups = ["lidarr-api"];})
+      (mkIf cfg.readarr.enable {readarr.serviceConfig.SupplementaryGroups = ["readarr-api"];})
+      (mkIf cfg.prowlarr.enable {prowlarr.serviceConfig.SupplementaryGroups = ["prowlarr-api"];})
+      # (mkIf cfg.bazarr.enable {bazarr.serviceConfig.SupplementaryGroups = ["bazarr-api"];})
+      (mkIf cfg.recyclarr.enable {
+        recyclarr.serviceConfig.SupplementaryGroups =
+          (optional cfg.sonarr.enable "sonarr-api")
+          ++ (optional cfg.radarr.enable "radarr-api");
+      })
+
+      # Create API key extractors for enabled services
+      (mkIf cfg.sonarr.enable {"sonarr-api-key" = mkApiKeyExtractor "sonarr" cfg.sonarr;})
+      (mkIf cfg.radarr.enable {"radarr-api-key" = mkApiKeyExtractor "radarr" cfg.radarr;})
+      (mkIf cfg.lidarr.enable {"lidarr-api-key" = mkApiKeyExtractor "lidarr" cfg.lidarr;})
+      (mkIf cfg.readarr.enable {"readarr-api-key" = mkApiKeyExtractor "readarr" cfg.readarr;})
+      (mkIf cfg.prowlarr.enable {"prowlarr-api-key" = mkApiKeyExtractor "prowlarr" cfg.prowlarr;})
+      # (mkIf cfg.bazarr.enable {"bazarr-api-key" = mkApiKeyExtractor "bazarr" cfg.bazarr;})
+    ];
+
+    # Create the api-keys directory
+    systemd.tmpfiles.rules = [
+      "d ${cfg.stateDir}/api-keys 0750 root root - -"
+    ];
+  };
+}

nixos/hosts/media/modules/monitoring.nix

@@ -0,0 +1,266 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib; let
+  cfg = config.nixarr;
+
+  # Helper to determine if an exporter should be enabled
+  shouldEnableExporter = service:
+    cfg.${service}.enable
+    && (cfg.${service}.exporter.enable == null || cfg.${service}.exporter.enable);
+in {
+  imports = [../lib/api-keys.nix];
+
+  options = {
+    nixarr = {
+      exporters = {
+        enable = mkEnableOption "Enable Prometheus exporters for all supported nixarr services";
+      };
+
+      sonarr.exporter = {
+        enable = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Whether to enable the Sonarr Prometheus exporter.
+            - null: enable if exporters.enable is true and sonarr service is enabled (default)
+            - true: force enable if exporters.enable is true
+            - false: always disable
+          '';
+        };
+        port = mkOption {
+          type = types.port;
+          default = 9707;
+          description = "Port for Sonarr metrics";
+        };
+        listenAddr = mkOption {
+          type = types.str;
+          default = "0.0.0.0";
+          description = ''
+            Address for Sonarr exporter to listen on.
+          '';
+        };
+      };
+      radarr.exporter = {
+        enable = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Whether to enable the Radarr Prometheus exporter.
+            - null: enable if exporters.enable is true and radarr service is enabled (default)
+            - true: force enable if exporters.enable is true
+            - false: always disable
+          '';
+        };
+        port = mkOption {
+          type = types.port;
+          default = 9708;
+          description = "Port for Radarr metrics";
+        };
+        listenAddr = mkOption {
+          type = types.str;
+          default = "0.0.0.0";
+          description = ''
+            Address for Radarr exporter to listen on.
+          '';
+        };
+      };
+      lidarr.exporter = {
+        enable = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Whether to enable the Lidarr Prometheus exporter.
+            - null: enable if exporters.enable is true and lidarr service is enabled (default)
+            - true: force enable if exporters.enable is true
+            - false: always disable
+          '';
+        };
+        port = mkOption {
+          type = types.port;
+          default = 9709;
+          description = "Port for Lidarr metrics";
+        };
+        listenAddr = mkOption {
+          type = types.str;
+          default = "0.0.0.0";
+          description = ''
+            Address for Lidarr exporter to listen on.
+          '';
+        };
+      };
+      readarr.exporter = {
+        enable = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Whether to enable the Readarr Prometheus exporter.
+            - null: enable if exporters.enable is true and readarr service is enabled (default)
+            - true: force enable if exporters.enable is true
+            - false: always disable
+          '';
+        };
+        port = mkOption {
+          type = types.port;
+          default = 9710;
+          description = "Port for Readarr metrics";
+        };
+        listenAddr = mkOption {
+          type = types.str;
+          default = "0.0.0.0";
+          description = ''
+            Address for Readarr exporter to listen on.
+          '';
+        };
+      };
+      prowlarr.exporter = {
+        enable = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Whether to enable the Prowlarr Prometheus exporter.
+            - null: enable if exporters.enable is true and prowlarr service is enabled (default)
+            - true: force enable if exporters.enable is true
+            - false: always disable
+          '';
+        };
+        port = mkOption {
+          type = types.port;
+          default = 9711;
+          description = "Port for Prowlarr metrics";
+        };
+        listenAddr = mkOption {
+          type = types.str;
+          default = "0.0.0.0";
+          description = ''
+            Address for Prowlarr exporter to listen on.
+          '';
+        };
+      };
+      bazarr.exporter = {
+        enable = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Whether to enable the Bazarr Prometheus exporter.
+            - null: enable if exporters.enable is true and bazarr service is enabled (default)
+            - true: force enable if exporters.enable is true
+            - false: always disable
+          '';
+        };
+        port = mkOption {
+          type = types.port;
+          default = 9712;
+          description = "Port for Bazarr metrics";
+        };
+        listenAddr = mkOption {
+          type = types.str;
+          default = "0.0.0.0";
+          description = ''
+            Address for Bazarr exporter to listen on.
+          '';
+        };
+      };
+    };
+  };
+
+  config = mkIf (cfg.enable && cfg.exporters.enable) {
+    # Configure Prometheus exporters
+    services.prometheus = {
+      exporters = {
+        # Enable exportarr for each supported service if it's enabled
+        exportarr-sonarr = mkIf (shouldEnableExporter "sonarr") {
+          enable = true;
+          url = "http://127.0.0.1:8989";
+          apiKeyFile = "${cfg.stateDir}/api-keys/sonarr.key";
+          port = cfg.sonarr.exporter.port;
+          listenAddress = cfg.sonarr.exporter.listenAddr;
+          openFirewall = true;
+        };
+
+        exportarr-radarr = mkIf (shouldEnableExporter "radarr") {
+          enable = true;
+          url = "http://127.0.0.1:7878";
+          apiKeyFile = "${cfg.stateDir}/api-keys/radarr.key";
+          port = cfg.radarr.exporter.port;
+          listenAddress = cfg.radarr.exporter.listenAddr;
+          openFirewall = true;
+        };
+
+        exportarr-lidarr = mkIf (shouldEnableExporter "lidarr") {
+          enable = true;
+          url = "http://127.0.0.1:8686";
+          apiKeyFile = "${cfg.stateDir}/api-keys/lidarr.key";
+          port = cfg.lidarr.exporter.port;
+          listenAddress = cfg.lidarr.exporter.listenAddr;
+          openFirewall = true;
+        };
+
+        exportarr-readarr = mkIf (shouldEnableExporter "readarr") {
+          enable = true;
+          url = "http://127.0.0.1:8787";
+          apiKeyFile = "${cfg.stateDir}/api-keys/readarr.key";
+          port = cfg.readarr.exporter.port;
+          listenAddress = cfg.readarr.exporter.listenAddr;
+          openFirewall = true;
+        };
+
+        exportarr-prowlarr = mkIf (shouldEnableExporter "prowlarr") {
+          enable = true;
+          url = "http://127.0.0.1:9696";
+          apiKeyFile = "${cfg.stateDir}/api-keys/prowlarr.key";
+          port = cfg.prowlarr.exporter.port;
+          listenAddress = cfg.prowlarr.exporter.listenAddr;
+          openFirewall = true;
+        };
+
+        # exportarr-bazarr = mkIf (shouldEnableExporter "bazarr") {
+        #   enable = true;
+        #   url = "http://127.0.0.1:6767";
+        #   apiKeyFile = "${cfg.stateDir}/api-keys/bazarr.key";
+        #   port = cfg.bazarr.exporter.port; # 9712;
+        #   openFirewall = true;
+        # };
+      };
+    };
+
+    # Add systemd services for VPN-confined exporters
+    systemd.services = mkMerge [
+      {
+        "prometheus-exportarr-sonarr-exporter" = mkIf (shouldEnableExporter "sonarr") {
+          after = ["sonarr-api-key.service"];
+          requires = ["sonarr-api-key.service"];
+          serviceConfig.SupplementaryGroups = ["sonarr-api"];
+        };
+        "prometheus-exportarr-radarr-exporter" = mkIf (shouldEnableExporter "radarr") {
+          after = ["radarr-api-key.service"];
+          requires = ["radarr-api-key.service"];
+          serviceConfig.SupplementaryGroups = ["radarr-api"];
+        };
+        "prometheus-exportarr-lidarr-exporter" = mkIf (shouldEnableExporter "lidarr") {
+          after = ["lidarr-api-key.service"];
+          requires = ["lidarr-api-key.service"];
+          serviceConfig.SupplementaryGroups = ["lidarr-api"];
+        };
+        "prometheus-exportarr-readarr-exporter" = mkIf (shouldEnableExporter "readarr") {
+          after = ["readarr-api-key.service"];
+          requires = ["readarr-api-key.service"];
+          serviceConfig.SupplementaryGroups = ["readarr-api"];
+        };
+        "prometheus-exportarr-prowlarr-exporter" = mkIf (shouldEnableExporter "prowlarr") {
+          after = ["prowlarr-api-key.service"];
+          requires = ["prowlarr-api-key.service"];
+          serviceConfig.SupplementaryGroups = ["prowlarr-api"];
+        };
+        # "prometheus-exportarr-bazarr-exporter" = mkIf (shouldEnableExporter "bazarr") {
+        #   after = ["bazarr-api-key.service"];
+        #   requires = ["bazarr-api-key.service"];
+        #   serviceConfig.SupplementaryGroups = ["bazarr-api"];
+        # };
+      }
+    ];
+  };
+}

nixos/hosts/media/nixarr.nix

@@ -1,4 +1,8 @@
 {config, ...}: {
+  services.sonarr.settings = {
+    auth.method = "External";
+  };
+
   nixarr = {
     enable = true;
     # These two values are also the default, but you can set them to whatever
@@ -34,6 +38,8 @@
     sonarr.enable = true;
     jellyseerr.enable = true;
 
+    exporters.enable = true;
+
     recyclarr = {
       enable = true;
       configFile = ./recyclarr.yml;

nixos/hosts/media/sops.nix

@@ -4,36 +4,6 @@
     mode = "0440";
   };
 
-  sops.secrets.sonarr-api-key = {
-    sopsFile = ../../secrets/nixarr/secrets.yml;
-    mode = "0440";
-  };
-
-  sops.secrets.radarr-api-key = {
-    sopsFile = ../../secrets/nixarr/secrets.yml;
-    mode = "0440";
-  };
-
-  sops.secrets.readarr-api-key = {
-    sopsFile = ../../secrets/nixarr/secrets.yml;
-    mode = "0440";
-  };
-
-  sops.secrets.bazarr-api-key = {
-    sopsFile = ../../secrets/nixarr/secrets.yml;
-    mode = "0440";
-  };
-
-  sops.secrets.lidarr-api-key = {
-    sopsFile = ../../secrets/nixarr/secrets.yml;
-    mode = "0440";
-  };
-
-  sops.secrets.prowlarr-api-key = {
-    sopsFile = ../../secrets/nixarr/secrets.yml;
-    mode = "0440";
-  };
-
   sops.secrets.jellyfin-exporter-config = {
     sopsFile = ../../secrets/nixarr/secrets.yml;
     owner = "jellyfin";

nixos/hosts/media/storage.nix

@@ -6,18 +6,52 @@
   fileSystems."/data/media/library/shows" = {
     device = "192.168.1.226:/volume1/Media/TV Shows";
     fsType = "nfs4";
-    options = ["x-systemd.automount" "noatime" "_netdev"];
+    options = [
+      "x-systemd.automount" # Automount on first access
+      "noatime" # Don't update access times (performance)
+      "_netdev" # This is a network device; wait for network
+      "defaults" # Standard default options
+      "rw" # Read/write access
+      "hard" # Hard mount (retry indefinitely on error)
+    ];
   };
 
   fileSystems."/data/media/library/movies" = {
     device = "192.168.1.226:/volume1/Media/Movies";
     fsType = "nfs4";
-    options = ["x-systemd.automount" "noatime" "_netdev"];
+    options = [
+      "x-systemd.automount" # Automount on first access
+      "noatime" # Don't update access times (performance)
+      "_netdev" # This is a network device; wait for network
+      "defaults" # Standard default options
+      "rw" # Read/write access
+      "hard" # Hard mount (retry indefinitely on error)
+    ];
   };
 
   fileSystems."/data/media/torrents" = {
     device = "192.168.1.226:/volume1/data/torrents";
     fsType = "nfs4";
-    options = ["x-systemd.automount" "noatime" "_netdev"];
+    options = [
+      "x-systemd.automount" # Automount on first access
+      "noatime" # Don't update access times (performance)
+      "_netdev" # This is a network device; wait for network
+      "defaults" # Standard default options
+      "rw" # Read/write access
+      "hard" # Hard mount (retry indefinitely on error)
+    ];
+  };
+
+  systemd.services = {
+    # jellyfin = {
+    #   requires = ["data-media-library-movies.mount" "data-media-library-shows.mount"];
+    #   after = ["data-media-library-movies.mount" "data-media-library-shows.mount"];
+    #   onFailure = ["data-media-library-movies.mount" "data-media-library-shows.mount"];
+    # };
+    # transmission = {
+    #   requires = ["data-media-torrents.mount"];
+    #   after = ["data-media-torrents.mount"];
+    #   onFailure = ["data-media-torrents.mount"];
+    # };
   };
 }

nixos/hosts/traefik/configuration/auth/routers.nix

@@ -6,6 +6,13 @@
     tls.certResolver = "letsencrypt";
   };
 
+  authelia = {
+    rule = "Host(`authelia.procopius.dk`)";
+    service = "authelia";
+    entryPoints = ["websecure"];
+    tls.certResolver = "letsencrypt";
+  };
+
   oauth2proxy = {
     rule = "Host(`radarr.procopius.dk`) && PathPrefix(`/oauth2/`)";
     service = "oauth2proxy";

nixos/hosts/traefik/configuration/auth/services.nix

@@ -1,5 +1,6 @@
 {
-  authentik.loadBalancer.servers = [{url = "http://authentik.lab:9000";}];
   keycloak.loadBalancer.servers = [{url = "http://keycloak.lab:8080";}];
   oauth2proxy.loadBalancer.servers = [{url = "http://localhost:4180";}];
+
+  authelia.loadBalancer.servers = [{url = "http://auth.lab:9091";}];
 }

nixos/hosts/traefik/configuration/monitoring/routers.nix

@@ -19,6 +19,13 @@
     middlewares = ["oauth-auth"];
     tls.certResolver = "letsencrypt";
   };
+  gatus = {
+    rule = "Host(`gatus.procopius.dk`)";
+    service = "gatus";
+    entryPoints = ["websecure"];
+    middlewares = ["oauth-auth"];
+    tls.certResolver = "letsencrypt";
+  };
   umami = {
     rule = "Host(`umami.procopius.dk`)";
     service = "umami";

nixos/hosts/traefik/configuration/monitoring/services.nix

@@ -2,5 +2,6 @@
   prometheus.loadBalancer.servers = [{url = "http://monitor.lab:9090";}];
   grafana.loadBalancer.servers = [{url = "http://monitor.lab:3000";}];
   alertmanager.loadBalancer.servers = [{url = "http://monitor.lab:9093";}];
+  gatus.loadBalancer.servers = [{url = "http://monitor.lab:8080";}];
   umami.loadBalancer.servers = [{url = "http://192.168.1.226:3333";}];
 }

secrets/secrets.yaml

@@ -4,6 +4,23 @@ alertmanager:
 influxdb:
     password: ENC[AES256_GCM,data:OP+4vK6ulZs7jVM4lgnpUatr+Qs=,iv:MEmD6yyy+Z7beVOdR1xNDn0c27DYDIDTYdnaNiaVHks=,tag:dyG7VPPV40JqSE4UAeVbtQ==,type:str]
     token: ENC[AES256_GCM,data:QraVWLW1uCSF0YvbkHCKYtPvqs0=,iv:pzkfEyLksjRFVj7wZS8LxO0idQTpEk7OTMpQSsuIRvQ=,tag:d6U6vMqEYbu3CaTpnc0gGw==,type:str]
+lldap:
+    key_seed: ENC[AES256_GCM,data:27b/wGDJWSTMnKcFCkLFp1F0ezE=,iv:jzqhuVwmeBmCT+dmuHi+c6mLUH3vI9m4km/ris+Nr3o=,tag:5ntBPJOEMtda9CiXw4bicg==,type:str]
+    jwt_secret: ENC[AES256_GCM,data:2eWUKM6x4F56mfyLLR7pfb8adP9V0ZXU6JHkTbd/kf8=,iv:3Cd+bNedLYeo+nnrBxImTdVCwPMtuzjDX18RmviUsYQ=,tag:sbXWkO3Du7grwnDTKDkEcA==,type:str]
+    admin_password: ENC[AES256_GCM,data:eikQyxCoZFf/ny9TfN9ziihLL+M=,iv:axnNHdcA1qWV2O1em0fN+UjPGMA7pUD9zzjZasB/n68=,tag:ZL9XA3w0HicElaQr8/Ynaw==,type:str]
+authelia:
+    jwt_secret: ENC[AES256_GCM,data:I0gnWMeEm+8tDRsWkP78ACIs7+6HTHBdrRaLder6h6yxJ6i5ObvYCwr4plHjD+vTx9hdwyn6+737MSQZO+VaA8Icxi0wu+cQp5HkC2o9oGuv09aWNJ3WVVWEoe2tuhKm2ui90bbNuMB7N/ANtPE3foxnT0cjLUFFYlvZqXYpPMc=,iv:QXImzq1kvG6ONmhYJYFXDuUgqllrmHWStycC2orgS8E=,tag:wfts1Xcfd7afLevQ/aVqIA==,type:str]
+    jwks: ENC[AES256_GCM,data:mwYleg/Vj9XPVTQ/LAN1xfP7IC/xfs3Tdx8tH2DKqyTWxiLQTd23x0ZarfHkBCsovOyi76txGO8ctbkRNJY/x06kfbqoP+t0P6bnzUB3Fwjl+/boh7+Wgw+T+/zuEZMMbUdeN4OHsuDPw2hGK5COs8Z7jt+LwaXhwCKedWCzSvpxZFIX8X267DeI532afPq+KGwOlzoScsIPuMOM9gB+VwuPmTQt8E9ihHCdc09fZ8M69MFGqyQZyfWLxX+ICWw1JSCq9RbDXNTAiRnMZIlb888RvEv2YU2y8j3txQk7/SUhnkK5CeHTGYyDxzQmdEr8ITZXWba+ejf8mJsbg7u50tZ4d8sJlASxD9VzL8YbiMUwam1SNyDByLDOXY8vwTe6uB1ws+DoOBcjP9p5WqxSNRefeFSzQzZjqDoK/zkEH0mEjhvljyVnBHGnTiAiaHQQnwVYbVLQkKih1XutUVhILww5fuV2hYHVWG61haATmAdTTMqMMXhIUAnr5hJ21CQhe2rmtDTyYRrz2Ep1+S9OSN6ixpqQEpV25n5zqe3WXOEqJMxEVbBAV7KovpecXQ+MXtsG2nnHSVWplgjKu5syTQWwfI/EHC1OhLubnPqQ9q/UMBXNbJm2ZyrUDltjh3eeSv39iaAxWexj7SeetTEDHyieYaZiBxxKBCMfnBuhRj6R1zNRLzTPHHsX0P3hQtsBXEqmCnCmuj3ZGl1pBYSDIuvqwQGSc0kv1iGvCQMNEG1oLX+4Je6r4A5cPcrZ2gesovlAzuV09WUF0/78Lrf072mBuJwoFtL3TUt9bAZqp72dVPeypv/5/Bx3GIRLJtZ7i7EDMMAYqnetsCBkgrwm/paXz2q/jNYXNu0hyjGNJ86hKeob1q33KuvM5C1vQrgjrbJrVJVJ//OU1vmyRqDS+wvSYZ68hKkyyd4M3yk1ooE6E2o6P6VFqB0XlTOYMmhTznXd5QYgxv8iWYyURIix8vP5PKrVojvh/6VI+at2rFQ453zKw7wv2EIThyCGaoe+tdnWfCdOUKxloQifRNSX2gWGHwBF2VPITKUvi+oqK5rVNUDgsnnOLUSwagF0ik/TqxQBhaomF4Bdptxh+JbYJY3cSqd5v44vMm3gyCYPKmrKZZsPuW2o/MKTkP0r2Ik8ZzMrzUsGki5wDYa+Ogl0tmWjOGwOKUTmUpxLhw35Qh1H6KpdNDQs+mjf6InqVmFYBWKlyfjmyTDBH3UoIjkUzb1dPg/1Fij1KBjnCsGdIeypj85W0V3gAQ2pPromxcwuDQt//mRY3x+MeEgygDfFLwgPR3lG1iRpiBj1fVBGqhv7Zkne/NHl/9iW0ZS3JXsai3YfWsz5mdoLJINSIRdNAjnMIzyN5NjYAqf6j928oocmB2Dtn6pQCPtO9eKWx2RrVj/6zrJmKYUXm0GU6bnswV1iaOSAx8lHNVkXnbr3p1NWw7hMAFBKFoB/1H99W8b8JuV+fGAyt+t3W4ztr0eBw9qHtXvhEAXXMFNjtvO3bsjBdIFjM3kZ4hvmtOVuFPY5wtUxwI86YGjPLzlb/CWLnTYv7tZfsBLmKXTxD8SXI83pXkmxK8AzI3Mravjh4FE5Vtkp0xGJC8anOJ7SUr6PIewy068XA63c494e1JwO43wq1TktZViXUQVODy7Lq36lsLxXGhbp7DkjuntBNGxeK0bhawQcSuR5tvZgzgt3DL7lVbZPqrd4S3pSpdXZ/y5Gv5VUHo8mnpCQ9ohUTalISH18XGIzVxNAq7RHP2ii56EUhNLxJ7AIrvPh0Oab+S09s1Onxhyrx1cDVK80N+E8pxd7tvUsqs7JfywPW4nzlf7hgMq1dcC7NNQuZXmEy44KI98F8bU/0SNCHJEvpos9frLMNE8p+u2QeeuHqkYZaa7y+gr6iR0srO/Q1HSlnRE5/b1pjs/0EO5ezoAI/qKGurzx2IKu0itkdDvTt+QnOA3blCQuxfWfwWabjm0wvuvTLuiPANrQrSMWVG37GttpS6bIUhqe689ZROAxYKvLFX0J7fHWVZ4ZnvMIqDsKn6kKcD7T4uSQ+8g+ADnwUiPJ6/9NzXrpoGOZMTgc3ARBFLzGc3CkHXvYmkSPriCA+gFu1D40/u5peX6k/jtEqAlDH4uNQK+0RZ3Ik62AYgkG2FIyT3xGZVrNJiUe/a3veX82urtSyd8k2h6iy0e8RV75BzFvGZm5z/7NaxxBDYQe6AE6wVagD8s2A3JOzB7nb16wjAtDed542JVLDdOd9zpk2GAnF6OpC20A,iv:W7MVuBEZlt88WNb0z16BXUSP4CszRCu+YH89HefaaoA=,tag:jYDCIFVeGWqGqKwbwAb2LA==,type:str]
+    hmac_secret: ENC[AES256_GCM,data:vq1I7MEj7Ja2Jc9dP4IMEuLaIGJ+plxaKva+trkpgHqJT45JXguhDBhjnxreskQlEJatlhu2J4GCTZkBIUFfDpnAsVGVOEZR7/9xV20hp0eNt2q2R5hWFMKbtsJq7S/5LUqvjXjRfz974xUsj4OuMCHemsKaeYJ4sNARh6btaxI=,iv:RpCT3iS0A4/djtWBi40cyCrG8qPVkNs/WUJ99DR1LU4=,tag:nsl10KTAqVtuxptj2kJ/0A==,type:str]
+    session_secret: ENC[AES256_GCM,data:nzoObQtqfAZ9TVhbO/PzO2JcN+at6LRhuHnBDabpod9S7D/VPIGLXC91w5Z2KII2geRS6iX2dFCdtJbJsuE9IrDAyL9JJD9QT4x7y1ByWHz8jX8atC8QWGQ8txRdilpYxAjNRoM9jScqvEg8w2J4FLs9lm73wHmia1DURV5Tdec=,iv:+n5Agi4HSWpRZ/sqpuQzgUY65ddSQxEo6oy+Trra6SA=,tag:oQOTYYgUMhaIj11rjnWkcw==,type:str]
+    storage_encryption_key: ENC[AES256_GCM,data:Bv2qZrryiolnliKFB5aG4VqxOLsuy2QFKIVkydCsXZr6n1Drew8KPFBynlAvaX6mMIG6EZDsqMIzQbiN3i0fdMai4hfBfdsC0rDLrxU2TY5SJktr8gkN7tua/cEXIks1NWm2wKDG/bg4wyLqoAj6t/ss1ERSb72gW0ogBK+SAuE=,iv:jNRWWAsgXoi3+KSbWv9gIto7PLZa2HlLHqUc6m8GZH0=,tag:FBlvewkNvUi2NF0bqA3bPw==,type:str]
+    lldap_authelia_password: ENC[AES256_GCM,data:83sRG2IU10hsVm8NVOGyZrGTxOSICnGJqa0oirSukSs=,iv:y1Rlk1l5urIUKsgo7/ngAFnQcprhGWYI3+OiH6JQH9M=,tag:SuNzGpgN0NrO6MklW8vbaw==,type:str]
+    smtp_authelia_password: ENC[AES256_GCM,data:qrEloI4A,iv:NFH3XyzCLOYCFuF6dMfKjaP47dV+ke+F/QZPiN8ogTY=,tag:+fjdnAD9xnOOjs83+EtKFw==,type:str]
+service_accounts:
+    authelia:
+        password: ENC[AES256_GCM,data:r4Qy09FOhUgD48SHSkWKtrlrMptvXYdScCL8h7gjJNs=,iv:IzsDdV4o35hnS/F2S713cJ5pQ+PGiaVTmTWe6YXgfYc=,tag:OisvmY7QI2Ph7R3g3Xy/Ww==,type:str]
+    forgejo:
+        password: ENC[AES256_GCM,data:HLEoGjx++9fkiJQoLWQvAjgg58mIs1vk1hvUJvr6TiA=,iv:mPIx9cSlHEK+0wLs1/1bjYcsxgdwgLReUoI5JrA4E1k=,tag:TdyznTIGiAFFq8D3Irb0rA==,type:str]
 sops:
     age:
         - recipient: age1n20y9kmdh324m3tkclvhmyuc7c8hk4w84zsal725adahwl8nzq0s04aq4y
@@ -15,7 +32,7 @@ sops:
             QzNYRk5ERmR4aGtLQ3dwQ1lPeDZyaEkKJMLXqv6tBBql7VVnWDIwAh24SfQ2O6Ca
             CEOQTGEonbqr5doWqTsXUXrdQAS0amL45UdT6ITFtfNAjaHwCMfhZg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-07-06T17:10:59Z"
-    mac: ENC[AES256_GCM,data:dXLWT5fmSs2ddpFPXA1yOtwaej7b3lPesFxN7aEZ/bV6YRr+Ht5dHFQcXO0TfJArzhFRRtAumdcdVsorMkR4tao4XCcimACcWrZgXlXGM6XgT3hdPJ4006QLePXU+uyzpqyEuOouaxF7fyuSTL68uDr+E/NAHgmP2dnqpWnebpY=,iv:oUkmH/ngp8wvbuXay+2X6YBqhesNdtOPZOV4lvsc/s4=,tag:GErA9zgdkTarUD6fWiMupg==,type:str]
+    lastmodified: "2025-07-16T00:07:58Z"
+    mac: ENC[AES256_GCM,data:kKSjCmLGbr7WaLb+Z1KZL/bjBszNgCCAb67CENaKKpFbqbCk3o5QFok/kVTs3k3JwKmODqbTe0ebP6uMENN/t85+1n4ofnMq5ire/NqCyoE1EJFDmG9xyys24CB+NJJZ2trdxm5CYutme7FfG4bQY0/2OgflmjiZeBsMZcxRxtI=,iv:md6qX+WlFFgMFbDs8MTTKXEOPWKFoVYAUMehWvGF5wk=,tag:Y3AelrfrsBMIJ0wYzFYtLQ==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.10.2