homelab framework module init (everything is a mess)
This commit is contained in:
parent
0347f4d325
commit
bcbcc8b17b
94 changed files with 7289 additions and 436 deletions
226
modules/homelab/lib/node-aggregation.nix
Normal file
226
modules/homelab/lib/node-aggregation.nix
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
{lib}: let
|
||||
inherit (lib) flatten mapAttrs mapAttrsToList filter groupBy length unique attrByPath splitString;
|
||||
|
||||
# Generic function to aggregate any attribute across nodes
|
||||
aggregateFromNodes = {
|
||||
nodes,
|
||||
attributePath, # e.g. "homelab.monitoring.endpoints" or "homelab.backups.jobs"
|
||||
enhancer ? null, # optional function to enhance each item with node context
|
||||
}: let
|
||||
# Extract the attribute from each node using the path
|
||||
getNestedAttr = path: config: let
|
||||
pathList = splitString "." path;
|
||||
in
|
||||
attrByPath pathList [] config;
|
||||
|
||||
# Get all items from all nodes
|
||||
allItems = flatten (mapAttrsToList
|
||||
(nodeName: nodeConfig: let
|
||||
items = getNestedAttr attributePath nodeConfig.config;
|
||||
baseEnhancer = item:
|
||||
item
|
||||
// {
|
||||
_nodeName = nodeName;
|
||||
_nodeConfig = nodeConfig;
|
||||
_nodeAddress = nodeConfig.config.networking.hostName or nodeName;
|
||||
};
|
||||
finalEnhancer =
|
||||
if enhancer != null
|
||||
then (item: enhancer (baseEnhancer item))
|
||||
else baseEnhancer;
|
||||
in
|
||||
map finalEnhancer items)
|
||||
nodes);
|
||||
in {
|
||||
# Raw aggregated data
|
||||
all = allItems;
|
||||
|
||||
# Common grouping patterns
|
||||
byNode = groupBy (item: item._nodeName) allItems;
|
||||
byType = groupBy (item: item.type or "unknown") allItems;
|
||||
byService = groupBy (item: item.service or "unknown") allItems;
|
||||
|
||||
# Utility functions for filtering
|
||||
filterBy = predicate: filter predicate allItems;
|
||||
ofType = type: filter (item: (item.type or "") == type) allItems;
|
||||
|
||||
count = length allItems;
|
||||
countBy = fn: mapAttrs (key: items: length items) (groupBy fn allItems);
|
||||
};
|
||||
|
||||
# Specialized aggregators for common use cases
|
||||
aggregators = {
|
||||
monitoring = nodes: let
|
||||
# Aggregate metrics endpoints
|
||||
metricsAgg = aggregateFromNodes {
|
||||
inherit nodes;
|
||||
attributePath = "homelab.monitoring.metrics";
|
||||
enhancer = endpoint:
|
||||
endpoint
|
||||
// {
|
||||
_fullAddress = "${endpoint.host or endpoint._nodeAddress}:${toString endpoint.port}";
|
||||
_metricsUrl = "http://${endpoint.host or endpoint._nodeAddress}:${toString endpoint.port}${endpoint.path or "/metrics"}";
|
||||
_type = "metrics";
|
||||
};
|
||||
};
|
||||
# Aggregate health checks
|
||||
healthChecksAgg = aggregateFromNodes {
|
||||
inherit nodes;
|
||||
attributePath = "homelab.monitoring.healthChecks";
|
||||
enhancer = check: let
|
||||
# Compute the actual host and URL
|
||||
actualHost =
|
||||
if check.useExternalDomain or false
|
||||
then "${check.subdomain}.${check._nodeConfig.config.homelab.externalDomain or "example.com"}"
|
||||
else check.host or check._nodeAddress;
|
||||
portPart =
|
||||
if check.port != null
|
||||
then ":${toString check.port}"
|
||||
else "";
|
||||
url = "${check.protocol or "http"}://${actualHost}${portPart}${check.path or "/"}";
|
||||
in
|
||||
check
|
||||
// {
|
||||
_actualHost = actualHost;
|
||||
_url = url;
|
||||
_type = "health-check";
|
||||
# Merge default labels with node context
|
||||
labels =
|
||||
(check.labels or {})
|
||||
// {
|
||||
node = check._nodeName;
|
||||
environment = check._nodeConfig.config.homelab.environment or "unknown";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
metricsAgg
|
||||
// healthChecksAgg
|
||||
// {
|
||||
# Metrics-specific aggregations
|
||||
allMetrics = metricsAgg.all;
|
||||
metricsByNode = metricsAgg.byNode;
|
||||
metricsByJobName = groupBy (m: m.jobName or "unknown") metricsAgg.all;
|
||||
|
||||
# Health checks-specific aggregations
|
||||
allHealthChecks = healthChecksAgg.all;
|
||||
healthChecksByNode = healthChecksAgg.byNode;
|
||||
healthChecksByGroup = groupBy (hc: hc.group or "default") healthChecksAgg.all;
|
||||
healthChecksByProtocol = groupBy (hc: hc.protocol or "http") healthChecksAgg.all;
|
||||
|
||||
# Filtered health checks
|
||||
externalHealthChecks = filter (hc: hc.useExternalDomain or false) healthChecksAgg.all;
|
||||
internalHealthChecks = filter (hc: !(hc.useExternalDomain or false)) healthChecksAgg.all;
|
||||
enabledHealthChecks = filter (hc: hc.enabled or true) healthChecksAgg.all;
|
||||
|
||||
# Summary statistics
|
||||
summary = {
|
||||
totalMetrics = length metricsAgg.all;
|
||||
totalHealthChecks = length healthChecksAgg.all;
|
||||
healthChecksByGroup =
|
||||
mapAttrs (group: checks: length checks)
|
||||
(groupBy (hc: hc.group or "default") healthChecksAgg.all);
|
||||
healthChecksByProtocol =
|
||||
mapAttrs (protocol: checks: length checks)
|
||||
(groupBy (hc: hc.protocol or "http") healthChecksAgg.all);
|
||||
externalChecksCount = length (filter (hc: hc.useExternalDomain or false) healthChecksAgg.all);
|
||||
internalChecksCount = length (filter (hc: !(hc.useExternalDomain or false)) healthChecksAgg.all);
|
||||
};
|
||||
};
|
||||
|
||||
# Promtail log configurations
|
||||
# logs = nodes:
|
||||
# aggregateFromNodes {
|
||||
# inherit nodes;
|
||||
# attributePath = "homelab.logging.sources";
|
||||
# enhancer = logSource:
|
||||
# logSource
|
||||
# // {
|
||||
# # Add log-specific computed fields
|
||||
# _logPath = logSource.path or "/var/log/${logSource.service}.log";
|
||||
# _labels =
|
||||
# (logSource.labels or {})
|
||||
# // {
|
||||
# node = logSource._nodeName;
|
||||
# service = logSource.service or "unknown";
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
|
||||
# Reverse proxy configurations
|
||||
reverseProxy = nodes:
|
||||
aggregateFromNodes {
|
||||
inherit nodes;
|
||||
attributePath = "homelab.reverseProxy.entries";
|
||||
enhancer = entry:
|
||||
entry
|
||||
// {
|
||||
# Add proxy-specific computed fields
|
||||
_upstream = "http://${entry.host or entry._nodeAddress}:${toString entry.port}";
|
||||
_fqdn = "${entry.subdomain or entry.service}.${entry.domain or "local"}";
|
||||
};
|
||||
};
|
||||
|
||||
# Backup jobs with enhanced aggregation
|
||||
backups = nodes: let
|
||||
baseAgg = aggregateFromNodes {
|
||||
inherit nodes;
|
||||
attributePath = "homelab.backups.jobs";
|
||||
enhancer = backup:
|
||||
backup
|
||||
// {
|
||||
_sourceNode = backup._nodeName;
|
||||
_backupId = "${backup._nodeName}-${backup.name}";
|
||||
_jobFqdn = "${backup.name}.${backup._nodeName}";
|
||||
};
|
||||
};
|
||||
|
||||
# Get all unique backends across all nodes
|
||||
allBackends = let
|
||||
allBackendConfigs =
|
||||
mapAttrsToList
|
||||
(nodeName: nodeConfig:
|
||||
attrByPath ["homelab" "backups" "backends"] {} nodeConfig.config)
|
||||
nodes;
|
||||
enabledBackends = flatten (map (backends:
|
||||
filter (name: backends.${name} != null) (lib.attrNames backends))
|
||||
allBackendConfigs);
|
||||
in
|
||||
unique enabledBackends;
|
||||
in
|
||||
baseAgg
|
||||
// {
|
||||
# Backup-specific aggregations
|
||||
byBackend = groupBy (job: job.backend) baseAgg.all;
|
||||
allBackends = allBackends;
|
||||
|
||||
# Enhanced summary
|
||||
summary = {
|
||||
totalJobs = length baseAgg.all;
|
||||
jobsByBackend =
|
||||
mapAttrs (backend: jobs: length jobs)
|
||||
(groupBy (job: job.backend) baseAgg.all);
|
||||
jobsByNode = baseAgg.countBy (job: job._nodeName);
|
||||
availableBackends = allBackends;
|
||||
backendsInUse = unique (map (job: job.backend) baseAgg.all);
|
||||
};
|
||||
};
|
||||
};
|
||||
in {
|
||||
inherit aggregateFromNodes aggregators;
|
||||
|
||||
# Convenience function to create a module that provides global aggregations
|
||||
mkGlobalModule = attributeName: aggregatorFn: {
|
||||
lib,
|
||||
nodes,
|
||||
...
|
||||
}: {
|
||||
options.homelab.global.${attributeName} = lib.mkOption {
|
||||
type = lib.types.attrs;
|
||||
readOnly = true;
|
||||
description = "Globally aggregated ${attributeName} from all nodes";
|
||||
};
|
||||
|
||||
config.homelab.global.${attributeName} = aggregatorFn nodes;
|
||||
};
|
||||
}
|
||||
295
modules/homelab/lib/service-interface.nix
Normal file
295
modules/homelab/lib/service-interface.nix
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
# Standard service interface for homelab services
|
||||
# This provides a consistent contract that all services should follow
|
||||
{lib}: let
|
||||
inherit (lib) mkOption mkEnableOption types;
|
||||
|
||||
# Define the standard service interface
|
||||
mkServiceInterface = {
|
||||
serviceName,
|
||||
defaultPort ? null,
|
||||
defaultSubdomain ? serviceName,
|
||||
defaultDescription ? "Homelab ${serviceName} service",
|
||||
monitoringPath ? "/metrics",
|
||||
healthCheckPath ? "/health",
|
||||
healthCheckConditions ? ["[STATUS] == 200"],
|
||||
# Custom options that the service wants to expose
|
||||
serviceOptions ? {},
|
||||
}:
|
||||
{
|
||||
# Standard interface options that all services must have
|
||||
enable = mkEnableOption defaultDescription;
|
||||
|
||||
port = mkOption {
|
||||
type = types.port;
|
||||
default =
|
||||
if defaultPort != null
|
||||
then defaultPort
|
||||
else throw "Service ${serviceName} must specify a default port";
|
||||
description = "Port for ${serviceName} service";
|
||||
};
|
||||
|
||||
openFirewall = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Whether to automatically open firewall ports";
|
||||
};
|
||||
|
||||
proxy = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Enable reverse proxy for this service";
|
||||
};
|
||||
|
||||
subdomain = mkOption {
|
||||
type = types.str;
|
||||
default = defaultSubdomain;
|
||||
description = "Subdomain for reverse proxy (${defaultSubdomain}.yourdomain.com)";
|
||||
};
|
||||
|
||||
enableAuth = mkOption {
|
||||
type = types.bool;
|
||||
default = false;
|
||||
description = "Enable authentication for reverse proxy";
|
||||
};
|
||||
|
||||
enableSSL = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Enable SSL for reverse proxy";
|
||||
};
|
||||
};
|
||||
|
||||
monitoring = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Enable monitoring (metrics and health checks)";
|
||||
};
|
||||
|
||||
metricsPath = mkOption {
|
||||
type = types.str;
|
||||
default = monitoringPath;
|
||||
description = "Path for metrics endpoint";
|
||||
};
|
||||
|
||||
jobName = mkOption {
|
||||
type = types.str;
|
||||
default = serviceName;
|
||||
description = "Prometheus job name";
|
||||
};
|
||||
|
||||
scrapeInterval = mkOption {
|
||||
type = types.str;
|
||||
default = "30s";
|
||||
description = "Prometheus scrape interval";
|
||||
};
|
||||
|
||||
healthCheck = {
|
||||
enable = mkOption {
|
||||
type = types.bool;
|
||||
default = true;
|
||||
description = "Enable health check monitoring";
|
||||
};
|
||||
|
||||
path = mkOption {
|
||||
type = types.str;
|
||||
default = healthCheckPath;
|
||||
description = "Path for health check endpoint";
|
||||
};
|
||||
|
||||
interval = mkOption {
|
||||
type = types.str;
|
||||
default = "30s";
|
||||
description = "Health check interval";
|
||||
};
|
||||
|
||||
timeout = mkOption {
|
||||
type = types.str;
|
||||
default = "10s";
|
||||
description = "Health check timeout";
|
||||
};
|
||||
|
||||
conditions = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = healthCheckConditions;
|
||||
description = "Health check conditions";
|
||||
};
|
||||
|
||||
group = mkOption {
|
||||
type = types.str;
|
||||
default = "services";
|
||||
description = "Health check group name";
|
||||
};
|
||||
};
|
||||
|
||||
extraLabels = mkOption {
|
||||
type = types.attrsOf types.str;
|
||||
default = {};
|
||||
description = "Additional labels for monitoring";
|
||||
};
|
||||
};
|
||||
|
||||
description = mkOption {
|
||||
type = types.str;
|
||||
default = defaultDescription;
|
||||
description = "Service description";
|
||||
};
|
||||
|
||||
extraOptions = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional service-specific configuration options";
|
||||
};
|
||||
|
||||
# Merge in service-specific options
|
||||
}
|
||||
// serviceOptions;
|
||||
|
||||
# Helper function to implement the standard service behavior
|
||||
mkServiceConfig = {
|
||||
config,
|
||||
cfg,
|
||||
homelabCfg,
|
||||
serviceName,
|
||||
# Function that returns the actual service configuration
|
||||
serviceConfig,
|
||||
# Optional: custom monitoring labels
|
||||
extraMonitoringLabels ? {},
|
||||
# Optional: custom health check configuration
|
||||
customHealthChecks ? [],
|
||||
# Optional: custom reverse proxy configuration
|
||||
customProxyConfig ? {},
|
||||
}: let
|
||||
# Standard monitoring labels
|
||||
standardLabels =
|
||||
{
|
||||
service = serviceName;
|
||||
component = "main";
|
||||
instance = "${homelabCfg.hostname}.${homelabCfg.domain}";
|
||||
}
|
||||
// extraMonitoringLabels // cfg.monitoring.extraLabels;
|
||||
|
||||
# Standard reverse proxy entry
|
||||
standardProxyEntry =
|
||||
{
|
||||
subdomain = cfg.proxy.subdomain;
|
||||
host = homelabCfg.hostname;
|
||||
port = cfg.port;
|
||||
enableAuth = cfg.proxy.enableAuth;
|
||||
enableSSL = cfg.proxy.enableSSL;
|
||||
}
|
||||
// customProxyConfig;
|
||||
|
||||
# Standard metrics configuration
|
||||
standardMetrics = lib.optional cfg.monitoring.enable {
|
||||
name = "${serviceName}-metrics";
|
||||
port = cfg.port;
|
||||
path = cfg.monitoring.metricsPath;
|
||||
jobName = cfg.monitoring.jobName;
|
||||
scrapeInterval = cfg.monitoring.scrapeInterval;
|
||||
labels = standardLabels;
|
||||
};
|
||||
|
||||
# Standard health check configuration
|
||||
standardHealthCheck = lib.optional (cfg.monitoring.enable && cfg.monitoring.healthCheck.enable) {
|
||||
name = "${serviceName}-health";
|
||||
port = cfg.port;
|
||||
path = cfg.monitoring.healthCheck.path;
|
||||
interval = cfg.monitoring.healthCheck.interval;
|
||||
timeout = cfg.monitoring.healthCheck.timeout;
|
||||
conditions = cfg.monitoring.healthCheck.conditions;
|
||||
group = cfg.monitoring.healthCheck.group;
|
||||
labels = standardLabels;
|
||||
};
|
||||
|
||||
# Merge service config with standard behaviors
|
||||
baseConfig = lib.mkMerge [
|
||||
# Service-specific configuration
|
||||
serviceConfig
|
||||
|
||||
# Standard firewall configuration
|
||||
(lib.mkIf cfg.openFirewall {
|
||||
networking.firewall.allowedTCPPorts = [cfg.port];
|
||||
})
|
||||
|
||||
# Standard monitoring configuration
|
||||
(lib.mkIf cfg.monitoring.enable {
|
||||
homelab.monitoring.metrics = standardMetrics;
|
||||
homelab.monitoring.healthChecks = standardHealthCheck ++ customHealthChecks;
|
||||
})
|
||||
|
||||
# Standard reverse proxy configuration
|
||||
(lib.mkIf cfg.proxy.enable {
|
||||
homelab.reverseProxy.entries = [standardProxyEntry];
|
||||
})
|
||||
];
|
||||
in
|
||||
lib.mkIf cfg.enable baseConfig;
|
||||
|
||||
# Validation helper to ensure required options are set
|
||||
validateServiceConfig = cfg: serviceName: [
|
||||
# Validate that if proxy is enabled, subdomain is set
|
||||
(lib.mkIf (cfg.proxy.enable && cfg.proxy.subdomain == "")
|
||||
(throw "Service ${serviceName}: proxy.subdomain is required when proxy.enable is true"))
|
||||
|
||||
# Validate that if monitoring is enabled, required paths are set
|
||||
(lib.mkIf (cfg.monitoring.enable && cfg.monitoring.metricsPath == "")
|
||||
(throw "Service ${serviceName}: monitoring.metricsPath cannot be empty when monitoring is enabled"))
|
||||
];
|
||||
in {
|
||||
inherit mkServiceInterface mkServiceConfig validateServiceConfig;
|
||||
|
||||
# Common service option patterns
|
||||
commonOptions = {
|
||||
# Log level option
|
||||
logLevel = mkOption {
|
||||
type = types.enum ["debug" "info" "warn" "error"];
|
||||
default = "info";
|
||||
description = "Log level";
|
||||
};
|
||||
|
||||
# Environment file option (for secrets)
|
||||
environmentFile = mkOption {
|
||||
type = types.nullOr types.path;
|
||||
default = null;
|
||||
description = "Environment file for secrets";
|
||||
};
|
||||
|
||||
# External URL option
|
||||
externalUrl = serviceName: homelabCfg:
|
||||
mkOption {
|
||||
type = types.str;
|
||||
default = "https://${serviceName}.${homelabCfg.externalDomain}";
|
||||
description = "External URL for ${serviceName}";
|
||||
};
|
||||
};
|
||||
|
||||
# Helper for creating service modules with the interface
|
||||
mkServiceModule = {
|
||||
serviceName,
|
||||
defaultPort,
|
||||
defaultSubdomain ? serviceName,
|
||||
serviceOptions ? {},
|
||||
...
|
||||
} @ args: {
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
cfg = config.homelab.services.${serviceName};
|
||||
homelabCfg = config.homelab;
|
||||
|
||||
serviceInterface = mkServiceInterface {
|
||||
inherit serviceName defaultPort defaultSubdomain serviceOptions;
|
||||
};
|
||||
in {
|
||||
options.homelab.services.${serviceName} = serviceInterface;
|
||||
|
||||
config = mkServiceConfig {
|
||||
inherit config cfg homelabCfg serviceName;
|
||||
# Service implementor must provide this function
|
||||
serviceConfig = args.serviceConfig or (throw "mkServiceModule requires serviceConfig function");
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue