From dd4fa3b28dd8e690397c4d2b5e7cefb60bf5294a Mon Sep 17 00:00:00 2001 From: RGBCube Date: Thu, 27 Feb 2025 23:40:34 +0300 Subject: [PATCH] feat: move postgres to shared modules --- hosts/best/postgresql.nix | 66 +++++++++++++ hosts/cube/forgejo/default.nix | 5 +- hosts/cube/grafana/default.nix | 5 +- hosts/cube/matrix/default.nix | 7 +- hosts/cube/nextcloud/default.nix | 5 +- hosts/cube/postgresql.nix | 163 ++++++++++--------------------- modules/postgresql.nix | 61 ++++++++++++ 7 files changed, 197 insertions(+), 115 deletions(-) create mode 100644 hosts/best/postgresql.nix create mode 100644 modules/postgresql.nix diff --git a/hosts/best/postgresql.nix b/hosts/best/postgresql.nix new file mode 100644 index 0000000..e3f8ba3 --- /dev/null +++ b/hosts/best/postgresql.nix @@ -0,0 +1,66 @@ +{ self, ... }: { + imports = [(self + /modules/postgresql.nix)]; + + # Generated by + services.postgresql.settings = { + # Connectivity + max_connections = 100; + superuser_reserved_connections = 3; + + # Memory Settings + shared_buffers = "10240 MB"; + work_mem = "64 MB"; + maintenance_work_mem = "620 MB"; + huge_pages = "try"; # TODO: Tune kernel: + effective_cache_size = "45 GB"; + effective_io_concurrency = 100; + random_page_cost = 1.25; + + # Monitoring + shared_preload_libraries = "pg_stat_statements"; + track_io_timing = "on"; + track_functions = "pl"; + + # Replication + wal_level = "replica"; + max_wal_senders = 10; + synchronous_commit = "on"; + + # Checkpointing: + checkpoint_timeout = "15 min"; + checkpoint_completion_target = 0.9; + max_wal_size = "1024 MB"; + min_wal_size = "512 MB"; + + # WAL writing + wal_compression = "on"; + wal_buffers = -1; + wal_writer_delay = "200ms"; + wal_writer_flush_after = "1MB"; + wal_keep_size = "3650 MB"; + + + # Background writer + bgwriter_delay = "200ms"; + bgwriter_lru_maxpages = 100; + bgwriter_lru_multiplier = 2.0; + bgwriter_flush_after = 0; + + # Parallel queries: + max_worker_processes = 20; + max_parallel_workers_per_gather = 10; + max_parallel_maintenance_workers = 10; + max_parallel_workers = 20; + parallel_leader_participation = "on"; + + # Advanced features + enable_partitionwise_join = "on"; + enable_partitionwise_aggregate = "on"; + jit = "on"; + max_slot_wal_keep_size = "1000 MB"; + track_wal_io_timing = "on"; + maintenance_io_concurrency = 100; + wal_recycle = "on"; + }; +} + diff --git a/hosts/cube/forgejo/default.nix b/hosts/cube/forgejo/default.nix index d1646ec..58b165b 100644 --- a/hosts/cube/forgejo/default.nix +++ b/hosts/cube/forgejo/default.nix @@ -5,7 +5,10 @@ fqdn = "git.${domain}"; port = 8001; in { - imports = [(self + /modules/nginx.nix)]; + imports = [ + (self + /modules/nginx.nix) + (self + /modules/postgresql.nix) + ]; secrets.forgejoPasswordRunner = { file = ./password.runner.age; diff --git a/hosts/cube/grafana/default.nix b/hosts/cube/grafana/default.nix index 4fce230..eded4c8 100644 --- a/hosts/cube/grafana/default.nix +++ b/hosts/cube/grafana/default.nix @@ -5,7 +5,10 @@ fqdn = "metrics.${domain}"; port = 8000; in { - imports = [(self + /modules/nginx.nix)]; + imports = [ + (self + /modules/nginx.nix) + (self + /modules/postgresql.nix) + ]; secrets.grafanaPassword = { file = ./password.age; diff --git a/hosts/cube/matrix/default.nix b/hosts/cube/matrix/default.nix index 71a767f..21a7b68 100644 --- a/hosts/cube/matrix/default.nix +++ b/hosts/cube/matrix/default.nix @@ -23,13 +23,18 @@ }; }; in { - imports = [(self + /modules/nginx.nix)]; + imports = [ + (self + /modules/nginx.nix) + (self + /modules/postgresql.nix) + ]; secrets.matrixSecret = { file = ./password.secret.age; owner = "matrix-synapse"; }; + services.postgresql.ensure = [ "matrix-synapse" ]; + services.restic.backups = genAttrs config.services.restic.hosts <| const { paths = [ "/var/lib/matrix-synapse" ]; }; diff --git a/hosts/cube/nextcloud/default.nix b/hosts/cube/nextcloud/default.nix index 29ecfb5..4220e38 100644 --- a/hosts/cube/nextcloud/default.nix +++ b/hosts/cube/nextcloud/default.nix @@ -6,7 +6,10 @@ packageNextcloud = pkgs.nextcloud30; in { - imports = [(self + /modules/nginx.nix)]; + imports = [ + (self + /modules/nginx.nix) + (self + /modules/postgresql.nix) + ]; secrets.nextcloudPassword = { file = ./password.age; diff --git a/hosts/cube/postgresql.nix b/hosts/cube/postgresql.nix index 7832909..80b63bf 100644 --- a/hosts/cube/postgresql.nix +++ b/hosts/cube/postgresql.nix @@ -1,122 +1,63 @@ -{ config, lib, pkgs, ... }: let - inherit (lib) const enabled flip genAttrs mkForce mkOverride mkValue; -in { - config.services.prometheus.exporters.postgres = enabled { - listenAddress = "[::]"; - runAsLocalSuperUser = true; - }; +{ self, ... }: { + imports = [(self + /modules/postgresql.nix)]; - config.services.restic.backups = genAttrs config.services.restic.hosts <| const { - paths = [ "/tmp/postgresql-dump.sql.gz" ]; + # Generated by + services.postgresql.settings = { + # Connectivity + max_connections = 20; + superuser_reserved_connections = 3; - backupPrepareCommand = /* sh */ '' - ${config.services.postgresql.package}/bin/pg_dumpall --clean \ - | ${lib.getExe pkgs.gzip} --rsyncable \ - > /tmp/postgresql-dump.sql.gz - ''; + # Memory Settings + shared_buffers = "1024 MB"; + work_mem = "32 MB"; + maintenance_work_mem = "320 MB"; + huge_pages = "off"; + effective_cache_size = "3 GB"; + effective_io_concurrency = 1; + random_page_cost = 4; - backupCleanupCommand = /* sh */ '' - rm /tmp/postgresql-dump.sql.gz - ''; - }; + # Monitoring + shared_preload_libraries = "pg_stat_statements"; + track_io_timing = "on"; + track_functions = "pl"; - options.services.postgresql.ensure = mkValue []; + # Replication + wal_level = "replica"; + max_wal_senders = 0; + synchronous_commit = "on"; - config.environment.systemPackages = [ - config.services.postgresql.package - ]; + # Checkpointing: + checkpoint_timeout = "15 min"; + checkpoint_completion_target = 0.9; + max_wal_size = "1024 MB"; + min_wal_size = "512 MB"; - config.services.postgresql = enabled { - package = pkgs.postgresql_17; + # WAL writing + wal_compression = "on"; + wal_buffers = -1; + wal_writer_delay = "200ms"; + wal_writer_flush_after = "1MB"; - enableJIT = true; + # Background writer + bgwriter_delay = "200ms"; + bgwriter_lru_maxpages = 100; + bgwriter_lru_multiplier = 2.0; + bgwriter_flush_after = 0; - initdbArgs = [ "--locale=C" "--encoding=UTF8" ]; + # Parallel queries: + max_worker_processes = 2; + max_parallel_workers_per_gather = 1; + max_parallel_maintenance_workers = 1; + max_parallel_workers = 2; + parallel_leader_participation = "on"; - authentication = mkOverride 10 /* ini */ '' - # DATABASE USER AUTHENTICATION - local all all peer - - # DATABASE USER ADDRESS AUTHENTICATION - host all all ::/0 md5 - ''; - - ensure = [ "postgres" "root" ]; - - ensureDatabases = config.services.postgresql.ensure; - - ensureUsers = flip map config.services.postgresql.ensure (name: { - inherit name; - - ensureDBOwnership = true; - - ensureClauses = { - login = true; - superuser = name == "postgres" || name == "root"; - }; - }); - - settings = { - listen_addresses = mkForce "::"; - - # Generated by - - # Connectivity - max_connections = 20; - superuser_reserved_connections = 3; - - # Memory Settings - shared_buffers = "1024 MB"; - work_mem = "32 MB"; - maintenance_work_mem = "320 MB"; - huge_pages = "off"; - effective_cache_size = "3 GB"; - effective_io_concurrency = 1; - random_page_cost = 4; - - # Monitoring - shared_preload_libraries = "pg_stat_statements"; - track_io_timing = "on"; - track_functions = "pl"; - - # Replication - wal_level = "replica"; - max_wal_senders = 0; - synchronous_commit = "on"; - - # Checkpointing: - checkpoint_timeout = "15 min"; - checkpoint_completion_target = 0.9; - max_wal_size = "1024 MB"; - min_wal_size = "512 MB"; - - # WAL writing - wal_compression = "on"; - wal_buffers = -1; - wal_writer_delay = "200ms"; - wal_writer_flush_after = "1MB"; - - # Background writer - bgwriter_delay = "200ms"; - bgwriter_lru_maxpages = 100; - bgwriter_lru_multiplier = 2.0; - bgwriter_flush_after = 0; - - # Parallel queries: - max_worker_processes = 2; - max_parallel_workers_per_gather = 1; - max_parallel_maintenance_workers = 1; - max_parallel_workers = 2; - parallel_leader_participation = "on"; - - # Advanced features - enable_partitionwise_join = "on" ; - enable_partitionwise_aggregate = "on"; - jit = "on"; - max_slot_wal_keep_size = "1000 MB"; - track_wal_io_timing = "on"; - maintenance_io_concurrency = 1; - wal_recycle = "on"; - }; + # Advanced features + enable_partitionwise_join = "on" ; + enable_partitionwise_aggregate = "on"; + jit = "on"; + max_slot_wal_keep_size = "1000 MB"; + track_wal_io_timing = "on"; + maintenance_io_concurrency = 1; + wal_recycle = "on"; }; } diff --git a/modules/postgresql.nix b/modules/postgresql.nix new file mode 100644 index 0000000..65829dc --- /dev/null +++ b/modules/postgresql.nix @@ -0,0 +1,61 @@ +{ config, lib, pkgs, ... }: let + inherit (lib) const enabled flip genAttrs mkForce mkOverride mkValue; +in { + config.services.prometheus.exporters.postgres = enabled { + listenAddress = "[::]"; + runAsLocalSuperUser = true; + }; + + config.services.restic.backups = genAttrs config.services.restic.hosts <| const { + paths = [ "/tmp/postgresql-dump.sql.gz" ]; + + backupPrepareCommand = /* sh */ '' + ${config.services.postgresql.package}/bin/pg_dumpall --clean \ + | ${lib.getExe pkgs.gzip} --rsyncable \ + > /tmp/postgresql-dump.sql.gz + ''; + + backupCleanupCommand = /* sh */ '' + rm /tmp/postgresql-dump.sql.gz + ''; + }; + + config.environment.systemPackages = [ + config.services.postgresql.package + ]; + + options.services.postgresql.ensure = mkValue []; + + config.services.postgresql = enabled { + package = pkgs.postgresql_17; + + enableJIT = true; + enableTCPIP = true; # We override it, but might as well. + + settings.listen_addresses = mkForce "::"; + authentication = mkOverride 10 /* ini */ '' + # DATABASE USER AUTHENTICATION + local all all peer + + # DATABASE USER ADDRESS AUTHENTICATION + host all all ::/0 md5 + ''; + + ensure = [ "postgres" "root" ]; + + initdbArgs = [ "--locale=C" "--encoding=UTF8" ]; + ensureDatabases = config.services.postgresql.ensure; + + ensureUsers = flip map config.services.postgresql.ensure (name: { + inherit name; + + ensureDBOwnership = true; + + ensureClauses = { + login = true; + superuser = name == "postgres" || name == "root"; + }; + }); + }; +} +