From 27840d695983e6d10d0d7849ba399bf3b261adfb Mon Sep 17 00:00:00 2001 From: Taylan Develioglu Date: Mon, 4 Aug 2014 13:06:07 +0200 Subject: [PATCH 1/2] Add read-database support --- README.md | 55 ++++++++++++ manifests/init.pp | 21 +++++ manifests/params.pp | 13 +++ manifests/server.pp | 32 +++++++ manifests/server/database_ini.pp | 5 ++ manifests/server/read_database_ini.pp | 117 ++++++++++++++++++++++++++ manifests/server/validate_read_db.pp | 24 ++++++ 7 files changed, 267 insertions(+) create mode 100644 manifests/server/read_database_ini.pp create mode 100644 manifests/server/validate_read_db.pp diff --git a/README.md b/README.md index cb35e59..7f23bc9 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,61 @@ Example: to set `-Xmx512m -Xms256m` options use `{ '-Xmx' => '512m', '-Xms' => ' Jetty option to explicetly set max-thread. The default is undef, so the PuppetDB-jetty default is used. +####`read_database` + +Which database backend to use for the read database; Currently only supports `postgres` (default). + +####`read_database_host` +*This parameter must be set to enable the puppetdb read-database.* + +The hostname or IP address of the read database server (defaults to `undef`). +The default is to use the regular database for reads and writes. + +####`read_database_port` + +The port that the read database server listens on (defaults to `5432`). + +####`read_database_username` + +The name of the read database user to connect as (defaults to `puppetdb`). + +####`read_database_password` + +The password for the read database user (defaults to `puppetdb`). + +####`read_database_name` + +The name of the read database instance to connect to (defaults to `puppetdb`). + +####`read_database_ssl` + +If true, puppetdb will use SSL to connect to the postgres read database (defaults to false). +Setting up proper trust- and keystores has to be managed outside of the puppetdb module. + +####`read_log_slow_statements` + +This sets the number of seconds before an SQL query to the read database is considered "slow." Slow SQL queries are logged as warnings, to assist in debugging and tuning. Note PuppetDB does not interrupt slow queries; it simply reports them after they complete. + +The default value is 10 seconds. A value of 0 will disable logging of slow queries. This option is supported in PuppetDB >= 1.1. + +####`read_conn_max_age` + +The maximum time (in minutes), for a pooled read database connection to remain unused before it is closed off. + +If not supplied, we default to 60 minutes. This option is supported in PuppetDB >= 1.1. + +####`read_conn_keep_alive` + +This sets the time (in minutes), for a read database connection to remain idle before sending a test query to the DB. This is useful to prevent a DB from timing out connections on its end. + +If not supplied, we default to 45 minutes. This option is supported in PuppetDB >= 1.1. + +####`read_conn_lifetime` + +The maximum time (in minutes) a pooled read database connection should remain open. Any connections older than this setting will be closed off. Connections currently in use will not be affected until they are returned to the pool. + +If not supplied, we won't terminate connections based on their age alone. This option is supported in PuppetDB >= 1.4. + ### puppetdb:server diff --git a/manifests/init.pp b/manifests/init.pp index 82ab9ff..3e4c28f 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -28,6 +28,16 @@ class puppetdb( $puppetdb_version = $puppetdb::params::puppetdb_version, $puppetdb_service = $puppetdb::params::puppetdb_service, $puppetdb_service_status = $puppetdb::params::puppetdb_service_status, + $read_database = $puppetdb::params::read_database, + $read_database_port = $puppetdb::params::read_database_port, + $read_database_username = $puppetdb::params::read_database_username, + $read_database_password = $puppetdb::params::read_database_password, + $read_database_name = $puppetdb::params::read_database_name, + $read_database_ssl = $puppetdb::params::read_database_ssl, + $read_log_slow_statements = $puppetdb::params::read_log_slow_statements, + $read_conn_max_age = $puppetdb::params::read_conn_max_age, + $read_conn_keep_alive = $puppetdb::params::read_conn_keep_alive, + $read_conn_lifetime = $puppetdb::params::read_conn_lifetime, $open_postgres_port = $puppetdb::params::open_postgres_port, $confdir = $puppetdb::params::confdir, $java_args = {}, @@ -98,6 +108,17 @@ class puppetdb( confdir => $confdir, java_args => $java_args, max_threads => $max_threads, + read_database => $puppetdb::params::read_database, + read_database_host => $puppetdb::params::read_database_host, + read_database_port => $puppetdb::params::read_database_port, + read_database_username => $puppetdb::params::read_database_username, + read_database_password => $puppetdb::params::read_database_password, + read_database_name => $puppetdb::params::read_database_name, + read_database_ssl => $puppetdb::params::read_database_ssl, + read_log_slow_statements => $puppetdb::params::read_log_slow_statements, + read_conn_max_age => $puppetdb::params::read_conn_max_age, + read_conn_keep_alive => $puppetdb::params::read_conn_keep_alive, + read_conn_lifetime => $puppetdb::params::read_conn_lifetime, } if ($database == 'postgres') { diff --git a/manifests/params.pp b/manifests/params.pp index 58588d2..0a0010a 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -37,6 +37,19 @@ class puppetdb::params { $max_threads = undef + # These settings are for the read database + $read_database = 'postgres' + $read_database_host = undef + $read_database_port = '5432' + $read_database_name = 'puppetdb' + $read_database_username = 'puppetdb' + $read_database_password = 'puppetdb' + $read_database_ssl = false + $read_log_slow_statements = '10' + $read_conn_max_age = '60' + $read_conn_keep_alive = '45' + $read_conn_lifetime = '0' + case $::osfamily { 'RedHat': { $firewall_supported = true diff --git a/manifests/server.pp b/manifests/server.pp index 25f88f0..a89a604 100644 --- a/manifests/server.pp +++ b/manifests/server.pp @@ -22,6 +22,17 @@ class puppetdb::server( $conn_max_age = $puppetdb::params::conn_max_age, $conn_keep_alive = $puppetdb::params::conn_keep_alive, $conn_lifetime = $puppetdb::params::conn_lifetime, + $read_database = $puppetdb::params::read_database, + $read_database_host = $puppetdb::params::read_database_host, + $read_database_port = $puppetdb::params::read_database_port, + $read_database_username = $puppetdb::params::read_database_username, + $read_database_password = $puppetdb::params::read_database_password, + $read_database_name = $puppetdb::params::read_database_name, + $read_database_ssl = $puppetdb::params::read_database_ssl, + $read_log_slow_statements = $puppetdb::params::read_log_slow_statements, + $read_conn_max_age = $puppetdb::params::read_conn_max_age, + $read_conn_keep_alive = $puppetdb::params::read_conn_keep_alive, + $read_conn_lifetime = $puppetdb::params::read_conn_lifetime, $puppetdb_package = $puppetdb::params::puppetdb_package, $puppetdb_version = $puppetdb::params::puppetdb_version, $puppetdb_service = $puppetdb::params::puppetdb_service, @@ -67,6 +78,11 @@ class puppetdb::server( fail("puppetdb_service_status valid values are 'true', 'running', 'false', and 'stopped'. You provided '${puppetdb_service_status}'") } + # Validate read-database type (Currently only postgres is supported) + if !($read_database in ['postgres']) { + fail("read_database must be 'postgres'. You provided '${read_database}'") + } + package { $puppetdb_package: ensure => $puppetdb_version, notify => Service[$puppetdb_service], @@ -102,6 +118,22 @@ class puppetdb::server( notify => Service[$puppetdb_service], } + class { 'puppetdb::server::read_database_ini': + database => $read_database, + database_host => $read_database_host, + database_port => $read_database_port, + database_username => $read_database_username, + database_password => $read_database_password, + database_name => $read_database_name, + database_ssl => $read_database_ssl, + log_slow_statements => $read_log_slow_statements, + conn_max_age => $read_conn_max_age, + conn_keep_alive => $read_conn_keep_alive, + conn_lifetime => $read_conn_lifetime, + confdir => $confdir, + notify => Service[$puppetdb_service], + } + class { 'puppetdb::server::jetty_ini': listen_address => $listen_address, listen_port => $listen_port, diff --git a/manifests/server/database_ini.pp b/manifests/server/database_ini.pp index 876437c..f780169 100644 --- a/manifests/server/database_ini.pp +++ b/manifests/server/database_ini.pp @@ -21,6 +21,11 @@ class puppetdb::server::database_ini( # Validate the database connection. If we can't connect, we want to fail # and skip the rest of the configuration, so that we don't leave puppetdb # in a broken state. + # + # NOTE: + # Because of a limitation in the postgres module this will break with + # a duplicate declaration if read and write database host+name are the + # same. class { 'puppetdb::server::validate_db': database => $database, database_host => $database_host, diff --git a/manifests/server/read_database_ini.pp b/manifests/server/read_database_ini.pp new file mode 100644 index 0000000..9ddf10d --- /dev/null +++ b/manifests/server/read_database_ini.pp @@ -0,0 +1,117 @@ +# PRIVATE CLASS - do not use directly +class puppetdb::server::read_database_ini( + $database = $puppetdb::params::read_database, + $database_host = $puppetdb::params::read_database_host, + $database_port = $puppetdb::params::read_database_port, + $database_username = $puppetdb::params::read_database_username, + $database_password = $puppetdb::params::read_database_password, + $database_name = $puppetdb::params::read_database_name, + $database_ssl = $puppetdb::params::read_database_ssl, + $log_slow_statements = $puppetdb::params::read_log_slow_statements, + $conn_max_age = $puppetdb::params::read_conn_max_age, + $conn_keep_alive = $puppetdb::params::read_conn_keep_alive, + $conn_lifetime = $puppetdb::params::read_conn_lifetime, + $confdir = $puppetdb::params::confdir, +) inherits puppetdb::params { + + # Only add the read database configuration if database host is defined. + if $database_host != undef { + # Validate the database connection. If we can't connect, we want to fail + # and skip the rest of the configuration, so that we don't leave puppetdb + # in a broken state. + # + # NOTE: + # Because of a limitation in the postgres module this will break with + # a duplicate declaration if read and write database host+name are the + # same. + class { 'puppetdb::server::validate_read_db': + database => $database, + database_host => $database_host, + database_port => $database_port, + database_username => $database_username, + database_password => $database_password, + database_name => $database_name, + } + + file { "${confdir}/read_database.ini": + ensure => file, + owner => 'puppetdb', + group => 'puppetdb', + mode => '0600'; + } + + #Set the defaults + Ini_setting { + path => "${confdir}/read_database.ini", + ensure => present, + section => 'read-database', + require => Class['puppetdb::server::validate_db'], + } + + if $database == 'postgres' { + $classname = 'org.postgresql.Driver' + $subprotocol = 'postgresql' + + $subname = $database_ssl ? { + true => "//${database_host}:${database_port}/${database_name}?ssl=true", + default => "//${database_host}:${database_port}/${database_name}", + } + + ini_setting {'puppetdb_read_psdatabase_username': + setting => 'username', + value => $database_username, + } + + if $database_password != undef { + ini_setting {'puppetdb_read_psdatabase_password': + setting => 'password', + value => $database_password, + } + } + } + + ini_setting {'puppetdb_read_classname': + setting => 'classname', + value => $classname, + } + + ini_setting {'puppetdb_read_subprotocol': + setting => 'subprotocol', + value => $subprotocol, + } + + ini_setting {'puppetdb_read_pgs': + setting => 'syntax_pgs', + value => true, + } + + ini_setting {'puppetdb_read_subname': + setting => 'subname', + value => $subname, + } + + ini_setting {'puppetdb_read_log_slow_statements': + setting => 'log-slow-statements', + value => $log_slow_statements, + } + + ini_setting {'puppetdb_read_conn_max_age': + setting => 'conn-max-age', + value => $conn_max_age, + } + + ini_setting {'puppetdb_read_conn_keep_alive': + setting => 'conn-keep-alive', + value => $conn_keep_alive, + } + + ini_setting {'puppetdb_read_conn_lifetime': + setting => 'conn-lifetime', + value => $conn_lifetime, + } + } else { + file { "${confdir}/read_database.ini": + ensure => absent, + } + } +} diff --git a/manifests/server/validate_read_db.pp b/manifests/server/validate_read_db.pp new file mode 100644 index 0000000..01d5da3 --- /dev/null +++ b/manifests/server/validate_read_db.pp @@ -0,0 +1,24 @@ +# This validates a database connection. See README.md for more details. +class puppetdb::server::validate_read_db( + $database = $puppetdb::params::database, + $database_host = $puppetdb::params::database_host, + $database_port = $puppetdb::params::database_port, + $database_username = $puppetdb::params::database_username, + $database_password = $puppetdb::params::database_password, + $database_name = $puppetdb::params::database_name, + $database_ssl = $puppetdb::params::database_ssl +) inherits puppetdb::params { + + # Currently we only support postgres + if ($database == 'postgres' and ( + $database_password != undef and $database_ssl == false) + ) { + postgresql::validate_db_connection { 'validate puppetdb postgres (read) connection': + database_host => $database_host, + database_port => $database_port, + database_username => $database_username, + database_password => $database_password, + database_name => $database_name, + } + } +} From 9a3055281b700dbd034accf4f4fba734b931a0f2 Mon Sep 17 00:00:00 2001 From: Taylan Develioglu Date: Mon, 4 Aug 2014 14:02:40 +0200 Subject: [PATCH 2/2] Mention puppetdb version in read-database parameters. --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7f23bc9..c776725 100644 --- a/README.md +++ b/README.md @@ -303,58 +303,58 @@ Jetty option to explicetly set max-thread. The default is undef, so the PuppetDB ####`read_database` -Which database backend to use for the read database; Currently only supports `postgres` (default). +Which database backend to use for the read database; Currently only supports `postgres` (default). This option is supported in PuppetDB >= 1.6. ####`read_database_host` *This parameter must be set to enable the puppetdb read-database.* The hostname or IP address of the read database server (defaults to `undef`). -The default is to use the regular database for reads and writes. +The default is to use the regular database for reads and writes. This option is supported in PuppetDB >= 1.6. ####`read_database_port` -The port that the read database server listens on (defaults to `5432`). +The port that the read database server listens on (defaults to `5432`). This option is supported in PuppetDB >= 1.6. ####`read_database_username` -The name of the read database user to connect as (defaults to `puppetdb`). +The name of the read database user to connect as (defaults to `puppetdb`). This option is supported in PuppetDB >= 1.6. ####`read_database_password` -The password for the read database user (defaults to `puppetdb`). +The password for the read database user (defaults to `puppetdb`). This option is supported in PuppetDB >= 1.6. ####`read_database_name` -The name of the read database instance to connect to (defaults to `puppetdb`). +The name of the read database instance to connect to (defaults to `puppetdb`). This option is supported in PuppetDB >= 1.6. ####`read_database_ssl` If true, puppetdb will use SSL to connect to the postgres read database (defaults to false). -Setting up proper trust- and keystores has to be managed outside of the puppetdb module. +Setting up proper trust- and keystores has to be managed outside of the puppetdb module. This option is supported in PuppetDB >= 1.6. ####`read_log_slow_statements` This sets the number of seconds before an SQL query to the read database is considered "slow." Slow SQL queries are logged as warnings, to assist in debugging and tuning. Note PuppetDB does not interrupt slow queries; it simply reports them after they complete. -The default value is 10 seconds. A value of 0 will disable logging of slow queries. This option is supported in PuppetDB >= 1.1. +The default value is 10 seconds. A value of 0 will disable logging of slow queries. This option is supported in PuppetDB >= 1.6. ####`read_conn_max_age` The maximum time (in minutes), for a pooled read database connection to remain unused before it is closed off. -If not supplied, we default to 60 minutes. This option is supported in PuppetDB >= 1.1. +If not supplied, we default to 60 minutes. This option is supported in PuppetDB >= 1.6. ####`read_conn_keep_alive` This sets the time (in minutes), for a read database connection to remain idle before sending a test query to the DB. This is useful to prevent a DB from timing out connections on its end. -If not supplied, we default to 45 minutes. This option is supported in PuppetDB >= 1.1. +If not supplied, we default to 45 minutes. This option is supported in PuppetDB >= 1.6. ####`read_conn_lifetime` The maximum time (in minutes) a pooled read database connection should remain open. Any connections older than this setting will be closed off. Connections currently in use will not be affected until they are returned to the pool. -If not supplied, we won't terminate connections based on their age alone. This option is supported in PuppetDB >= 1.4. +If not supplied, we won't terminate connections based on their age alone. This option is supported in PuppetDB >= 1.6. ### puppetdb:server