Make puppetdb startup timeout configurable

In some environments, puppetdb can take longer than 10 seconds
to start up.  Prior to this commit, that value was hard coded
and the module would sometimes fail when it wouldn't have failed
with a slightly larger timeout.  This commit makes the timeout
configurable, and also increases the default value to 15 seconds.

lib/puppet/provider/puppetdb_conn_validator/puppet_https.rb

@@ -26,7 +26,7 @@ def attempt_connection
     end
     return true
   rescue Errno::ECONNREFUSED => e
-    Puppet.warning "Unable to connect to puppetdb server (#{host}:#{port}): #{e.inspect} "
+    Puppet.notice "Unable to connect to puppetdb server (#{host}:#{port}): #{e.inspect} "
     return false
   end
 end
@@ -38,17 +38,26 @@ Puppet::Type.type(:puppetdb_conn_validator).provide(:puppet_https) do
         setup from the local puppet environment to authenticate."
 
   def exists?
+    start_time = Time.now
+    timeout = resource[:timeout]
+
     success = attempt_connection
     unless success
-      # It can take several seconds for the puppetdb server to start up;
-      # especially on the first install.  Therefore, our first connection attempt
-      # may fail.  Here we have somewhat arbitrarily chosen to retry one time
-      # after ten seconds if that situation arises.  May want to revisit this,
-      # but it seems to work OK for the common use case.
-      Puppet.notice("Failed to connect to puppetdb; sleeping 10 seconds before retry")
-      sleep 10
-      success = attempt_connection
+      while (Time.now - start_time) < timeout
+        # It can take several seconds for the puppetdb server to start up;
+        # especially on the first install.  Therefore, our first connection attempt
+        # may fail.  Here we have somewhat arbitrarily chosen to retry every 10
+        # seconds until the configurable timeout has expired.
+        Puppet.notice("Failed to connect to puppetdb; sleeping 2 seconds before retry")
+        sleep 2
+        success = attempt_connection
+      end
     end
+
+    unless success
+      Puppet.notice("Failed to connect to puppetdb within timeout window of #{timeout} seconds; giving up.")
+    end
+
     success
   end
 

lib/puppet/type/puppetdb_conn_validator.rb

@@ -23,4 +23,18 @@ Puppet::Type.newtype(:puppetdb_conn_validator) do
     desc 'The port that the puppetdb server should be listening on.'
   end
 
+  newparam(:timeout) do
+    desc 'The max number of seconds that the validator should wait before giving up and deciding that puppetdb is not running; defaults to 15 seconds.'
+    defaultto 15
+
+    validate do |value|
+      # This will raise an error if the string is not convertible to an integer
+      Integer(value)
+    end
+
+    munge do |value|
+      Integer(value)
+    end
+  end
+
 end

manifests/master/config.pp

@@ -23,6 +23,10 @@
 #                         be installed.  You may specify an explicit version
 #                         number, 'present', or 'latest'.  Defaults to
 #                         'present'.
+#   ['puppetdb_startup_timeout']  - The maximum amount of time that the module
+#                         should wait for puppetdb to start up; this is most
+#                         important during the initial install of puppetdb.
+#                         Defaults to 15 seconds.
 #   ['restart_puppet']  - If true, the module will restart the puppet master when
 #                         necessary.  The default is 'true'.  If set to 'false',
 #                         you must restart the service manually in order to pick
@@ -43,15 +47,16 @@
 # TODO: finish porting this to use params
 #
 class puppetdb::master::config(
-  $puppetdb_server      = $::clientcert,
-  $puppetdb_port        = 8081,
-  $manage_routes        = true,
-  $manage_storeconfigs  = true,
-  $puppet_confdir       = $puppetdb::params::puppet_confdir,
-  $puppet_conf          = $puppetdb::params::puppet_conf,
-  $puppetdb_version     = $puppetdb::params::puppetdb_version,
-  $terminus_package     = $puppetdb::params::terminus_package,
-  $puppet_service_name  = $puppetdb::params::puppet_service_name,
+  $puppetdb_server          = $::clientcert,
+  $puppetdb_port            = 8081,
+  $manage_routes            = true,
+  $manage_storeconfigs      = true,
+  $puppet_confdir           = $puppetdb::params::puppet_confdir,
+  $puppet_conf              = $puppetdb::params::puppet_conf,
+  $puppetdb_version         = $puppetdb::params::puppetdb_version,
+  $terminus_package         = $puppetdb::params::terminus_package,
+  $puppet_service_name      = $puppetdb::params::puppet_service_name,
+  $puppetdb_startup_timeout = $puppetdb::params::puppetdb_startup_timeout,
   $restart_puppet       = true,
 ) inherits puppetdb::params {
 
@@ -64,6 +69,7 @@ class puppetdb::master::config(
   puppetdb_conn_validator { 'puppetdb_conn':
     puppetdb_server => $puppetdb_server,
     puppetdb_port   => $puppetdb_port,
+    timeout         => $puppetdb_startup_timeout,
     require         => Package[$terminus_package],
   }
 

manifests/params.pp

@@ -62,5 +62,6 @@ class puppetdb::params {
     $terminus_package     = 'puppetdb-terminus'
   }
 
-  $puppet_conf          = "${puppet_confdir}/puppet.conf"
+  $puppet_conf              = "${puppet_confdir}/puppet.conf"
+  $puppetdb_startup_timeout = 15
 }