Implementata richiesta dei certificati per le url e fix permessi

OTcerts.py

@@ -1,4 +1,5 @@
 
+
 import os
 import sys
 import errno
@@ -7,6 +8,9 @@ import configparser
 import logging
 import mysql.connector
 import subprocess
+from pwd import getpwnam
+from grp import getgrnam
+
 
 # Query for IMAP/POP3 certificate
 mbox_list_stmt = "SELECT DISTINCT(name) FROM records WHERE content in ({}) and (name LIKE 'imap.%' or name LIKE 'pop3.%' or name LIKE 'mail.%')"
@@ -16,9 +20,18 @@ smtp_list_stmt = "SELECT DISTINCT(name) FROM records WHERE content in ({}) and (
 
 
 # Get list of defined domains in vhosts configuration database
-domains_list_stmt = """SELECT DISTINCT(SUBSTRING_INDEX(urls.dns_name, '.', -2)) AS domain_names 
+domains_list_stmt = """SELECT DISTINCT(SUBSTRING_INDEX(urls.dns_name, '.', -2)) AS domain_names
-FROM urls INNER JOIN (hosts_urls, hosts, vhosts_features, vhosts) 
+FROM hosts INNER JOIN (hosts_urls, urls, vhosts_features, vhosts)
-ON (urls.url_id = hosts_urls.url_id and urls.url_id = vhosts.url_id and vhosts.vhost_id = vhosts_features.vhost_id) 
+ON (urls.url_id = hosts_urls.url_id and urls.url_id = vhosts.url_id 
+and vhosts.vhost_id = vhosts_features.vhost_id  and hosts.host_id = hosts_urls.host_id) 
+WHERE (hosts_urls.http = 'Y' and hosts.hostname =  %(webserver)s)
+"""
+
+# Get list of defined urls for specific webserver
+urls_list_stmt = """SELECT DISTINCT(urls.dns_name) AS urls
+FROM hosts INNER JOIN (hosts_urls, urls, vhosts_features, vhosts)
+ON (urls.url_id = hosts_urls.url_id and urls.url_id = vhosts.url_id 
+and vhosts.vhost_id = vhosts_features.vhost_id  and hosts.host_id = hosts_urls.host_id) 
 WHERE (hosts_urls.http = 'Y' and hosts.hostname =  %(webserver)s)
 """
 
@@ -47,18 +60,23 @@ def init_prog(argv):
                         required=False,
                         default=default_conf_file,
                         help="Specifity config file (default: {})".format(default_conf_file))
-    parser.add_argument("--liste", default=False, action='store_true', required=False,
-                        help="Richiedi i certificati per liste.indivia.net")
-    parser.add_argument("--hosting", default=False, action='store_true', required=False,
-                        help="Richiedi i certificati per i siti in hosting")
-    parser.add_argument("--webmail", default=False, action='store_true', required=False,
-                        help="Richiedi i certificati per le webmail")
-    parser.add_argument("--smtp", default=False, action='store_true', required=False,
-                        help="Richiedi i certificati per il server SMTP")
-    parser.add_argument("--mbox", default=False, action='store_true', required=False,
-                        help="Richiedi i certificati per il server POP/IMAP")
     parser.add_argument("--renew", default=False, action='store_true', required=False,
                         help="Invoca solamente il renew per i certificati gia' presenti")
+
+    service_group = parser.add_mutually_exclusive_group(required=True)
+    service_group.add_argument("--proxy", default=False, action='store_true', required=False,
+                        help="Richiedi i certificati per i siti proxaty")
+    service_group.add_argument("--liste", default=False, action='store_true', required=False,
+                        help="Richiedi i certificati per liste.indivia.net")
+    service_group.add_argument("--hosting", default=False, action='store_true', required=False,
+                        help="Richiedi i certificati per i siti in hosting")
+    service_group.add_argument("--webmail", default=False, action='store_true', required=False,
+                        help="Richiedi i certificati per le webmail")
+    service_group.add_argument("--smtp", default=False, action='store_true', required=False,
+                        help="Richiedi i certificati per il server SMTP")
+    service_group.add_argument("--mbox", default=False, action='store_true', required=False,
+                        help="Richiedi i certificati per il server POP/IMAP")
+
     args = parser.parse_args()
     try:
         config = configparser.ConfigParser()
@@ -87,7 +105,7 @@ def connect_db(conf_dict):
 
 def get_subdomain_list(config,  domain, ot_conn, ex_subdomains=tuple()):
     """
-    Return a Python list containing subdomain of domain paramer
+    Return a Python list containing subdomain of domain parameter
     eg:  ['app.arkiwi.org']
     """
     result_dict=dict()
@@ -100,8 +118,12 @@ def get_subdomain_list(config,  domain, ot_conn, ex_subdomains=tuple()):
     subdomains_res = ot_cursor.fetchall()
     ot_cursor.close()
 
-    subdomains_filtered = [s[0].decode('utf-8') for s in subdomains_res
+    try:
+        subdomains_filtered = [s[0].decode('utf-8') for s in subdomains_res
                                if not(s[0].decode('utf-8').startswith(ex_subdomains))]
+    except AttributeError:
+        subdomains_filtered = [s[0] for s in subdomains_res
+                               if not(s[0].startswith(ex_subdomains))]
 
     return subdomains_filtered
 
@@ -119,7 +141,11 @@ def get_domain_list(config,  ot_conn, dns_conn):
 
     dns_cursor=dns_conn.cursor()
     for domain_barr, in ot_cursor:
-        domain_name = domain_barr.decode("utf-8")
+        try:
+            domain_name = domain_barr.decode("utf-8")
+        except AttributeError:
+            domain_name = domain_barr;
+        logger.debug(domain_barr)
         try:
             dns_cursor.execute(domain_id_stmt, {'domain':domain_name})
         except Exception as e:
@@ -137,6 +163,30 @@ def get_domain_list(config,  ot_conn, dns_conn):
     ot_cursor.close()
     return result_dict
 
+
+def get_url_list(config_section, server_name, ot_conn, dns_conn):
+    """
+    Ritorna la lista delle url configurate per uno specifico server_name
+    NB: il questo momento il dato viene estratto dal db di ortiche, ma non viene
+    controllato se il dns e' configurato in maniera coerente. Questo potrebbe generare
+    errori in momenti successivi (es, durante il challenge HTTP-01)
+    """
+
+    urls_list = []
+
+    ot_cursor=ot_conn.cursor()
+    ot_cursor.execute(urls_list_stmt, {'webserver':server_name})
+
+    ot_res = ot_cursor.fetchall()
+
+    logger.debug(ot_res)
+
+    urls_list = [t[0] for t in ot_res]
+
+    ot_cursor.close()
+    return urls_list
+
+
 def  get_alias_list(config, dns_conn, query, aliases):
     """
     Return a list of domains to get the certificate for
@@ -156,6 +206,33 @@ def  get_alias_list(config, dns_conn, query, aliases):
     return result_list
 
 
+def acme_renew(config, pre_hook_cmd, post_hook_cmd, dryrun=False):
+
+    args = config['certbot']['base_args']
+    # args += " -m {} ".format(config['certbot']['email'])
+    # args += "--server {} ".format(config['certbot']['server'])
+
+
+    if dryrun:
+        args += "--dry-run "
+
+    if not pre_hook_cmd is None:
+        args +=' --pre-hook "{}"'.format(pre_hook_cmd)
+
+    if not post_hook_cmd is None:
+        args +=' --post-hook "{}"'.format(post_hook_cmd)
+
+    args += " renew"
+
+    if dryrun:
+        logging.info("{} {}".format(config['certbot']['bin'], args))
+    else:
+        os.system("{} {}".format(config['certbot']['bin'], args))
+
+    return True
+
+
+
 def acme_request(config, domain_name, acme_test='DNS-01', webroot=None, dryrun=False, domains_list=None):
 
     args = config['certbot']['base_args']
@@ -213,127 +290,221 @@ def link_cert(config, source, dest, dryrun=False):
         symlink_force(src_name, link_name)
 
 
+def fix_permissions(config):
+    """
+    Sistema i permessi dei certificati affinche' risultino leggibili dai demoni interessati
+
+    """
+    archive_dir = config['certbot']['archive_certificates_dir']
+    uid = getpwnam(config['certbot']['certificates_user'])[2]
+    gid = getgrnam(config['certbot']['certificates_group'])[2]
+    for root, dirs, files in os.walk(archive_dir):
+        for momo in dirs:
+            logger.debug('Fixing user/group and permissions on {}'.format(os.path.join(root, momo)))
+            os.chown(os.path.join(root, momo), uid, gid)
+            os.chmod(os.path.join(root, momo), 0o755)
+
+        for momo in files:
+            logger.debug('Fixing user/group and permissions on {}'.format(os.path.join(root, momo)))
+            os.chown(os.path.join(root, momo), uid, gid)
+            if momo.startswith('privkey'):
+                os.chmod(os.path.join(root, momo), 0o640)
+            else:
+                os.chmod(os.path.join(root, momo), 0o644)
+
 if __name__ == '__main__':
     args, config = init_prog(sys.argv)
 
     dryrun=config['main'].getboolean('dryrun')
     service_reload = dict()
 
-    ot_conn=connect_db(dict(config['ot_db']))
-    dns_conn=connect_db(dict(config['dns_db']))
-
     if  dryrun:
         print("DRYRUN, nessun certificato verra' richiesto, nessun link/file creato o modificato")
 
+    if args.renew:
+        pre_hook_cmd = None
+        post_hook_cmd = None
+        logging.info('Renewing certificates ')
+        if args.webmail or args.hosting or args.liste:
+            post_hook_cmd = "systemctl reload apache2"
+        elif args.smtp:
+            post_hook_cmd = "systemctl reload postfix"
+        elif args.mbox:
+            post_hook_cmd = "systemctl restart dovecot"
+        elif args.proxy:
+            post_hook_cmd = "systemctl reload nginx"
 
-    # Caso speciale per le webmail
+        logger.debug("post_hook_cmd: {}".format(post_hook_cmd))
-    if args.webmail:
-        logging.info('Asking certificates for webmail')
-        vhost_name = config['webmail']['vhost'].strip()
-        webmails_list = ["webmail.{}".format(d.strip()) for d in config['webmail']['domains'].split(',') if len(d.strip())>0]
-        logging.info('vhost {}, domains_list {}'.format(vhost_name, webmails_list))
-        if acme_request(config, vhost_name, acme_test='HTTP-01', dryrun=dryrun, domains_list=webmails_list):
-            link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
-            service_reload['webmail'] = True
-        else:
-            logger.error('Error asking certificate for {}'.format(vhost_name))
 
-    # Caso speciale per il server POP/IMAP
+        if acme_renew(config, pre_hook_cmd, post_hook_cmd, dryrun=dryrun):
-    if args.mbox:
+            logger.info("Done renew")
-        logging.info('Asking certificates for POP/IMAP server')
+    else:
-        vhost_name = config['mail']['mbox_vhost'].strip()
+        # Fai le nuove richieste per i certificati
-        server_addresses = [s.strip() for s in config['mail']['mbox_server_addresses'].split(',') if len(s.strip())>0]
-        mbox_fmt = ','.join(['%s'] * len(server_addresses))
-        mbox_query = mbox_list_stmt.format(mbox_fmt)
-        alias_list = get_alias_list(config, dns_conn, mbox_query, server_addresses)
-        # Per usi futuri, aggiungo l'alias 'mail.indivia.net'
-        alias_list.append('mail.indivia.net')
-        logging.info('vhost {}, domains_list {}'.format(vhost_name, alias_list))
-        if acme_request(config, vhost_name, acme_test='HTTP-01', webroot=config['mail']['mbox_webroot'].strip(),
-                        dryrun=dryrun, domains_list=alias_list):
-            # non e' richiesto il link, punto direttamente le configurazioni alle dir di letsencrypt
-            # link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
-            service_reload['mbox'] = True
-            pass
-        else:
-            logger.error('Error asking certificate for {}'.format(vhost_name))
 
-    # Caso speciale per il server SMTP
+        # Caso speciale per le webmail
-    if args.smtp:
+        if args.webmail:
-        logging.info('Asking certificates for SMTP server')
+            logging.info('Asking certificates for webmail')
-        vhost_name = config['mail']['smtp_vhost'].strip()
+            vhost_name = config['webmail']['vhost'].strip()
-        server_addresses = [s.strip() for s in config['mail']['smtp_server_addresses'].split(',') if len(s.strip())>0]
+            webmails_list = ["webmail.{}".format(d.strip()) for d in config['webmail']['domains'].split(',') if len(d.strip())>0]
-        smtp_fmt = ','.join(['%s'] * len(server_addresses))
+            logging.info('vhost {}, domains_list {}'.format(vhost_name, webmails_list))
-        smtp_query = smtp_list_stmt.format(smtp_fmt)
+            if acme_request(config, vhost_name, acme_test='HTTP-01', dryrun=dryrun, domains_list=webmails_list):
-        alias_list = get_alias_list(config, dns_conn, smtp_query, server_addresses)
+                link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
-        logging.info('vhost {}, domains_list {}'.format(vhost_name, alias_list))
+            else:
-        if acme_request(config, vhost_name, acme_test='HTTP-01', webroot=config['mail']['smtp_webroot'].strip(),
+                logger.error('Error asking certificate for {}'.format(vhost_name))
-                        dryrun=dryrun, domains_list=alias_list):
-            # non e' richiesto il link, punto direttamente le configurazioni alle dir di letsencrypt
-            # link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
-            service_reload['smtp'] = True
-            pass
-        else:
-            logger.error('Error asking certificate for {}'.format(vhost_name))
 
-    # Caso speciale per l'hosting
+            # reload apache
-    if args.hosting:
+            logger.info("Reloading apache")
-        logging.info('Asking certificates for hosted web domains')
+            # ret = subprocess.run("systemctl reload apache2")
-        # Subdomains da escludere
+            ret = os.system("systemctl reload apache2")
-        ex_subdomains = tuple([s.strip() for s in config['main']['special_subdomains'].split(',') if len(s.strip())>0])
+            logger.info(ret)
-        domains_dict = get_domain_list(config, ot_conn, dns_conn)
 
-        for domain_name, domain_feat in domains_dict.items():
+        # Caso speciale per il proxy
-            domain_feat['subdomains']=get_subdomain_list(config, domain_name, ot_conn, ex_subdomains=ex_subdomains)
+        if args.proxy:
-            # Controlla se i nameserver sono gestiti da noi
+            logging.info('Asking certificates for proxy web domains')
-            if domain_feat['managed_ns']:
+            try:
-                # Nel caso il nameserver sia gestito, chiedi certificati per il dominio e la wildcard
+                proxy_conf = config['nginx']
-                logger.info('Get certificates for {}, *.{}'.format(domain_name, domain_name))
+            except KeyError as e:
-                if acme_request(config, domain_name, acme_test='DNS-01', dryrun=dryrun):
+                logger.error("Error parsing configuration, KeyError {}".format(e))
-                    link_cert(config, domain_name, domain_name, dryrun=dryrun)
+                exit(-1)
+            ot_conn=connect_db(dict(config['ot_db']))
+
+            upstream_servers = [s.strip() for s in proxy_conf['upstream_servers'].split(',') if len(s.strip())>0]
+            for server_name in upstream_servers:
+                logger.debug("Upstream server {}".format(server_name))
+                url_list = get_url_list(proxy_conf, server_name,
+                                    ot_conn, None)
+                logger.debug(url_list)
+                for url in url_list:
+                    acme_request(config, url, acme_test='HTTP-01', webroot=proxy_conf['http-01_webroot'],
+                                 dryrun=dryrun, domains_list=[url])
+
+            ot_conn.close()
+
+            if not dryrun:
+                fix_permissions(config)
+
+            logger.info("Reloading nginx")
+            ret = os.system("systemctl reload nginx")
+            logger.info(ret)
+
+
+        # Caso speciale per l'hosting
+        if args.hosting:
+            logging.info('Asking certificates for hosted web domains')
+            try:
+                hosting_conf = config['apache']
+            except KeyError as e:
+                logger.error("Error parsing configuration, KeyError {}".format(e))
+                exit(-1)
+
+            ot_conn=connect_db(dict(config['ot_db']))
+            dns_conn=connect_db(dict(config['dns_db']))
+            # Subdomains da escludere
+            ex_subdomains = tuple([s.strip() for s in config['main']['special_subdomains'].split(',') if len(s.strip())>0])
+            domains_dict = get_domain_list(config, ot_conn, dns_conn)
+            logger.debug(domains_dict)
+            for domain_name, domain_feat in domains_dict.items():
+                domain_feat['subdomains']=get_subdomain_list(config, domain_name, ot_conn, ex_subdomains=ex_subdomains)
+                # Controlla se i nameserver sono gestiti da noi
+                if domain_feat['managed_ns']:
+                    # Nel caso il nameserver sia gestito, chiedi certificati per il dominio e la wildcard
+                    logger.info('Get certificates for {}, *.{}'.format(domain_name, domain_name))
+                    if acme_request(config, domain_name, acme_test='DNS-01', dryrun=dryrun):
+                        link_cert(config, domain_name, domain_name, dryrun=dryrun)
+                        # Crea il link per ogni subdomain
+                        for subdomain in domain_feat['subdomains']:
+                            link_cert(config, domain_name, subdomain, dryrun=dryrun)
+
+                else:
+                    # Nel caso i nameserver NON siano gestiti, allora chiedi un certificato per ogni sottodominio
                     # Crea il link per ogni subdomain
                     for subdomain in domain_feat['subdomains']:
-                        link_cert(config, domain_name, subdomain, dryrun=dryrun)
+                        logger.info('Get certificates for {}'.format(subdomain))
-                    service_reload['hosting'] = True
+                        if acme_request(config, subdomain, acme_test='HTTP-01', dryrun=dryrun):
+                            link_cert(config, subdomain, subdomain, dryrun=dryrun)
 
+            ot_conn.close()
+            dns_conn.close()
+
+            # reload apache
+            logger.info("Reloading apache")
+            # ret = subprocess.run("systemctl reload apache2")
+            ret = os.system("systemctl reload apache2")
+            logger.info(ret)
+
+
+        # Caso speciale per l'interfaccia di mailman
+        if args.liste:
+            logging.info('Asking certificates for liste.indivia.net')
+            vhost_name = config['mailman']['vhost'].strip()
+            liste_list = ["liste.{}".format(d.strip()) for d in config['mailman']['domains'].split(',') if len(d.strip())>0]
+            if acme_request(config, vhost_name, acme_test='HTTP-01', dryrun=dryrun, domains_list=liste_list):
+                link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
             else:
-                # Nel caso i nameserver NON siano gestiti, allora chiedi un certificato per ogni sottodominio
+                logger.error('Error asking certificate for {}'.format(vhost_name))
-                # Crea il link per ogni subdomain
-                for subdomain in domain_feat['subdomains']:
-                    logger.info('Get certificates for {}'.format(subdomain))
-                    if acme_request(config, subdomain, acme_test='HTTP-01', dryrun=dryrun):
-                        link_cert(config, subdomain, subdomain, dryrun=dryrun)
-                        service_reload['hosting'] = True
-        ot_conn.close()
-        dns_conn.close()
 
-    # Genero il certificato per l'interfaccia di mailman
+            # reload apache
-    if args.liste:
+            logger.info("Reloading apache")
-        logging.info('Asking certificates for liste.indivia.net')
+            # ret = subprocess.run("systemctl reload apache2")
-        vhost_name = config['mailman']['vhost'].strip()
+            ret = os.system("systemctl reload apache2")
-        liste_list = ["liste.{}".format(d.strip()) for d in config['mailman']['domains'].split(',') if len(d.strip())>0]
+            logger.info(ret)
-        if acme_request(config, vhost_name, acme_test='HTTP-01', dryrun=dryrun, domains_list=liste_list):
-            link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
-            service_reload['liste'] = True
-        else:
-            logger.error('Error asking certificate for {}'.format(vhost_name))
 
 
-    if set(['webmail','hosting','liste']) & set(service_reload.keys()):
+        # Caso speciale per il server POP/IMAP
-        # reload apache
+        if args.mbox:
-        logger.info("Restarting apache")
+            dns_conn=connect_db(dict(config['dns_db']))
-        # ret = subprocess.run("systemctl reload apache2")
+            logging.info('Asking certificates for POP/IMAP server')
-        ret = os.system("systemctl reload apache2")
+            vhost_name = config['mail']['mbox_vhost'].strip()
-        logger.info(ret)
+            server_addresses = [s.strip() for s in config['mail']['mbox_server_addresses'].split(',') if len(s.strip())>0]
-    if set(['smtp',]) & set(service_reload.keys()):
+            mbox_fmt = ','.join(['%s'] * len(server_addresses))
-        # reload postfix
+            mbox_query = mbox_list_stmt.format(mbox_fmt)
-        logger.info("Restarting postfix")
+            alias_list = get_alias_list(config, dns_conn, mbox_query, server_addresses)
-        # ret = subprocess.run("systemctl reload postfix")
+            # Per usi futuri, aggiungo l'alias 'mail.indivia.net'
-        ret = os.system("systemctl reload postfix")
+            alias_list.append('mail.indivia.net')
-        logger.info(ret)
+            logging.info('vhost {}, domains_list {}'.format(vhost_name, alias_list))
-    if set(['mbox',]) & set(service_reload.keys()):
+            if acme_request(config, vhost_name, acme_test='HTTP-01', webroot=config['mail']['mbox_webroot'].strip(),
-        # restart dovecot
+                            dryrun=dryrun, domains_list=alias_list):
-        logger.info("Restarting dovecot")
+                # non e' richiesto il link, punto direttamente le configurazioni alle dir di letsencrypt
-        # ret = subprocess.run("systemctl restart dovecot")
+                # link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
-        ret = os.system("systemctl restart dovecot")
+                service_reload['mbox'] = True
-        logger.info(ret)
+                pass
+            else:
+                logger.error('Error asking certificate for {}'.format(vhost_name))
+
+            dns_conn.close()
+
+            # restart dovecot
+            logger.info("Restarting dovecot")
+            # ret = subprocess.run("systemctl restart dovecot")
+            ret = os.system("systemctl restart dovecot")
+            logger.info(ret)
+
+
+        # Caso speciale per il server SMTP
+        if args.smtp:
+            logging.info('Asking certificates for SMTP server')
+            dns_conn=connect_db(dict(config['dns_db']))
+            vhost_name = config['mail']['smtp_vhost'].strip()
+            server_addresses = [s.strip() for s in config['mail']['smtp_server_addresses'].split(',') if len(s.strip())>0]
+            smtp_fmt = ','.join(['%s'] * len(server_addresses))
+            smtp_query = smtp_list_stmt.format(smtp_fmt)
+            alias_list = get_alias_list(config, dns_conn, smtp_query, server_addresses)
+            logging.info('vhost {}, domains_list {}'.format(vhost_name, alias_list))
+            if acme_request(config, vhost_name, acme_test='HTTP-01', webroot=config['mail']['smtp_webroot'].strip(),
+                            dryrun=dryrun, domains_list=alias_list):
+                # non e' richiesto il link, punto direttamente le configurazioni alle dir di letsencrypt
+                # link_cert(config, vhost_name, vhost_name, dryrun=dryrun)
+                service_reload['smtp'] = True
+                pass
+            else:
+                logger.error('Error asking certificate for {}'.format(vhost_name))
+
+            dns_conn.close()
+                
+            # reload postfix
+            logger.info("Restarting postfix")
+            # ret = subprocess.run("systemctl reload postfix")
+            ret = os.system("systemctl reload postfix")
+            logger.info(ret)
+
+