bn8/check_syncrepl_extended
1
0
Fork 0
Code Issues Pull requests Projects Releases 4 Wiki Activity Actions

Code cleaning

Browse source
This commit is contained in:
Benjamin Renard 2020-12-18 11:24:09 +01:00
parent 99aa93a497
commit d19d121843
2 changed files with 473 additions and 392 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.pylintrc Normal file
View file

@ -0,0 +1,8 @@
[MESSAGES CONTROL]
disable=line-too-long,
missing-docstring,
invalid-name,
locally-disabled,
too-many-arguments,
too-many-branches,
redefined-outer-name,

291
check_syncrepl_extended
View file

@ -35,140 +35,187 @@
# Source: http://git.zionetrix.net/check_syncrepl_extended # Source: http://git.zionetrix.net/check_syncrepl_extended
# #
import ldap import argparse
from ldap.controls import SimplePagedResultsControl
import ldap.modlist as modlist
import logging import logging
import sys import sys
import getpass import getpass
from optparse import OptionParser import ldap
from ldap import LDAPError # pylint: disable=no-name-in-module
from ldap.controls import SimplePagedResultsControl
import ldap.modlist as modlist
TOUCH_VALUE = '%%TOUCH%%' TOUCH_VALUE = '%%TOUCH%%'
parser = OptionParser(version="%prog version 1.0\n\nDate : Mon, 10 Dec 2012 18:04:24 +0100\nAuthor : Benjamin Renard <brenard@easter-eggs.com>\nSource : http://git.zionetrix.net/check_syncrepl_extended") parser = argparse.ArgumentParser(
version="""%prog version 1.1
parser.add_option( "-p", "--provider", Date: Fri, 18 Dec 2020 10:48:45 +0100
Author: Benjamin Renard <brenard@easter-eggs.com>
Source: https://gogs.zionetrix.net/bn8/check_syncrepl_extended
"""
)
parser.add_argument(
"-p", "--provider",
dest="provider", dest="provider",
action="store", action="store",
type='string', type=str,
help="LDAP provider URI (example : ldaps://ldapmaster.foo:636)") help="LDAP provider URI (example: ldaps://ldapmaster.foo:636)"
)
parser.add_option( "-c", "--consumer", parser.add_argument(
"-c", "--consumer",
dest="consumer", dest="consumer",
action="store", action="store",
type='string', type=str,
help="LDAP consumer URI (example : ldaps://ldapslave.foo:636)") help="LDAP consumer URI (example: ldaps://ldapslave.foo:636)"
)
parser.add_option( "-i", "--serverID", parser.add_argument(
"-i", "--serverID",
dest="serverid", dest="serverid",
action="store", action="store",
type='int', type=int,
help="Compare contextCSN of a specific master. Useful in MultiMaster setups where each master has a unique ID and a contextCSN for each replicated master exists. A valid serverID is a integer value from 0 to 4095 (limited to 3 hex digits, example: '12' compares the contextCSN matching '#00C#')", help="Compare contextCSN of a specific master. Useful in MultiMaster setups where each master has a unique ID and a contextCSN for each replicated master exists. A valid serverID is a integer value from 0 to 4095 (limited to 3 hex digits, example: '12' compares the contextCSN matching '#00C#')",
default=False) default=False
)
parser.add_option( "-T", "--starttls", parser.add_argument(
"-T", "--starttls",
dest="starttls", dest="starttls",
action="store_true", action="store_true",
help="Start TLS on LDAP provider/consumers connections", help="Start TLS on LDAP provider/consumers connections",
default=False) default=False
)
parser.add_option( "-D", "--dn", parser.add_argument(
"-D", "--dn",
dest="dn", dest="dn",
action="store", action="store",
type='string', type=str,
help="LDAP bind DN (example : uid=nagios,ou=sysaccounts,o=example") help="LDAP bind DN (example: uid=nagios,ou=sysaccounts,o=example"
)
parser.add_option( "-P", "--pwd", parser.add_argument(
"-P", "--pwd",
dest="pwd", dest="pwd",
action="store", action="store",
type='string', type=str,
help="LDAP bind password", help="LDAP bind password",
default=None) default=None
)
parser.add_option( "-b", "--basedn", parser.add_argument(
"-b", "--basedn",
dest="basedn", dest="basedn",
action="store", action="store",
type='string', type=str,
help="LDAP base DN (example : o=example)") help="LDAP base DN (example: o=example)"
)
parser.add_option( "-f", "--filter", parser.add_argument(
dest="filter", "-f", "--filter",
dest="filterstr",
action="store", action="store",
type='string', type=str,
help="LDAP filter (default: (objectClass=*))", help="LDAP filter (default: (objectClass=*))",
default='(objectClass=*)') default='(objectClass=*)'
)
parser.add_option( "-d", "--debug", parser.add_argument(
"-d", "--debug",
dest="debug", dest="debug",
action="store_true", action="store_true",
help="Debug mode", help="Debug mode",
default=False) default=False
)
parser.add_option( "-n", "--nagios", parser.add_argument(
"-n", "--nagios",
dest="nagios", dest="nagios",
action="store_true", action="store_true",
help="Nagios check plugin mode", help="Nagios check plugin mode",
default=False) default=False
)
parser.add_option( "-q", "--quiet", parser.add_argument(
"-q", "--quiet",
dest="quiet", dest="quiet",
action="store_true", action="store_true",
help="Quiet mode", help="Quiet mode",
default=False) default=False
)
parser.add_option( "--no-check-certificate", parser.add_argument(
"--no-check-certificate",
dest="nocheckcert", dest="nocheckcert",
action="store_true", action="store_true",
help="Don't check the server certificate (Default: False)", help="Don't check the server certificate (Default: False)",
default=False) default=False
)
parser.add_option( "--no-check-contextCSN", parser.add_argument(
"--no-check-contextCSN",
dest="nocheckcontextcsn", dest="nocheckcontextcsn",
action="store_true", action="store_true",
help="Don't check servers contextCSN (Default: False)", help="Don't check servers contextCSN (Default: False)",
default=False) default=False
)
parser.add_option( "-a", "--attributes", parser.add_argument(
"-a", "--attributes",
dest="attrs", dest="attrs",
action="store_true", action="store_true",
help="Check attributes values (Default: check only entryCSN)", help="Check attributes values (Default: check only entryCSN)",
default=False) default=False
)
parser.add_option( "--exclude-attributes", parser.add_argument(
"--exclude-attributes",
dest="excl_attrs", dest="excl_attrs",
action="store", action="store",
type='string', type=str,
help="Don't check this attribut (only in attribute check mode)", help="Don't check this attribut (only in attribute check mode)",
default=None) default=None
)
parser.add_option( "--touch", parser.add_argument(
"--touch",
dest="touch", dest="touch",
action="store", action="store",
type='string', type=str,
help="Touch attribute giving in parameter to force resync a this LDAP object from provider. A value '%s' will be add to this attribute and remove after. The user use to connect to the LDAP directory must have write permission on this attribute on each object." % TOUCH_VALUE, help="Touch attribute giving in parameter to force resync a this LDAP object from provider. A value '%s' will be add to this attribute and remove after. The user use to connect to the LDAP directory must have write permission on this attribute on each object." % TOUCH_VALUE,
default=None) default=None
)
parser.add_option( "--replace-touch", parser.add_argument(
"--replace-touch",
dest="replacetouch", dest="replacetouch",
action="store_true", action="store_true",
help="In touch mode, replace value instead of adding.", help="In touch mode, replace value instead of adding.",
default=False) default=False
)
parser.add_option( "--remove-touch-value", parser.add_argument(
"--remove-touch-value",
dest="removetouchvalue", dest="removetouchvalue",
action="store_true", action="store_true",
help="In touch mode, remove touch value if present.", help="In touch mode, remove touch value if present.",
default=False) default=False
)
parser.add_option( "--page-size", parser.add_argument(
"--page-size",
dest="page_size", dest="page_size",
action="store", action="store",
type='int', type=int,
help="Page size: if defined, paging control using LDAP v3 extended control will be enabled.", help="Page size: if defined, paging control using LDAP v3 extended control will be enabled.",
default=None) default=None
)
(options, args) = parser.parse_args() options = parser.parse_args()
if not options.provider or not options.consumer: if not options.provider or not options.consumer:
print "You must provide provider and customer URI" print "You must provide provider and customer URI"
@ -204,7 +251,7 @@ FORMAT="%(asctime)s - %(levelname)s : %(message)s"
if options.debug: if options.debug:
logging.basicConfig(level=logging.DEBUG, format=FORMAT) logging.basicConfig(level=logging.DEBUG, format=FORMAT)
ldap.set_option(ldap.OPT_DEBUG_LEVEL,0) ldap.set_option(ldap.OPT_DEBUG_LEVEL, 0) # pylint: disable=no-member
ldapmodule_trace_level = 1 ldapmodule_trace_level = 1
ldapmodule_trace_file = sys.stderr ldapmodule_trace_file = sys.stderr
elif options.nagios: elif options.nagios:
@ -234,65 +281,69 @@ class LdapServer(object):
if self.con == 0: if self.con == 0:
try: try:
con = ldap.initialize(self.uri) con = ldap.initialize(self.uri)
con.protocol_version = ldap.VERSION3 con.protocol_version = ldap.VERSION3 # pylint: disable=no-member
if self.start_tls: if self.start_tls:
con.start_tls_s() con.start_tls_s()
if self.dn: if self.dn:
con.simple_bind_s(self.dn, self.pwd) con.simple_bind_s(self.dn, self.pwd)
self.con = con self.con = con
except LDAPError:
logging.error("LDAP Error", exc_info=True)
return False
return True return True
except ldap.LDAPError, e:
logging.error("LDAP Error : %s" % e)
return
def getContextCSN(self, basedn=False, serverid=False): def getContextCSN(self, basedn=False, serverid=False):
if not basedn: if not basedn:
basedn = self.dn basedn = self.dn
data = self.search(basedn, '(objectclass=*)', ['contextCSN']) data = self.search(basedn, '(objectclass=*)', ['contextCSN'])
if len(data)>0: if data:
contextCSNs = data[0][0][1]['contextCSN'] contextCSNs = data[0][0][1]['contextCSN']
logging.debug('Found contextCSNs %s' % contextCSNs) logging.debug('Found contextCSNs %s', contextCSNs)
if serverid is False: if serverid is False:
return contextCSNs[0] return contextCSNs[0]
else:
csnid = str(format(serverid, 'X')).zfill(3) csnid = str(format(serverid, 'X')).zfill(3)
sub = '#%s#' % csnid sub = '#%s#' % csnid
CSN = [s for s in contextCSNs if sub in s] CSN = [s for s in contextCSNs if sub in s]
if not CSN: if not CSN:
logging.error("No contextCSN matching with ServerID %s (=%s) could be found." % (serverid,sub)) logging.error(
"No contextCSN matching with ServerID %s (=%s) could be found.",
serverid, sub
)
return False return False
else:
return CSN[0] return CSN[0]
else:
return False return False
def search(self,basedn,filter,attrs): def search(self, basedn, filterstr, attrs):
if self.page_size: if self.page_size:
return self.paged_search(basedn,filter,attrs) return self.paged_search(basedn, filterstr, attrs)
res_id = self.con.search(basedn,ldap.SCOPE_SUBTREE,filter,attrs) res_id = self.con.search(basedn, ldap.SCOPE_SUBTREE, filterstr, attrs) # pylint: disable=no-member
ret = [] ret = []
while 1: while 1:
res_type, res_data = self.con.result(res_id, 0) res_type, res_data = self.con.result(res_id, 0)
if res_data == []: if res_data == []:
break break
else: else:
if res_type == ldap.RES_SEARCH_ENTRY: if res_type == ldap.RES_SEARCH_ENTRY: # pylint: disable=no-member
ret.append(res_data) ret.append(res_data)
return ret return ret
def paged_search(self,basedn,filter,attrs): def paged_search(self, basedn, filterstr, attrs):
ret = [] ret = []
page = 0 page = 0
pg_ctrl = SimplePagedResultsControl(True, self.page_size, '') pg_ctrl = SimplePagedResultsControl(True, self.page_size, '')
pg_oid = SimplePagedResultsControl.controlType
while page == 0 or pg_ctrl.cookie: while page == 0 or pg_ctrl.cookie:
page += 1 page += 1
logging.debug('Page search : loading page %d' % page) logging.debug('Page search: loading page %d', page)
res_id = self.con.search_ext(basedn,ldap.SCOPE_SUBTREE,filter,attrs,serverctrls=[pg_ctrl]) res_id = self.con.search_ext(
res_type, res_data, res_id, serverctrls = self.con.result3(res_id) basedn, ldap.SCOPE_SUBTREE, # pylint: disable=no-member
filterstr, attrs,
serverctrls=[pg_ctrl]
)
res_type, res_data, res_id, serverctrls = self.con.result3(res_id) # pylint: disable=unused-variable
for serverctrl in serverctrls: for serverctrl in serverctrls:
if serverctrl.controlType == pg_oid: if serverctrl.controlType == SimplePagedResultsControl.controlType:
pg_ctrl.cookie = serverctrl.cookie pg_ctrl.cookie = serverctrl.cookie
break
for item in res_data: for item in res_data:
ret.append([item]) ret.append([item])
return ret return ret
@ -302,14 +353,15 @@ class LdapServer(object):
if ldif == []: if ldif == []:
return True return True
try: try:
logging.debug('Update object %s : %s' % (dn,ldif)) logging.debug('Update object %s: %s', dn, ldif)
self.con.modify_s(dn, ldif) self.con.modify_s(dn, ldif)
return True return True
except ldap.LDAPError, e: except LDAPError:
logging.error('Error updating object %s : %s' % (dn,e)) logging.error('Error updating object %s', dn, exc_info=True)
return False return False
def get_attr(self,obj,attr): @staticmethod
def get_attr(obj, attr):
if attr in obj[0][1]: if attr in obj[0][1]:
return obj[0][1][attr] return obj[0][1][attr]
return [] return []
@ -330,19 +382,22 @@ class LdapServer(object):
else: else:
new[attr].append(TOUCH_VALUE) new[attr].append(TOUCH_VALUE)
try: try:
logging.info('Touch object "%s" on attribute "%s" : %s => %s', dn, attr, old, new) logging.info(
'Touch object "%s" on attribute "%s": %s => %s',
dn, attr, old, new
)
if self.update_object(dn, old, new): if self.update_object(dn, old, new):
logging.info('Restore original value of attribute "%s" of object "%s"', attr, dn) logging.info('Restore original value of attribute "%s" of object "%s"', attr, dn)
if options.removetouchvalue and TOUCH_VALUE in old[attr]: if options.removetouchvalue and TOUCH_VALUE in old[attr]:
old[attr].remove(TOUCH_VALUE) old[attr].remove(TOUCH_VALUE)
self.update_object(dn, new, old) self.update_object(dn, new, old)
return True return True
except ldap.LDAPError: except LDAPError:
logging.error('Error touching object "%s"', dn, exc_info=True) logging.error('Error touching object "%s"', dn, exc_info=True)
return False return False
if options.nocheckcert: if options.nocheckcert:
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT,ldap.OPT_X_TLS_NEVER) ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER) # pylint: disable=no-member
servers = [options.provider, options.consumer] servers = [options.provider, options.consumer]
@ -351,7 +406,7 @@ LdapObjects={}
LdapServersCSN = {} LdapServersCSN = {}
for srv in servers: for srv in servers:
logging.info('Connect to %s' % srv) logging.info('Connect to %s', srv)
LdapServers[srv] = LdapServer(srv, options.dn, options.pwd, options.starttls, page_size=options.page_size) LdapServers[srv] = LdapServer(srv, options.dn, options.pwd, options.starttls, page_size=options.page_size)
if not LdapServers[srv].connect(): if not LdapServers[srv].connect():
@ -363,21 +418,24 @@ for srv in servers:
if not options.nocheckcontextcsn: if not options.nocheckcontextcsn:
LdapServersCSN[srv] = LdapServers[srv].getContextCSN(options.basedn, options.serverid) LdapServersCSN[srv] = LdapServers[srv].getContextCSN(options.basedn, options.serverid)
logging.info('ContextCSN of %s : %s' % (srv, LdapServersCSN[srv])) logging.info('ContextCSN of %s: %s', srv, LdapServersCSN[srv])
logging.info('List objects from %s' % srv) logging.info('List objects from %s', srv)
LdapObjects[srv] = {} LdapObjects[srv] = {}
if options.attrs: if options.attrs:
for obj in LdapServers[srv].search(options.basedn,options.filter,[]): for obj in LdapServers[srv].search(options.basedn, options.filterstr, []):
logging.debug('Found on %s : %s' % (srv,obj[0][0])) logging.debug('Found on %s: %s', srv, obj[0][0])
LdapObjects[srv][obj[0][0]] = obj[0][1] LdapObjects[srv][obj[0][0]] = obj[0][1]
else: else:
for obj in LdapServers[srv].search(options.basedn,options.filter,['entryCSN']): for obj in LdapServers[srv].search(options.basedn, options.filterstr, ['entryCSN']):
logging.debug('Found on %s : %s / %s' % (srv,obj[0][0],obj[0][1]['entryCSN'][0])) logging.debug(
'Found on %s: %s / %s',
srv, obj[0][0], obj[0][1]['entryCSN'][0]
)
LdapObjects[srv][obj[0][0]] = obj[0][1]['entryCSN'][0] LdapObjects[srv][obj[0][0]] = obj[0][1]['entryCSN'][0]
logging.info('%s objects founds' % len(LdapObjects[srv])) logging.info('%s objects founds', len(LdapObjects[srv]))
not_found = {} not_found = {}
@ -392,7 +450,7 @@ if options.attrs:
else: else:
logging.info('Check if objets are synchronized (by comparing entryCSN)') logging.info('Check if objets are synchronized (by comparing entryCSN)')
for obj in LdapObjects[options.provider]: for obj in LdapObjects[options.provider]:
logging.debug('Check obj %s' % (obj)) logging.debug('Check obj %s', obj)
for srv in LdapObjects: for srv in LdapObjects:
if srv == options.provider: if srv == options.provider:
continue continue
@ -406,19 +464,28 @@ for obj in LdapObjects[options.provider]:
continue continue
if attr not in LdapObjects[srv][obj]: if attr not in LdapObjects[srv][obj]:
attrs_list.append(attr) attrs_list.append(attr)
logging.debug("Obj %s not synchronized : %s not present on %s" % (obj,','.join(attrs_list),srv)) logging.debug(
"Obj %s not synchronized: %s not present on %s",
obj, ','.join(attrs_list), srv
)
touch = True touch = True
else: else:
LdapObjects[srv][obj][attr].sort() LdapObjects[srv][obj][attr].sort()
LdapObjects[options.provider][obj][attr].sort() LdapObjects[options.provider][obj][attr].sort()
if LdapObjects[srv][obj][attr] != LdapObjects[options.provider][obj][attr]: if LdapObjects[srv][obj][attr] != LdapObjects[options.provider][obj][attr]:
attrs_list.append(attr) attrs_list.append(attr)
logging.debug("Obj %s not synchronized : %s not same value(s)" % (obj,','.join(attrs_list))) logging.debug(
"Obj %s not synchronized: %s not same value(s)",
obj, ','.join(attrs_list)
)
touch = True touch = True
if len(attrs_list)>0: if attrs_list:
not_sync[srv].append("%s (%s)" % (obj, ','.join(attrs_list))) not_sync[srv].append("%s (%s)" % (obj, ','.join(attrs_list)))
else: else:
logging.debug("Obj %s not synchronized : %s <-> %s" % (obj,LdapObjects[options.provider][obj],LdapObjects[srv][obj])) logging.debug(
"Obj %s not synchronized: %s <-> %s",
obj, LdapObjects[options.provider][obj], LdapObjects[srv][obj]
)
not_sync[srv].append(obj) not_sync[srv].append(obj)
if touch and options.touch: if touch and options.touch:
orig_value = [] orig_value = []
@ -426,7 +493,7 @@ for obj in LdapObjects[options.provider]:
orig_value = LdapObjects[options.provider][obj][options.touch] orig_value = LdapObjects[options.provider][obj][options.touch]
LdapServers[options.provider].touch_object(obj, options.touch, orig_value) LdapServers[options.provider].touch_object(obj, options.touch, orig_value)
else: else:
logging.debug('Obj %s : not found on %s' % (obj,srv)) logging.debug('Obj %s: not found on %s', obj, srv)
not_found[srv].append(obj) not_found[srv].append(obj)
if options.touch: if options.touch:
orig_value = [] orig_value = []
@ -435,9 +502,9 @@ for obj in LdapObjects[options.provider]:
LdapServers[options.provider].touch_object(obj, options.touch, orig_value) LdapServers[options.provider].touch_object(obj, options.touch, orig_value)
for obj in LdapObjects[options.consumer]: for obj in LdapObjects[options.consumer]:
logging.debug('Check obj %s of consumer' % obj) logging.debug('Check obj %s of consumer', obj)
if obj not in LdapObjects[options.provider]: if obj not in LdapObjects[options.provider]:
logging.debug('Obj %s : not found on provider' % obj) logging.debug('Obj %s: not found on provider', obj)
not_found[options.provider].append(obj) not_found[options.provider].append(obj)
if options.nagios: if options.nagios:
@ -458,22 +525,22 @@ if options.nagios:
errors.append('ContextCSN of %s not the same of provider' % srv) errors.append('ContextCSN of %s not the same of provider' % srv)
long_output.append('ContextCSN on LDAP server %s = %s' % (srv, LdapServersCSN[srv])) long_output.append('ContextCSN on LDAP server %s = %s' % (srv, LdapServersCSN[srv]))
if len(not_found[options.consumer])>0: if not_found[options.consumer]:
errors.append("%s not found object(s) on consumer" % len(not_found[options.consumer])) errors.append("%s not found object(s) on consumer" % len(not_found[options.consumer]))
long_output.append("Object(s) not found on server %s (consumer) :" % options.consumer) long_output.append("Object(s) not found on server %s (consumer) :" % options.consumer)
for obj in not_found[options.consumer]: for obj in not_found[options.consumer]:
long_output.append(" - %s" % obj) long_output.append(" - %s" % obj)
if len(not_found[options.provider])>0: if not_found[options.provider]:
errors.append("%s not found object(s) on provider" % len(not_found[options.provider])) errors.append("%s not found object(s) on provider" % len(not_found[options.provider]))
long_output.append("Object(s) not found on server %s (provider) :" % options.provider) long_output.append("Object(s) not found on server %s (provider) :" % options.provider)
for obj in not_found[options.provider]: for obj in not_found[options.provider]:
long_output.append(" - %s" % obj) long_output.append(" - %s" % obj)
if len(not_sync[options.consumer])>0: if not_sync[options.consumer]:
errors.append("%s not synchronized object(s) on consumer" % len(not_sync[options.consumer])) errors.append("%s not synchronized object(s) on consumer" % len(not_sync[options.consumer]))
long_output.append("Object(s) not synchronized on server %s (consumer) :" % options.consumer) long_output.append("Object(s) not synchronized on server %s (consumer) :" % options.consumer)
for obj in not_sync[options.consumer]: for obj in not_sync[options.consumer]:
long_output.append(" - %s" % obj) long_output.append(" - %s" % obj)
if len(errors)>0: if errors:
print "CRITICAL: " + ', '.join(errors) + "\n\n" + "\n".join(long_output) print "CRITICAL: " + ', '.join(errors) + "\n\n" + "\n".join(long_output)
sys.exit(2) sys.exit(2)
else: else:
@ -491,17 +558,23 @@ else:
if srv == options.provider: if srv == options.provider:
continue continue
if not LdapServersCSN[srv]: if not LdapServersCSN[srv]:
logging.warning('ContextCSN of %s not found' % srv) logging.warning('ContextCSN of %s not found', srv)
noerror = False noerror = False
elif LdapServersCSN[srv] != LdapServersCSN[options.provider]: elif LdapServersCSN[srv] != LdapServersCSN[options.provider]:
logging.warning('ContextCSN of %s not the same of provider' % srv) logging.warning('ContextCSN of %s not the same of provider', srv)
noerror = False noerror = False
if len(not_found[srv])>0: if not_found[srv]:
logging.warning('Not found objects on %s :\n - %s' % (srv,'\n - '.join(not_found[srv]))) logging.warning(
'Not found objects on %s :\n - %s',
srv, '\n - '.join(not_found[srv])
)
noerror = False noerror = False
if len(not_sync[srv])>0: if not_sync[srv]:
logging.warning('Not sync objects on %s : %s' % (srv,'\n - '.join(not_sync[srv]))) logging.warning(
'Not sync objects on %s: %s',
srv, '\n - '.join(not_sync[srv])
)
noerror = False noerror = False
if noerror: if noerror: