From mboxrd@z Thu Jan 1 00:00:00 1970 From: rmccabe@sourceware.org Date: 27 Jun 2007 06:19:23 -0000 Subject: [Cluster-devel] conga/luci cluster/index_html site/luci/Extens ... Message-ID: <20070627061923.32515.qmail@sourceware.org> List-Id: To: cluster-devel.redhat.com MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit CVSROOT: /cvs/cluster Module name: conga Branch: RHEL5 Changes by: rmccabe at sourceware.org 2007-06-27 06:19:22 Modified files: luci/cluster : index_html luci/site/luci/Extensions: HelperFunctions.py LuciDB.py RicciQueries.py cluster_adapters.py ricci_communicator.py luci/storage : index_html Log message: Add support for issuing ricci batch jobs on all cluster nodes simultaneously Patches: http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/cluster/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.20.2.10&r2=1.20.2.11 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.4.2.3&r2=1.4.2.4 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciDB.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/RicciQueries.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.120.2.31&r2=1.120.2.32 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/ricci_communicator.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.9.2.11&r2=1.9.2.12 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/storage/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.7.2.2&r2=1.7.2.3 --- conga/luci/cluster/index_html 2007/06/18 20:14:55 1.20.2.10 +++ conga/luci/cluster/index_html 2007/06/27 06:19:21 1.20.2.11 @@ -44,7 +44,7 @@ + tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" /> @@ -52,7 +52,7 @@ + tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" /> --- conga/luci/site/luci/Extensions/HelperFunctions.py 2007/06/18 18:39:32 1.4.2.3 +++ conga/luci/site/luci/Extensions/HelperFunctions.py 2007/06/27 06:19:22 1.4.2.4 @@ -18,11 +18,13 @@ return '%s; %s' % (str1, str2) class Worker(threading.Thread): - def __init__(self, mutex, hosts, riccis): + def __init__(self, mutex, hosts, riccis, func=None, *args): threading.Thread.__init__(self) self.mutex = mutex self.hosts = hosts self.riccis = riccis + self.query_func = func + self.query_args = args def run(self): while True: @@ -32,21 +34,45 @@ return host = self.hosts.pop() self.mutex.release() - r = None + + r = { 'ricci': None } try: - r = RicciCommunicator(host) + rc = RicciCommunicator(host) + r['ricci'] = rc + + if self.query_func is not None: + if self.query_args: + args = list(self.query_args) + else: + args = list() + args.insert(0, rc) + r['batch_result'] = self.query_func(*args) except Exception, e: - #print host, 'failed', str(e) - pass - except: - #print host, 'failed' pass self.mutex.acquire() self.riccis[host] = r self.mutex.release() +def send_batch_to_hosts(system_list, max_threads, func, *args): + mutex = threading.RLock() + threads = list() + hosts = list() + num_hosts = 0 + ret = {} + + for host in system_list: + hosts.append(host) + num_hosts += 1 + if num_hosts <= max_threads: + threads.append(Worker(mutex, hosts, ret, func, *args)) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return ret + def get_system_info(self, system_list): mutex = threading.RLock() hive = [] # workers @@ -72,7 +98,7 @@ trusted = '' authed = False trusted = False - ricci = ss[hostname] + ricci = ss[hostname]['ricci'] if ricci is not None: OS = ricci.os() --- conga/luci/site/luci/Extensions/LuciDB.py 2007/06/18 18:39:32 1.1.4.1 +++ conga/luci/site/luci/Extensions/LuciDB.py 2007/06/27 06:19:22 1.1.4.2 @@ -6,7 +6,6 @@ # Free Software Foundation. from AccessControl import getSecurityManager -import RicciQueries as rq from ricci_communicator import RicciCommunicator from LuciZopePerm import isAdmin from LuciSyslog import get_logger @@ -158,7 +157,7 @@ return True batch_id = item[1].getProperty(BATCH_ID) - batch_ret = rq.checkBatch(rc, batch_id) + batch_ret = rc.batch_status(batch_id) finished = batch_ret[0] if finished is True or finished == -1: if finished == -1: --- conga/luci/site/luci/Extensions/RicciQueries.py 2007/06/18 18:39:32 1.1.4.1 +++ conga/luci/site/luci/Extensions/RicciQueries.py 2007/06/27 06:19:22 1.1.4.2 @@ -6,52 +6,12 @@ # Free Software Foundation. from xml.dom import minidom -from ricci_communicator import RicciCommunicator, extract_module_status +from ricci_communicator import RicciCommunicator from LuciSyslog import get_logger from conga_constants import LUCI_DEBUG_MODE luci_log = get_logger() -def checkBatch(rc, batch_id): - err_msg = 'An unknown Ricci error occurred on %s' % rc.hostname() - - try: - batch = rc.batch_report(batch_id) - if batch is None: - if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('checkBatch0: batch id %s not found' \ - % batch_id) - return (True, 'batch id %s was not found' % batch_id) - except Exception, e: - if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('checkBatch1: %s: %r %s' \ - % (rc.hostname(), e, str(e))) - return (-1, err_msg) - - try: - code, new_err_msg = extract_module_status(batch, 1) - if new_err_msg: - err_msg = 'A Ricci error occurred on %s: %s' \ - % (rc.hostname(), str(new_err_msg)) - except Exception, e: - if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('checkBatch2: %s: %r %s' \ - % (rc.hostname(), e, str(e))) - return (-1, err_msg) - - # In progress. - if code == -101 or code == -102: - return (False, 'in progress') - - # Done successfully. - if code == 0: - return (True, 'completed sucessfully') - - # Error - if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('checkBatch3: %d: %s' % (code, rc.hostname())) - return (-1, err_msg) - def addClusterNodeBatch(cluster_name, install_base, install_services, @@ -310,9 +270,9 @@ for i in batch: try: - batch_number = i.getAttribute('batch_id') - result = i.getAttribute('status') - return (str(batch_number), str(result)) + batch_number = str(i.getAttribute('batch_id')) + result = str(i.getAttribute('status')) + return (batch_number, result) except Exception, e: if LUCI_DEBUG_MODE is True: luci_log.debug_verbose('BAR1: %s' % e) --- conga/luci/site/luci/Extensions/cluster_adapters.py 2007/06/19 15:54:10 1.120.2.31 +++ conga/luci/site/luci/Extensions/cluster_adapters.py 2007/06/27 06:19:22 1.120.2.32 @@ -640,7 +640,7 @@ # abort the whole process. try: while True: - batch_ret = rq.checkBatch(cluster_ricci, batch_number) + batch_ret = cluster_ricci.batch_status(batch_number) code = batch_ret[0] if code is True: break @@ -3106,7 +3106,7 @@ if rc is not None: batch_num = item[1].getProperty(BATCH_ID) - batch_res = rq.checkBatch(rc, batch_num) + batch_res = rc.batch_status(batch_num) finished = batch_res[0] err_msg = batch_res[1] --- conga/luci/site/luci/Extensions/ricci_communicator.py 2007/06/18 18:39:33 1.9.2.11 +++ conga/luci/site/luci/Extensions/ricci_communicator.py 2007/06/27 06:19:22 1.9.2.12 @@ -53,7 +53,10 @@ % (self.__hostname, hello.toxml())) except: if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('RC:init0: error receiving header from %s: %r %s' % (self.__hostname, e, str(e))) + luci_log.debug_verbose('RC:init0: error receiving header from %s:%d: %r %s' % (self.__hostname, self.__port, e, str(e))) + else: + luci_log.info('Error reading from %s:%d: %s' \ + % (self.__hostname, self.__port, str(e))) self.__authed = hello.firstChild.getAttribute('authenticated') == 'true' self.__cluname = hello.firstChild.getAttribute('clustername') @@ -231,8 +234,8 @@ if doc.firstChild.getAttribute('success') != '0': if LUCI_DEBUG_MODE is True: luci_log.debug_verbose('RC:PB4: batch command failed') - raise RicciError, 'The last ricci command to host %s failed' \ - % self.__hostname + raise RicciError, 'The last ricci command to host %s:%d failed' \ + % (self.__hostname, self.__port) batch_node = None for node in doc.firstChild.childNodes: @@ -243,8 +246,8 @@ if LUCI_DEBUG_MODE is True: luci_log.debug_verbose('RC:PB5: no from %s' \ % self.__hostname) - raise RicciError, 'missing in ricci response from "%s"' \ - % self.__hostname + raise RicciError, 'missing in ricci response from %s:%d' \ + % (self.__hostname, self.__port) return batch_node @@ -264,7 +267,7 @@ ricci_xml = self.process_batch(batch_xml, async) if LUCI_DEBUG_NET is True: try: - luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command.' % (ricci_xml.toxml(), self.__hostname)) + luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command' % (ricci_xml.toxml(), self.__hostname)) except: pass except: @@ -276,6 +279,48 @@ doc.appendChild(ricci_xml) return doc + def batch_status(self, batch_id): + err_msg = 'An unknown ricci error occurred on %s:%d' \ + % (self.__hostname, self.__port) + + try: + batch = self.batch_report(batch_id) + if batch is None: + if LUCI_DEBUG_MODE is True: + luci_log.debug_verbose('RCCB0: batch id %s not found' \ + % batch_id) + return (True, 'batch id %s was not found' % batch_id) + except Exception, e: + if LUCI_DEBUG_MODE is True: + luci_log.debug_verbose('RCCB1: %s:%d: %r %s' \ + % (self.__hostname, self.__port, e, str(e))) + return (-1, err_msg) + + try: + code, new_err_msg = extract_module_status(batch, 1) + if new_err_msg: + err_msg = 'A ricci error occurred on %s:%d: %s' \ + % (self.__hostname, self.__port, str(new_err_msg)) + except Exception, e: + if LUCI_DEBUG_MODE is True: + luci_log.debug_verbose('RCCB2: %s:%d %r %s: %s' \ + % (self.__hostname, self.__port, e, str(e), err_msg)) + return (-1, err_msg) + + # In progress. + if code == -101 or code == -102: + return (False, 'in progress') + + # Done successfully. + if code == 0: + return (True, 'completed sucessfully') + + # Error + if LUCI_DEBUG_MODE is True: + luci_log.debug_verbose('RCCB3: %s:%d code %d: %s' \ + % (self.__hostname, self.__port, code, err_msg)) + return (-1, err_msg) + def batch_report(self, batch_id): if LUCI_DEBUG_NET is True: luci_log.debug_net('RC:BRep0: [auth=%d] asking for batchid# %s for host %s' % (self.__authed, batch_id, self.__hostname)) @@ -374,6 +419,9 @@ return doc def get_ricci_communicator(self, hostname, allowed_systems): + if not hostname: + return None + if not self.access_to_host_allowed(hostname, allowed_systems): if LUCI_DEBUG_MODE is True: luci_log.debug_verbose('GRC0: access to host %s is not allowed' % hostname) --- conga/luci/storage/index_html 2007/06/26 17:00:53 1.7.2.2 +++ conga/luci/storage/index_html 2007/06/27 06:19:22 1.7.2.3 @@ -222,7 +222,7 @@

Future Site of Forms

+ tal:condition="python: report_cached and not (request.has_key('pagetype') and request['pagetype']=='commit_changes')">