From mboxrd@z Thu Jan 1 00:00:00 1970 From: rmccabe@sourceware.org Date: 22 Jan 2008 15:05:20 -0000 Subject: [Cluster-devel] conga/luci/site/luci/Extensions HelperFunction ... Message-ID: <20080122150520.16715.qmail@sourceware.org> List-Id: To: cluster-devel.redhat.com MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit CVSROOT: /cvs/cluster Module name: conga Changes by: rmccabe at sourceware.org 2008-01-22 15:05:18 Modified files: luci/site/luci/Extensions: HelperFunctions.py LuciValidation.py LuciZopeAsync.py LuciZopeExternal.py cluster_adapters.py homebase_adapters.py Log message: sync up with my local tree - fixes for 252348 and 230462 Patches: http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&r1=1.13&r2=1.14 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciValidation.py.diff?cvsroot=cluster&r1=1.4&r2=1.5 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciZopeAsync.py.diff?cvsroot=cluster&r1=1.5&r2=1.6 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciZopeExternal.py.diff?cvsroot=cluster&r1=1.6&r2=1.7 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&r1=1.278&r2=1.279 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/homebase_adapters.py.diff?cvsroot=cluster&r1=1.55&r2=1.56 --- conga/luci/site/luci/Extensions/HelperFunctions.py 2008/01/15 04:41:40 1.13 +++ conga/luci/site/luci/Extensions/HelperFunctions.py 2008/01/22 15:05:17 1.14 @@ -9,8 +9,7 @@ import threading def resolveOSType(os_str): - rhel5_clu_ver = [ 'Tikanga', 'Zod', 'Moonshine', 'Werewolf' ] - if not os_str or os_str in rhel5_clu_ver: + if not os_str or os_str.find('Tikanga') != (-1) or os_str.find('FC6') != (-1) or os_str.find('Zod') != (-1) or os_str.find('Moonshine') != (-1) or os_str.find('Werewolf') != (-1): return 'rhel5' else: return 'rhel4' @@ -18,6 +17,46 @@ def add_commas(self, str1, str2): return '%s; %s' % (str1, str2) +class PWorker(threading.Thread): + # triple := [ host, function, *args ] + def __init__(self, mutex, ret, host_triples): + threading.Thread.__init__(self) + self.mutex = mutex + self.ret = ret + self.triples = host_triples + + def run(self): + while True: + self.mutex.acquire() + if len(self.triples) == 0: + self.mutex.release() + return + triple = self.triples.pop() + self.mutex.release() + + r = { 'ricci': None } + + try: + rc = RicciCommunicator(triple[0], enforce_trust=False) + rc.trust() + r['ricci'] = rc + + if triple[1] is not None: + if triple[2]: + args = list(triple[2]) + else: + args = list() + args.insert(0, rc) + r['batch_result'] = triple[1](*args) + except Exception, e: + r['error'] = True + r['err_msg'] = str(e) + r['ricci'] = None + + self.mutex.acquire() + self.ret[triple[0]] = r + self.mutex.release() + class Worker(threading.Thread): def __init__(self, mutex, hosts, riccis, func=None, *args): threading.Thread.__init__(self) @@ -39,7 +78,8 @@ r = { 'ricci': None } try: - rc = RicciCommunicator(host) + rc = RicciCommunicator(host, enforce_trust=False) + rc.trust() r['ricci'] = rc try: r['cluster_name'] = rc.cluster_info()[0] @@ -55,7 +95,8 @@ r['batch_result'] = self.query_func(*args) except Exception, e: r['error'] = True - r['err_msg'] = '%s' % str(e) + r['err_msg'] = str(e) + r['ricci'] = None self.mutex.acquire() self.riccis[host] = r @@ -73,6 +114,26 @@ num_hosts += 1 if num_hosts <= max_threads: threads.append(Worker(mutex, hosts, ret, func, *args)) + + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return ret + +def send_batch_parallel(triples, max_threads): + mutex = threading.RLock() + threads = list() + num_trips = 0 + trips = list() + ret = {} + + for trip in triples: + trips.append(trip) + num_trips += 1 + if num_trips <= max_threads: + threads.append(PWorker(mutex, ret, triples)) + for thread in threads: thread.start() for thread in threads: @@ -111,7 +172,7 @@ cluname = ricci.cluster_info()[0] cluali = ricci.cluster_info()[1] authed = ricci.authed() - key_fp = ricci.fingerprint() + key_fp = ricci.fingerprint()[1] trusted = ricci.trusted() else: OS = 'System not accessible' @@ -124,7 +185,8 @@ 'clualias' : cluali, 'available' : ricci is not None, 'trusted' : trusted, - 'authed' : authed + 'authed' : authed, + 'err_msg' : ss[hostname].get('err_msg') } # replace ricci with system's info --- conga/luci/site/luci/Extensions/LuciValidation.py 2008/01/17 16:36:30 1.4 +++ conga/luci/site/luci/Extensions/LuciValidation.py 2008/01/22 15:05:17 1.5 @@ -1189,3 +1189,66 @@ return (False, { 'errors': [ 'Error removing fence device %s: %s' \ % (fencedev_name, str(e)) ]}) return (True, { 'name': fencedev_name }) + +def validate_auth(self, request): + from types import ListType + from ricci_communicator import RicciCommunicator + from HelperFunctions import send_batch_parallel + + fvars = GetReqVars(request, [ 'numStorage' ]) + + num_storage = fvars['numStorage'] + form = None + try: + form = request.form + if not form: + raise Exception, 'no form was submitted' + num_storage = int(num_storage) + except Exception, e: + return (False, { 'errors': [ 'Unable to determine the number of systems given' ] }) + + if num_storage < 0: + return (False, { 'errors': [ 'A negative number of systems was given' ] }) + if num_storage < 1: + return (False, { 'errors': [ 'No systems were given' ] }) + + host_trips = [] + for i in map(lambda x: '__SYSTEM%d' % x, xrange(num_storage)): + cur_system = form.get(i) + if not cur_system or type(cur_system) != ListType or not cur_system[0]: + continue + cur_system[0] = cur_system[0].strip() + if not cur_system[0]: + continue + host_trips.append((cur_system[0], RicciCommunicator.auth, [ cur_system[1] ])) + + ret = send_batch_parallel(host_trips, 10) + + ret_code = False + system_list = list() + for i in ret.keys(): + r = { 'hostname': i } + try: + cur_system = ret[i] + rc = cur_system.get('ricci') + if rc: + r['available'] = True + r['authed'] = rc.authed() + r['trusted'] = rc.trusted() + r['hostname'] = rc.hostname() + r['os'] = rc.os() + r['system_name'] = rc.system_name() + r['cluname'] = rc.cluster_info()[0] + r['clualias'] = rc.cluster_info()[1] + r['key_fp'] = rc.fingerprint()[1] + ret_code = True + else: + r['hostname'] = i + r['available'] = False + r['authed'] = False + r['err_msg'] = cur_system['err_msg'] + except Exception, e: + if LUCI_DEBUG_MODE is True: + luci_log.debug_verbose('VALAUTH0: %s: %r %s' % (i, e, str(e))) + system_list.append(r) + return (ret_code, {'clusternodes': system_list }) --- conga/luci/site/luci/Extensions/LuciZopeAsync.py 2008/01/02 21:00:31 1.5 +++ conga/luci/site/luci/Extensions/LuciZopeAsync.py 2008/01/22 15:05:17 1.6 @@ -99,7 +99,7 @@ fvars = GetReqVars(request, [ 'QUERY_STRING' ]) if fvars['QUERY_STRING'] is None: if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('GCNA: No query string was given') + luci_log.debug_verbose('GCNA0: No query string was given') write_err_async(request, 'No node names were given') return None @@ -126,10 +126,9 @@ continue except Exception, e: if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('GCNA0: ricci: %s: %r %s' \ + luci_log.debug_verbose('GCNA1: ricci: %s: %r %s' \ % (node_host, e, str(e))) - errors.append('Unable to communicate with the ricci agent on %s' \ - % node_host) + errors.append('Unable to communicate with the ricci agent on %s: %s' % (node_host, str(e))) continue try: @@ -139,11 +138,11 @@ raise Exception, 'no nodes' except Exception, e: if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('GCNA1: ricci: %s: %r %s' \ + luci_log.debug_verbose('GCNA2: ricci: %s: %r %s' \ % (node_host, e, str(e))) - errors.append('Unable to retrieve a list of cluster nodes from %s' \ - % node_host) + errors.append('Unable to retrieve a list of cluster nodes from %s: %s' % (node_host, str(e))) continue + ret[cluster_name] = { 'cluster': cluster_name, 'num_nodes': len(node_names), @@ -158,9 +157,11 @@ from HelperFunctions import get_system_info fvars = GetReqVars(request, [ 'QUERY_STRING' ]) + query_str = fvars['QUERY_STRING'] + try: - nodes = fvars['QUERY_STRING'].split('&') - node_list = map(lambda x: x[1], filter(lambda x: x[0][:4] == 'node', map(lambda x: x.split('='), nodes))) + nodes = query_str.split('&') + node_list = map(lambda x: (x[1], None), filter(lambda x: x[0][:4] == 'node', map(lambda x: x.split('='), nodes))) if not node_list or len(node_list) < 1: raise Exception, 'No node list' except Exception, e: @@ -174,15 +175,21 @@ ret = get_system_info(self, node_list) except Exception, e: if LUCI_DEBUG_MODE is True: - luci_log.debug_verbose('GNFPA %r: %r %s' \ - % (request['nodenames'], e, str(e))) - write_err_async(request, 'Error retrieving information') + luci_log.debug_verbose('GSIA1: %r %s' % (e, str(e))) + write_err_async(request, 'Error retrieving information: %s' % str(e)) return None - xml_obj = result_to_xml((True, { 'result': ret })) + + xml_obj = result_to_xml((True, {'clusternodes':ret})) + write_xml_resp(request, xml_obj) + +def validate_auth_async(self, request): + from LuciValidation import validate_auth + ret = validate_auth(self, request) + xml_obj = result_to_xml(ret) write_xml_resp(request, xml_obj) def validate_clusvc_async(self, request): from LuciValidation import validate_clusvc_add ret = validate_clusvc_add(self, request) - xml_obj = result_to_xml((True, { 'result': ret })) + xml_obj = result_to_xml(ret) write_xml_resp(request, xml_obj) --- conga/luci/site/luci/Extensions/LuciZopeExternal.py 2008/01/15 04:41:40 1.6 +++ conga/luci/site/luci/Extensions/LuciZopeExternal.py 2008/01/22 15:05:17 1.7 @@ -54,4 +54,5 @@ from system_adapters import get_sys_svc_list, validate_manage_svc -from LuciZopeAsync import get_cluster_nodes_async, get_sysinfo_async, validate_clusvc_async +from LuciZopeAsync import get_cluster_nodes_async, get_sysinfo_async, \ + validate_clusvc_async --- conga/luci/site/luci/Extensions/cluster_adapters.py 2008/01/14 20:51:42 1.278 +++ conga/luci/site/luci/Extensions/cluster_adapters.py 2008/01/22 15:05:17 1.279 @@ -15,7 +15,8 @@ from homebase_adapters import parseHostForm from LuciClusterActions import propagateClusterConfAsync -from LuciZopeAsync import validate_clusvc_async +from LuciZopeAsync import validate_clusvc_async, get_sysinfo_async, \ + validate_auth_async from LuciClusterInfo import getClusterInfo, \ getModelBuilder, LuciExtractCluModel @@ -1207,7 +1208,9 @@ 60: validate_xvm_key_dist, 61: validate_stop_waiting, 80: process_cluster_conf_editor, - 1001: validate_clusvc_async + 1001: validate_clusvc_async, + 1002: get_sysinfo_async, + 1003: validate_auth_async, } def validatePost(self, request): --- conga/luci/site/luci/Extensions/homebase_adapters.py 2008/01/15 04:41:40 1.55 +++ conga/luci/site/luci/Extensions/homebase_adapters.py 2008/01/22 15:05:18 1.56 @@ -240,10 +240,10 @@ request.SESSION.set('add_cluster_initial', cur_entry) luci_log.info('SECURITY: %s' % errmsg) return (False, { 'errors': [ errmsg ] }) - if trust_shown is True and cur_host_trusted is False: - errmsg = 'You must elect to trust "%s" or abort the addition of the cluster to Luci.' % cur_host - request.SESSION.set('add_cluster_initial', cur_entry) - return (False, { 'errors': [ errmsg ] }) +# if trust_shown is True and cur_host_trusted is False: +# errmsg = 'You must elect to trust "%s" or abort the addition of the cluster to Luci.' % cur_host +# request.SESSION.set('add_cluster_initial', cur_entry) +# return (False, { 'errors': [ errmsg ] }) rc.trust() except Exception, e: if LUCI_DEBUG_MODE is True: @@ -532,6 +532,7 @@ except: pass + cluster_name = '[unknown]' try: cluster_name = request.form['clustername'].strip() except: @@ -539,6 +540,7 @@ luci_log.debug_verbose('VAC0: no cluster name') errors.append('No cluster name was given.') + cluster_os = '[unknown]' try: cluster_os = request.form['cluster_os'].strip() except: @@ -617,7 +619,7 @@ if cur_cluster_name: err_msg = 'Node %s reports it is in cluster "%s" and we expect "%s"' \ - % (cur_host, cur_cluster_name % cluster_name) + % (cur_host, cur_cluster_name, cluster_name) else: err_msg = 'Node %s reports it is not a member of any cluster' % cur_host