[Cluster-devel] conga/luci/site/luci/Extensions HelperFunction ...

[rmccabe@sourceware.org <rmccabe@sourceware.org>]

CVSROOT:	/cvs/cluster
Module name:	conga
Changes by:	rmccabe at sourceware.org	2008-01-22 15:05:18

Modified files:
	luci/site/luci/Extensions: HelperFunctions.py LuciValidation.py 
	                           LuciZopeAsync.py LuciZopeExternal.py 
	                           cluster_adapters.py 
	                           homebase_adapters.py 

Log message:
	sync up with my local tree
	- fixes for 252348 and 230462

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&r1=1.13&r2=1.14
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciValidation.py.diff?cvsroot=cluster&r1=1.4&r2=1.5
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciZopeAsync.py.diff?cvsroot=cluster&r1=1.5&r2=1.6
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciZopeExternal.py.diff?cvsroot=cluster&r1=1.6&r2=1.7
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&r1=1.278&r2=1.279
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/homebase_adapters.py.diff?cvsroot=cluster&r1=1.55&r2=1.56

--- conga/luci/site/luci/Extensions/HelperFunctions.py	2008/01/15 04:41:40	1.13
+++ conga/luci/site/luci/Extensions/HelperFunctions.py	2008/01/22 15:05:17	1.14
@@ -9,8 +9,7 @@
 import threading
 
 def resolveOSType(os_str):
-	rhel5_clu_ver = [ 'Tikanga', 'Zod', 'Moonshine', 'Werewolf' ]
-	if not os_str or os_str in rhel5_clu_ver:
+	if not os_str or os_str.find('Tikanga') != (-1) or os_str.find('FC6') != (-1) or os_str.find('Zod') != (-1) or os_str.find('Moonshine') != (-1) or os_str.find('Werewolf') != (-1):
 		return 'rhel5'
 	else:
 		return 'rhel4'
@@ -18,6 +17,46 @@
 def add_commas(self, str1, str2):
 	return '%s; %s' % (str1, str2)
 
+class PWorker(threading.Thread):
+	# triple := [ host, function, *args ]
+	def __init__(self, mutex, ret, host_triples):
+		threading.Thread.__init__(self)
+		self.mutex = mutex
+		self.ret = ret
+		self.triples = host_triples
+
+	def run(self):
+		while True:
+			self.mutex.acquire()
+			if len(self.triples) == 0:
+				self.mutex.release()
+				return
+			triple = self.triples.pop()
+			self.mutex.release()
+
+			r = { 'ricci': None }
+
+			try:
+				rc = RicciCommunicator(triple[0], enforce_trust=False)
+				rc.trust()
+				r['ricci'] = rc
+
+				if triple[1] is not None:
+					if triple[2]:
+						args = list(triple[2])
+					else:
+						args = list()
+					args.insert(0, rc)
+					r['batch_result'] = triple[1](*args)
+			except Exception, e:
+				r['error'] = True
+				r['err_msg'] = str(e)
+				r['ricci'] = None
+
+			self.mutex.acquire()
+			self.ret[triple[0]] = r
+			self.mutex.release()
+
 class Worker(threading.Thread):
 	def __init__(self, mutex, hosts, riccis, func=None, *args):
 		threading.Thread.__init__(self)
@@ -39,7 +78,8 @@
 			r = { 'ricci': None }
 
 			try:
-				rc = RicciCommunicator(host)
+				rc = RicciCommunicator(host, enforce_trust=False)
+				rc.trust()
 				r['ricci'] = rc
 				try:
 					r['cluster_name'] = rc.cluster_info()[0]
@@ -55,7 +95,8 @@
 					r['batch_result'] = self.query_func(*args)
 			except Exception, e:
 				r['error'] = True
-				r['err_msg'] = '%s' % str(e)
+				r['err_msg'] = str(e)
+				r['ricci'] = None
 
 			self.mutex.acquire()
 			self.riccis[host] = r
@@ -73,6 +114,26 @@
 		num_hosts += 1
 		if num_hosts <= max_threads:
 			threads.append(Worker(mutex, hosts, ret, func, *args))
+
+	for thread in threads:
+		thread.start()
+	for thread in threads:
+		thread.join()
+	return ret
+
+def send_batch_parallel(triples, max_threads):
+	mutex = threading.RLock()
+	threads = list()
+	num_trips = 0
+	trips = list()
+	ret = {}
+
+	for trip in triples:
+		trips.append(trip)
+		num_trips += 1
+		if num_trips <= max_threads:
+			threads.append(PWorker(mutex, ret, triples))
+
 	for thread in threads:
 		thread.start()
 	for thread in threads:
@@ -111,7 +172,7 @@
 			cluname = ricci.cluster_info()[0]
 			cluali = ricci.cluster_info()[1]
 			authed = ricci.authed()
-			key_fp = ricci.fingerprint()
+			key_fp = ricci.fingerprint()[1]
 			trusted = ricci.trusted()
 		else:
 			OS = 'System not accessible'
@@ -124,7 +185,8 @@
 			'clualias'	: cluali,
 			'available'	: ricci is not None,
 			'trusted'	: trusted,
-			'authed'	: authed
+			'authed'	: authed,
+			'err_msg'	: ss[hostname].get('err_msg')
 		}
 
 		# replace ricci with system's info
--- conga/luci/site/luci/Extensions/LuciValidation.py	2008/01/17 16:36:30	1.4
+++ conga/luci/site/luci/Extensions/LuciValidation.py	2008/01/22 15:05:17	1.5
@@ -1189,3 +1189,66 @@
 		return (False, { 'errors': [ 'Error removing fence device %s: %s' \
 										% (fencedev_name, str(e)) ]})
 	return (True, { 'name': fencedev_name })
+
+def validate_auth(self, request):
+	from types import ListType
+	from ricci_communicator import RicciCommunicator
+	from HelperFunctions import send_batch_parallel
+
+	fvars = GetReqVars(request, [ 'numStorage' ])
+
+	num_storage = fvars['numStorage']
+	form = None
+	try:
+		form = request.form
+		if not form:
+			raise Exception, 'no form was submitted'
+		num_storage = int(num_storage)
+	except Exception, e:
+		return (False, { 'errors': [ 'Unable to determine the number of systems given' ] })
+
+	if num_storage < 0:
+		return (False, { 'errors': [ 'A negative number of systems was given' ] })
+	if num_storage < 1:
+		return (False, { 'errors': [ 'No systems were given' ] })
+
+	host_trips = []
+	for i in map(lambda x: '__SYSTEM%d' % x, xrange(num_storage)):
+		cur_system = form.get(i)
+		if not cur_system or type(cur_system) != ListType or not cur_system[0]:
+			continue
+		cur_system[0] = cur_system[0].strip()
+		if not cur_system[0]:
+			continue
+		host_trips.append((cur_system[0], RicciCommunicator.auth, [ cur_system[1] ]))
+
+	ret = send_batch_parallel(host_trips, 10)
+
+	ret_code = False
+	system_list = list()
+	for i in ret.keys():
+		r = { 'hostname': i }
+		try:
+			cur_system = ret[i]
+			rc = cur_system.get('ricci')
+			if rc:
+				r['available'] = True
+				r['authed'] = rc.authed()
+				r['trusted'] = rc.trusted()
+				r['hostname'] = rc.hostname()
+				r['os'] = rc.os()
+				r['system_name'] = rc.system_name()
+				r['cluname'] = rc.cluster_info()[0]
+				r['clualias'] = rc.cluster_info()[1]
+				r['key_fp'] = rc.fingerprint()[1]
+				ret_code = True
+			else:
+				r['hostname'] = i
+				r['available'] = False
+				r['authed'] = False
+				r['err_msg'] = cur_system['err_msg']
+		except Exception, e:
+			if LUCI_DEBUG_MODE is True:
+				luci_log.debug_verbose('VALAUTH0: %s: %r %s' % (i, e, str(e)))
+		system_list.append(r)
+	return (ret_code, {'clusternodes': system_list })
--- conga/luci/site/luci/Extensions/LuciZopeAsync.py	2008/01/02 21:00:31	1.5
+++ conga/luci/site/luci/Extensions/LuciZopeAsync.py	2008/01/22 15:05:17	1.6
@@ -99,7 +99,7 @@
 	fvars = GetReqVars(request, [ 'QUERY_STRING' ])
 	if fvars['QUERY_STRING'] is None:
 		if LUCI_DEBUG_MODE is True:
-			luci_log.debug_verbose('GCNA: No query string was given')
+			luci_log.debug_verbose('GCNA0: No query string was given')
 		write_err_async(request, 'No node names were given')
 		return None
 
@@ -126,10 +126,9 @@
 				continue
 		except Exception, e:
 			if LUCI_DEBUG_MODE is True:
-				luci_log.debug_verbose('GCNA0: ricci: %s: %r %s' \
+				luci_log.debug_verbose('GCNA1: ricci: %s: %r %s' \
 					% (node_host, e, str(e)))
-			errors.append('Unable to communicate with the ricci agent on %s' \
-				% node_host)
+			errors.append('Unable to communicate with the ricci agent on %s: %s' % (node_host, str(e)))
 			continue
 
 		try:
@@ -139,11 +138,11 @@
 				raise Exception, 'no nodes'
 		except Exception, e:
 			if LUCI_DEBUG_MODE is True:
-				luci_log.debug_verbose('GCNA1: ricci: %s: %r %s' \
+				luci_log.debug_verbose('GCNA2: ricci: %s: %r %s' \
 					% (node_host, e, str(e)))
-			errors.append('Unable to retrieve a list of cluster nodes from %s' \
-				% node_host)
+			errors.append('Unable to retrieve a list of cluster nodes from %s: %s' % (node_host, str(e)))
 			continue
+
 		ret[cluster_name] = {
 			'cluster': cluster_name,
 			'num_nodes': len(node_names),
@@ -158,9 +157,11 @@
 	from HelperFunctions import get_system_info
 
 	fvars = GetReqVars(request, [ 'QUERY_STRING' ])
+	query_str = fvars['QUERY_STRING']
+
 	try:
-		nodes = fvars['QUERY_STRING'].split('&')
-		node_list = map(lambda x: x[1], filter(lambda x: x[0][:4] == 'node', map(lambda x: x.split('='), nodes)))
+		nodes = query_str.split('&')
+		node_list = map(lambda x: (x[1], None), filter(lambda x: x[0][:4] == 'node', map(lambda x: x.split('='), nodes)))
 		if not node_list or len(node_list) < 1:
 			raise Exception, 'No node list'
 	except Exception, e:
@@ -174,15 +175,21 @@
 		ret = get_system_info(self, node_list)
 	except Exception, e:
 		if LUCI_DEBUG_MODE is True:
-			luci_log.debug_verbose('GNFPA %r: %r %s' \
-				% (request['nodenames'], e, str(e)))
-		write_err_async(request, 'Error retrieving information')
+			luci_log.debug_verbose('GSIA1: %r %s' % (e, str(e)))
+		write_err_async(request, 'Error retrieving information: %s' % str(e))
 		return None
-	xml_obj = result_to_xml((True, { 'result': ret }))
+
+	xml_obj = result_to_xml((True, {'clusternodes':ret}))
+	write_xml_resp(request, xml_obj)
+
+def validate_auth_async(self, request):
+	from LuciValidation import validate_auth
+	ret = validate_auth(self, request)
+	xml_obj = result_to_xml(ret)
 	write_xml_resp(request, xml_obj)
 
 def validate_clusvc_async(self, request):
 	from LuciValidation import validate_clusvc_add
 	ret = validate_clusvc_add(self, request)
-	xml_obj = result_to_xml((True, { 'result': ret }))
+	xml_obj = result_to_xml(ret)
 	write_xml_resp(request, xml_obj)
--- conga/luci/site/luci/Extensions/LuciZopeExternal.py	2008/01/15 04:41:40	1.6
+++ conga/luci/site/luci/Extensions/LuciZopeExternal.py	2008/01/22 15:05:17	1.7
@@ -54,4 +54,5 @@
 
 from system_adapters import get_sys_svc_list, validate_manage_svc
 
-from LuciZopeAsync import get_cluster_nodes_async, get_sysinfo_async, validate_clusvc_async
+from LuciZopeAsync import get_cluster_nodes_async, get_sysinfo_async, \
+	validate_clusvc_async
--- conga/luci/site/luci/Extensions/cluster_adapters.py	2008/01/14 20:51:42	1.278
+++ conga/luci/site/luci/Extensions/cluster_adapters.py	2008/01/22 15:05:17	1.279
@@ -15,7 +15,8 @@
 from homebase_adapters import parseHostForm
 from LuciClusterActions import propagateClusterConfAsync
 
-from LuciZopeAsync import validate_clusvc_async
+from LuciZopeAsync import validate_clusvc_async, get_sysinfo_async, \
+	validate_auth_async
 
 from LuciClusterInfo import getClusterInfo, \
 	getModelBuilder, LuciExtractCluModel
@@ -1207,7 +1208,9 @@
 	60: validate_xvm_key_dist,
 	61: validate_stop_waiting,
 	80: process_cluster_conf_editor,
-	1001: validate_clusvc_async 
+	1001: validate_clusvc_async,
+	1002: get_sysinfo_async,
+	1003: validate_auth_async,
 }
 
 def validatePost(self, request):
--- conga/luci/site/luci/Extensions/homebase_adapters.py	2008/01/15 04:41:40	1.55
+++ conga/luci/site/luci/Extensions/homebase_adapters.py	2008/01/22 15:05:18	1.56
@@ -240,10 +240,10 @@
 				request.SESSION.set('add_cluster_initial', cur_entry)
 				luci_log.info('SECURITY: %s' % errmsg)
 				return (False, { 'errors': [ errmsg ] })
-			if trust_shown is True and cur_host_trusted is False:
-				errmsg = 'You must elect to trust "%s" or abort the addition of the cluster to Luci.' % cur_host
-				request.SESSION.set('add_cluster_initial', cur_entry)
-				return (False, { 'errors': [ errmsg ] })
+#			if trust_shown is True and cur_host_trusted is False:
+#				errmsg = 'You must elect to trust "%s" or abort the addition of the cluster to Luci.' % cur_host
+#				request.SESSION.set('add_cluster_initial', cur_entry)
+#				return (False, { 'errors': [ errmsg ] })
 			rc.trust()
 		except Exception, e:
 			if LUCI_DEBUG_MODE is True:
@@ -532,6 +532,7 @@
 	except:
 		pass
 
+	cluster_name = '[unknown]'
 	try:
 		cluster_name = request.form['clustername'].strip()
 	except:
@@ -539,6 +540,7 @@
 			luci_log.debug_verbose('VAC0: no cluster name')
 		errors.append('No cluster name was given.')
 
+	cluster_os = '[unknown]'
 	try:
 		cluster_os = request.form['cluster_os'].strip()
 	except:
@@ -617,7 +619,7 @@
 
 				if cur_cluster_name:
 					err_msg = 'Node %s reports it is in cluster "%s" and we expect "%s"' \
-						% (cur_host, cur_cluster_name % cluster_name)
+						% (cur_host, cur_cluster_name, cluster_name)
 				else:
 					err_msg = 'Node %s reports it is not a member of any cluster' % cur_host