[Cluster-devel] conga/luci/site/luci/Extensions homebase_adapt ...

[rmccabe@sourceware.org <rmccabe@sourceware.org>]

CVSROOT:	/cvs/cluster
Module name:	conga
Changes by:	rmccabe at sourceware.org	2006-07-18 19:26:04

Modified files:
	luci/site/luci/Extensions: homebase_adapters.py 

Log message:
	pull in clusters using cluster.conf from a cluster node

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/homebase_adapters.py.diff?cvsroot=cluster&r1=1.12&r2=1.13

--- conga/luci/site/luci/Extensions/homebase_adapters.py	2006/07/05 20:13:03	1.12
+++ conga/luci/site/luci/Extensions/homebase_adapters.py	2006/07/18 19:26:03	1.13
@@ -16,6 +16,7 @@
 HOMEBASE_DEL_USER="4"
 HOMEBASE_DEL_SYSTEM="5"
 HOMEBASE_ADD_CLUSTER="6"
+HOMEBASE_ADD_CLUSTER_INITIAL="7"
 
 PLONE_ROOT='luci'
 
@@ -37,9 +38,6 @@
 	errors = list()
 	messages = list()
 
-	sys.stdout = sys.stderr
-	print request.form
-
 	if '__SYSTEM' in request.form:
 		sysNames = request.form['__SYSTEM']
 		for i in sysNames:
@@ -136,50 +134,258 @@
 	messages.append('Added new user \"' + user + '\" successfully')
 	return (True, {'messages': messages, 'params': { 'user': user }})
 
+def nodeUnauth(nodeList):
+	for i in nodeList:
+		try:
+			if i['prev_auth'] == True:
+				host = i['host']
+				rc = RicciCommunicator(host)
+				rc.unauth()
+		except:
+			pass
+
+def nodeAuth(cluster, host, passwd):
+	systemName = host
+
+	try:
+		rc = RicciCommunicator(host)
+		if not rc:
+			raise
+		systemName = rc.system_name()
+	except:
+		error = 'Unable to establish a connection to the ricci agent on \"' + host + '\"'
+		return { 'host': host, 'ricci_host': systemName, 'errors': error, 'cur_auth': False }
+
+
+	if rc.authed():
+		prevAuth = True
+	else:
+		prevAuth = False
+		if not passwd:
+			return { 'host': host, 'ricci_host': systemName, 'prev_auth': False, 'cur_auth': False }
+		else:
+			try:
+				rc.auth(passwd)
+			except: pass
+
+	if rc.authed():
+		systemName = rc.system_name()
+		if systemName[:9] == 'localhost' or systemName[:5] == '127.0':
+			systemName = host
+		node = { 'host': host, 'ricci_host': systemName, 'prev_auth': prevAuth, 'cur_auth': True }
+		cluster_info = rc.cluster_info()
+		if not cluster_info or cluster_info[0] != cluster:
+			node['errors'] = 'Node ' + host + ' is reporting it is not a member of cluster \"' + cluster + '\"'
+		return node
+
+	error = 'Unable to authenticate to the ricci agent on \"' + host + '\"'
+	return { 'host': host, 'ricci_host': systemName, 'prev_auth': False , 'cur_auth': False, 'errors': error }
+
+def validateAddClusterInitial(self, request):
+	errors = list()
+	newNodeList = list()
+	nodeHash = {}
+	rnodeHash = {}
+
+	try:
+		sysData = request.form['__SYSTEM0']
+		if not sysData or len(sysData) < 2:
+			raise
+	except:
+		return (False, { 'errors': [ 'At least one system and its root password must be given' ] })
+
+	try:
+		rc = RicciCommunicator(sysData[0])
+	except:
+		return (False, { 'errors': [ 'Unable to establish a connection to the Ricci agent on \"' + sysData[0] + '\"' ] })
+
+	prevAuth = 0
+	if not rc.authed():
+		try:
+			rc.auth(sysData[1])
+		except: pass
+		if not rc.authed():
+			return (False, { 'errors': [ 'Unable to authenticate to the Ricci agent on \"' + sysData[0] + '\"' ] })
+	else:
+		prevAuth = 1
+
+	try:
+		cluster_info = rc.cluster_info()
+	except:
+		cluster_info = None
+
+	if not cluster_info or not cluster_info[0]:
+		if not prevAuth:
+			rc.unauth()
+		if not cluster_info:
+			errmsg = 'An error occurred while attempting to retrieve the cluster.conf file for \"' + sysData[0] + '\"'
+		else:
+			errmsg = '\"' + sysData[0] + '\" is not a member of a cluster'
+		return (False, { 'errors': [ errmsg ] })
+
+	clusterName = cluster_info[0]
+	cluConf = getClusterConf(rc)
+	if cluConf:
+		nodeList = getClusterConfNodes(cluConf)
+
+	if not cluConf or not nodeList or len(nodeList) < 2:
+		if not prevAuth:
+			rc.unauth()
+		return (False, { 'errors': [ 'Error retrieving member nodes for cluster \"' + clusterName + '\"' ] })
+
+	systemName = rc.system_name()
+	if systemName[:9] == 'localhost':
+		systemName = sysData[0]
+
+	node = { 'host': sysData[0], 'ricci_host': systemName, 'prev_auth': prevAuth, 'cur_auth': rc.authed() }
+	nodeHash[sysData[0]] = node
+	rnodeHash[systemName] = node
+	newNodeList.append(node)
+
+	if 'allSameCheckBox' in request.form:
+		passwd = sysData[1]
+	else:
+		passwd = None
+		
+	for i in nodeList:
+		node = nodeAuth(clusterName, i, passwd)
+		if node['host'] in nodeHash or node['ricci_host'] in rnodeHash:
+			continue
+		nodeHash[node['host']] = node
+		if 'ricci_host' in node:
+			rnodeHash[node['ricci_host']] = node
+
+		if 'errors' in node:
+			errors.append(node['errors'])
+			node['errors'] = True
+		newNodeList.append(node)
+
+	sfn = lambda x, y: \
+		x['cur_auth'] - y['cur_auth'] or (('errors' in y) - ('errors' in x)) 
+	newNodeList.sort(sfn)
+
+	dfn = lambda x: not 'cur_auth' in x or x['cur_auth'] != True
+
+	cluster_properties = {
+		'clusterName': clusterName,
+		'nodeList': newNodeList,
+		'nodeHash': nodeHash,
+		'rnodeHash': rnodeHash,
+		'isComplete': len(filter(dfn, newNodeList)) == 0
+	}
+
+	if len(errors) < 1:
+		cluster_properties['redirect'] = HOMEBASE_ADD_CLUSTER
+
+	return (len(errors) < 1,
+		{'errors': errors, 'requestResults': cluster_properties })
+
 def validateAddCluster(self, request):
 	errors = list()
 	messages = list()
+	requestResults = None
+	nodeList = None
 
-	if not 'clusterName' in request.form:
-		return (False, ['No cluster name given'])
-
-	# This should never fail
 	try:
-		numStorage = request.form['numStorage']
+		sessionData = request.SESSION.get('checkRet')
+		requestResults = sessionData['requestResults']
 	except:
-		return (False, {'errors': ['Unknown number of systems entered']})
+		return (False, { 'errors': [ 'A data integrity error has occurred. Please attempt adding the cluster again.' ], 'requestResults': { 'clusterName': clusterName } })
+		
+	if not 'clusterName' in request.form or request.form['clusterName'] == '':
+		return (False, { 'errors': ['No cluster name given'], 'requestResults': requestResults })
 
 	clusterName = request.form['clusterName']
-	if clusterName == '':
-		return (False, ['No cluster name given'])
+	nodeList = requestResults['nodeList']
+	nodeHash = requestResults['nodeHash']
+	rnodeHash = requestResults['rnodeHash']
 
-	cclRet = createCluster(self, clusterName)
-	if cclRet:
-		return (False, {'errors': [cclRet] })
-	else:
-		messages.append('Created cluster \"' + clusterName + '\" successfully')
+	# This should never fail
+	try:
+		numStorage = int(request.form['numStorage'])
+		if numStorage != len(nodeList):
+			raise
+	except:
+		nodeUnauth(nodeList)
+		return (False, {
+				'errors': [ 'Unknown number of nodes entered' ],
+				'requestResults': requestResults })
 
 	i = 0
 	while i < numStorage:
-		try:
-			sysData = request.form['__SYSTEM' + str(i)]
-		except:
-			break
-
-		if len(sysData) == 2 and sysData[0] != '' and sysData[1] != '':
-			csResult = createClusterSystem(self, clusterName, sysData[0], sysData[1])
-			if csResult:
-				errors.append(csResult)
-			else:
-				messages.append('Added storage system \"' + sysData[0] + '\" successfully')
+		sysData = request.form['__SYSTEM' + str(i)]
+		if not sysData:
+			i += 1
+			continue
+
+		oldNode = None
+		node = nodeAuth(clusterName, sysData[0], sysData[1])
+		if node['host'] in nodeHash:
+			oldNode = nodeHash[node['host']]
+		elif 'ricci_host' in node and node['ricci_host'] in rnodeHash:
+			oldNode = rnodeHash[node['ricci_host']]
+		elif not oldNode:
+			for k in nodeHash.keys():
+				if node['host'][:len(k) + 1] == k + '.':
+					oldNode = nodeHash[k]
+		elif not oldNode:
+			for k in rnodeHash.keys():
+				if node['host'][:len(k) + 1] == k + '.':
+					oldNode = rnodeHash[k]
+
+		if not oldNode:
+			nodeUnauth(nodeList)
+			return (False, { 'errors': [ 'A data integrity error has occurred. Please attempt adding the cluster again.' ], 'requestResults': { 'clusterName': clusterName } })
+
+		if oldNode['host'] != node['host']:
+			del nodeHash[oldNode['host']]
+			oldNode['host'] = node['host']
+			nodeHash[node['host']] = oldNode
+
+		if 'ricci_host' in node and (not 'ricci_host' in oldNode or node['ricci_host'] != oldNode['ricci_host']):
+			if oldNode['ricci_host'] in rnodeHash:
+				del rnodeHash[oldNode['ricci_host']]
+				oldNode['ricci_host'] = node['ricci_host']
+				rnodeHashnode[node['ricci_host']] = oldNode
+
+		oldNode['cur_auth'] = node['cur_auth']
+		if 'errors' in node:
+			errors.append(node['errors'])
+			oldNode['errors'] = True
 		i += 1
 
-	if len(errors) > 0:
-		retCode = False
+	dfn = lambda x: not 'cur_auth' in x or x['cur_auth'] != True
+	clusterComplete = len(filter(dfn, nodeList)) == 0
+
+	if clusterComplete:
+		err = manageCluster(self, clusterName, nodeList)
+		if err:
+			errors.append(err)
+		else:
+			messages.append('Cluster \"' + clusterName + '\" has been added to the Luci management interface.') 
 	else:
-		retCode = True
+		sfn = lambda x, y: \
+			x['cur_auth'] - y['cur_auth'] or (('errors' in y) - ('errors' in x)) 
+		nodeList.sort(sfn)
+
+	ret = { 'messages': messages, 'errors': errors }
+
+	if len(errors) > 0 or not clusterComplete:
+		ret['requestResults'] = {
+			'clusterName': clusterName,
+			'nodeList': nodeList,
+			'nodeHash': nodeHash,
+			'rnodeHash': rnodeHash,
+			'isComplete': clusterComplete
+		}
+	else:
+		ret['requestResults'] = {
+			'redirect': HOMEBASE_ADD_CLUSTER_INITIAL,
+			'clusterName': clusterName,
+			'isComplete': True
+		}
 
-	return (retCode, {'messages': messages, 'errors': errors })
+	return (len(errors) < 1, ret)
 
 def validateAddSystem(self, request):
 	errors = list()
@@ -188,7 +394,7 @@
 	try:
 		numStorage = request.form['numStorage']
 	except:
-		return (False, {'errors': ['Unknown number of systems entered']})
+		return (False, { 'errors': ['Unknown number of systems entered'] })
 
 	i = 0
 	while i < numStorage:
@@ -200,22 +406,8 @@
 		if len(sysData) == 2 and sysData[0] != '' and sysData[1] != '':
 			try:
 				csResult = createSystem(self, sysData[0], sysData[1])
-			except InCluster, e:
-				csResult = None
-				host = str(e)
-				rc = RicciCommunicator(e)
-				if not rc.authed():
-					rc.auth(sysData[1])
-				try:
-					cluConf = getClusterConf(rc)
-					if cluConf:
-						nodeList = getClusterConfNodes(cluConf)
-						if NodeList:
-							pass
-							# XXX fix this
-							#print "other nodes in",cluster_info[0],"::",nodeList
-				except:
-					pass
+			except:
+				pass
 
 			if csResult:
 				errors.append(csResult)
@@ -341,7 +533,8 @@
 	validatePerms,
 	validateDelUser,
 	validateDelSystem,
-	validateAddCluster
+	validateAddCluster,
+	validateAddClusterInitial
 ]
 
 def userAuthenticated(self):
@@ -365,6 +558,12 @@
 	return False
 
 def homebaseControlPost(self, request):
+	try:
+		sessionData = request.SESSION.get('checkRet')
+	except:
+		sessionData = None
+
+
 	if 'ACTUAL_URL' in request:
 		url = request['ACTUAL_URL']
 	else:
@@ -373,30 +572,42 @@
 	if 'pagetype' in request.form:
 		pagetype = request.form['pagetype']
 	else:
+		try: request.SESSION.set('checkRet', {})
+		except: pass
 		return homebasePortal(self, request, '.', '0')
 
 	try:
 		validatorFn = formValidators[int(request.form['pagetype']) - 1]
 	except:
+		try: request.SESSION.set('checkRet', {})
+		except: pass
 		return homebasePortal(self, request, '.', '0')
-
 	ret = validatorFn(self, request)
 	params = None
+
 	if 'params' in ret[1]:
 		params = ret[1]['params']
 
+	if 'requestResults' in ret[1]:
+		requestResults = ret[1]['requestResults']
+
+		if 'redirect' in requestResults:
+			pagetype = requestResults['redirect']
+			request['pagetype'] = pagetype
+			request.form['pagetype'] = pagetype
+
 	request.SESSION.set('checkRet', ret[1])
 	return homebasePortal(self, request, url, pagetype, params)
 
 def homebaseControl(self, request):
+	if request.REQUEST_METHOD == 'POST':
+		return homebaseControlPost(self, request)
+
 	try:
 		request.SESSION.set('checkRet', {})
 	except:
 		pass
 
-	if request.REQUEST_METHOD == 'POST':
-		return homebaseControlPost(self, request)
-
 	if 'ACTUAL_URL' in request:
 		url = request['ACTUAL_URL']
 	else:
@@ -467,12 +678,34 @@
 			temp.append(addSystem)
 	except: pass
 
+	# Initial add cluster page
+	try:
+		if pagetype == HOMEBASE_ADD_CLUSTER:
+			raise
+		if havePermAddCluster(self):
+			addCluster = {}
+			addCluster['Title'] = 'Manage a Cluster'
+			addCluster['absolute_url'] = url + '?pagetype=' + HOMEBASE_ADD_CLUSTER_INITIAL
+			addCluster['Description'] = 'Manage a Cluster'
+			if pagetype == HOMEBASE_ADD_CLUSTER_INITIAL:
+				addCluster['currentItem'] = True
+				ret['curIndex'] = index
+				cur = addCluster
+			else:
+				addCluster['currentItem'] = False
+			index += 1
+			temp.append(addCluster)
+	except: pass
+
+	# Add cluster - screen 2
 	try:
+		if pagetype != HOMEBASE_ADD_CLUSTER:
+			raise
 		if havePermAddCluster(self):
 			addCluster = {}
-			addCluster['Title'] = 'Add a Cluster'
+			addCluster['Title'] = 'Manage a Cluster'
 			addCluster['absolute_url'] = url + '?pagetype=' + HOMEBASE_ADD_CLUSTER
-			addCluster['Description'] = 'Add a Cluster'
+			addCluster['Description'] = 'Manage a Cluster'
 			if pagetype == HOMEBASE_ADD_CLUSTER:
 				addCluster['currentItem'] = True
 				ret['curIndex'] = index
@@ -490,9 +723,9 @@
 	try:
 		if (havePermRemStorage(self) and havePermRemCluster(self) and (getStorage(self) or getClusters(self))):
 			remSystem = {}
-			remSystem['Title'] = 'Remove a System'
+			remSystem['Title'] = 'Remove a Storage System'
 			remSystem['absolute_url'] = url + '?pagetype=' + HOMEBASE_DEL_SYSTEM
-			remSystem['Description'] = 'Remove a system'
+			remSystem['Description'] = 'Remove a Storage System'
 			if pagetype == HOMEBASE_DEL_SYSTEM:
 				remSystem['currentItem'] = True
 				ret['curIndex'] = index
@@ -529,9 +762,11 @@
 		cur['base_url'] = cur['absolute_url']
 		for i in params:
 			cur['absolute_url'] += '&' + cgi.escape(i) + '=' + cgi.escape(params[i])
+	elif cur and 'absolute_url' in cur:
+		cur['base_url'] = cur['absolute_url']
 	else:
-		if cur and 'absolute_url' in cur:
-			cur['base_url'] = cur['absolute_url']
+		cur = {}
+		cur['base_url'] = '#'
 
 	ret['children'] = temp
 	return ret
@@ -652,21 +887,20 @@
 	except:
 		return 'Unable to set permissions on new system \"' + host + '\"'
 
-#	cluster_info = rc.cluster_info()
-#	if cluster_info[0] != '':
-#		raise InCluster, host
-
+def manageCluster(self, clusterName, nodeList):
+	clusterName = str(clusterName)
 
-def createCluster(self, clusterName):
 	try:
 		clusters = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/')
 	except:
+		nodeUnauth(nodeList)
 		return 'Unable to create cluster \"' + clusterName + '\"'
 
 	try:
 		newCluster = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + clusterName)
 		if newCluster:
-			return 'Cluster \"' + clusterName + '\" is already managed by Luci'
+			nodeUnauth(nodeList)
+			return 'A cluster named \"' + clusterName + '\" is already managed by Luci'
 	except:
 		pass
 
@@ -674,89 +908,55 @@
 		clusters.manage_addFolder(clusterName, '__luci__:cluster')
 		newCluster = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + clusterName)
 	except:
+		nodeUnauth(nodeList)
 		return 'Unable to create cluster \"' + clusterName + '\"'
 
 	try:
 		newCluster.manage_acquiredPermissions([])
 		newCluster.manage_role('View', ['Access Contents Information','View'])
 	except:
-		return 'Unable to set permissions on new cluster \"' + clusterName + '\"'
-
-def createClusterSystem(self, cluster, host, passwd):
-	try:
-		rc = RicciCommunicator(host)
-	except:
-		return 'Unable to establish connection to the ricci agent on \"' + host + '\"'
-
-	try:
-		if not rc.authed():
-			rc.auth(passwd)
-	except:
-		return 'Unable to communicate with the ricci agent on \"' + host + '\" for authentication'
-
-	try:
-		i = rc.authed()
-	except:
-		return 'Unable to authenticate to the ricci agent on \"' + host + '\"'
-
-	if i != True:
-		return 'Authentication for host \"' + host + '\" failed'
-
-#	rhost = rc.system_name()
-#	if rhost and rhost != host:
-#		host = rhost
-
-	cluster_info = rc.cluster_info()
-	if len(cluster_info) > 0 and cluster_info[0] != '' and cluster_info[0] != cluster:
-		return 'The host \"' + host + '\" is already a member of the cluster \"' + cluster_info[0] + '\"'
-
-	try:
-		exists = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + cluster + '/' + host)
-		if exists:
-			return None
-	except:
-		pass
-
-	try:
-		ssystem = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + cluster)
-	except:
-		return 'Unable to create storage system \"' + host + '\" for cluster \"' + cluster + '\": Cannot find cluster'
-
-	try:
-		ssystem.manage_addFolder(host, '__luci__:csystem:' + cluster)
-		newSystem = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + cluster + '/' + host)
-	except:
-		return 'Unable to create storage system \"' + host + '\" for cluster \"' + cluster + '\"'
+		nodeUnauth(nodeList)
+		try: clusters.manage_delObjects([clusterName])
+		except: pass
+		return 'Unable to set permissions on new cluster \"' + clusterName + '\"-- Cluster creation failed'
 
 	try:
 		ssystem = self.restrictedTraverse(PLONE_ROOT + '/systems/storage/')
 	except:
-		return 'Unable to create storage system \"' + host + '\"'
+		ssystem = None
 
-	try:
-		newSystem = self.restrictedTraverse(PLONE_ROOT + '/systems/storage/' + host)
-		# It's already there.
-		return None
-	except:
-		pass
+	for i in nodeList:
+		if 'ricci_host' in i:
+			host = str(i['ricci_host'])
+		else:
+			host = str(i['host'])
 
-	try:
-		ssystem.manage_addFolder(host, '__luci__:system')
-	except:
-		pass
+		try:
+			newCluster.manage_addFolder(host, '__luci__:csystem:' + clusterName)
+			newSystem = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + clusterName + '/' + host)
+			if not newSystem:
+				raise 'not there'
+			newSystem.manage_acquiredPermissions([])
+			newSystem.manage_role('View', [ 'Access contents information' , 'View' ])
+		except Exception, e:
+			nodeUnauth(nodeList)
+			try: clusters.manage_delObjects([clusterName])
+			except: pass
+			return 'Unable to create cluster node \"' + host + '\" for cluster \"' + clusterName + '\" -- Cluster creation failed."'
 
-	try:
-		newSystem = self.restrictedTraverse(PLONE_ROOT + '/systems/storage/' + host)
-		if not newSystem:
-			raise
-	except:
-		return 'Unable to create storage system \"' + host + '\"'
+		if ssystem:
+			try:
+				# It's already there, as a storage system, no problem.
+				exists = self.restrictedTraverse(PLONE_ROOT + '/systems/storage/' + host)
+				continue
+			except: pass
 
-	try:
-		newSystem.manage_acquiredPermissions([])
-		newSystem.manage_role('View', ['Access contents information','View'])
-	except:
-		return 'Unable to set permissions on new storage system \"' + host + '\"'
+			try:
+				ssystem.manage_addFolder(host, '__luci__:system')
+				newSystem = self.restrictedTraverse(PLONE_ROOT + '/systems/storage/' + host)
+				newSystem.manage_acquiredPermissions([])
+				newSystem.manage_role('View', [ 'Access contents information' , 'View' ])
+			except: pass
 
 def delSystem(self, systemName):
 	try:
@@ -798,6 +998,10 @@
 	except:
 		return 'Unable to find cluster \"' + clusterName + '\"'
 
+	err = delClusterSystems(self, clusterName)
+	if err:
+		return err
+
 	try:
 		clusters.manage_delObjects([clusterName])
 	except:
@@ -805,6 +1009,14 @@
 
 def delClusterSystem(self, clusterName, systemName):
 	cluster = self.restrictedTraverse(PLONE_ROOT + '/systems/cluster/' + clusterName)
+	try:
+		if not self.restrictedTraverse(PLONE_ROOT + '/systems/storage/' + systemName):
+			raise
+	except:
+		try:
+			rc = RicciCommunicator(systemName)
+			rc.unauth()
+		except: pass
 	cluster.manage_delObjects([systemName])
 
 def delClusterSystems(self, clusterName):
@@ -817,7 +1029,6 @@
 	errors = ''
 	for i in csystems:
 		try:
-			delSystem(i)
 			cluster.manage_delObjects([i])
 			delClusterSystem(i)
 		except:
@@ -895,7 +1106,15 @@
 	module.appendChild(request)
 	batch.appendChild(module)
 
-	ret = rc.process_batch(batch)
+	# temporary workaround for ricci bug
+	system_info = rc.system_name()
+	rc = RicciCommunicator(system_info)
+	# end workaround
+
+	try:
+		ret = rc.process_batch(batch)
+	except Exception, e:
+		return str(e)
 
 	if not ret:
 		return None