From mboxrd@z Thu Jan 1 00:00:00 1970 From: rmccabe@sourceware.org Date: 4 May 2007 19:10:27 -0000 Subject: [Cluster-devel] conga/luci/site/luci/Extensions HelperFunction ... Message-ID: <20070504191027.20003.qmail@sourceware.org> List-Id: To: cluster-devel.redhat.com MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit CVSROOT: /cvs/cluster Module name: conga Branch: EXPERIMENTAL Changes by: rmccabe at sourceware.org 2007-05-04 19:10:24 Modified files: luci/site/luci/Extensions: HelperFunctions.py LuciDB.py PropsObject.py StorageReport.py Variable.py cluster_adapters.py conga_constants.py conga_ssl.py homebase_adapters.py ricci_communicator.py storage_adapters.py Added files: luci/site/luci/Extensions: LuciZope.py Removed files: luci/site/luci/Extensions: NodeData.py Log message: Cleanup/refactor, part 2 Patches: http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciZope.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=NONE&r2=1.1.2.1 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.6.4.1&r2=1.6.4.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciDB.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.1.2.1&r2=1.1.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/PropsObject.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.2&r2=1.2.8.1 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/StorageReport.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.23.2.1&r2=1.23.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/Variable.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.4.8.1&r2=1.4.8.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.255.2.1&r2=1.255.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/conga_constants.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.39.2.1&r2=1.39.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/conga_ssl.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.2&r2=1.2.4.1 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/homebase_adapters.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.50.2.1&r2=1.50.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/ricci_communicator.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.25.2.1&r2=1.25.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/storage_adapters.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.9.4.1&r2=1.9.4.2 http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/NodeData.py.diff?cvsroot=cluster&only_with_tag=EXPERIMENTAL&r1=1.1&r2=NONE --- conga/luci/site/luci/Extensions/HelperFunctions.py 2007/05/03 20:16:38 1.6.4.1 +++ conga/luci/site/luci/Extensions/HelperFunctions.py 2007/05/04 19:10:24 1.6.4.2 @@ -1,236 +1,135 @@ -from AccessControl import getSecurityManager -from ricci_communicator import RicciCommunicator, CERTS_DIR_PATH -from conga_constants import PLONE_ROOT +from ricci_communicator import RicciCommunicator import threading -def siteIsSetup(self): - import os - try: - return os.path.isfile('%sprivkey.pem' % CERTS_DIR_PATH) and os.path.isfile('%scacert.pem' % CERTS_DIR_PATH) - except: - pass - return False - -def strFilter(regex, replaceChar, arg): - import re - return re.sub(regex, replaceChar, arg) - -def userAuthenticated(self): - try: - if (isAdmin(self) or getSecurityManager().getUser().has_role('Authenticated', self.restrictedTraverse(PLONE_ROOT))): - return True - except Exception, e: - luci_log.debug_verbose('UA0: %s' % str(e)) - return False - -def isAdmin(self): - try: - return getSecurityManager().getUser().has_role('Owner', self.restrictedTraverse(PLONE_ROOT)) - except Exception, e: - luci_log.debug_verbose('IA0: %s' % str(e)) - return False - -def userIsAdmin(self, userId): - try: - return self.portal_membership.getMemberById(userId).has_role('Owner', self.restrictedTraverse(PLONE_ROOT)) - except Exception, e: - luci_log.debug_verbose('UIA0: %s: %s' % (userId, str(e))) - return False - def resolveOSType(os_str): - if not os_str or os_str.find('Tikanga') != (-1) or os_str.find('FC6') != (-1) or os_str.find('Zod') != (-1): + if not os_str or os_str.find('Tikanga') != (-1) or os_str.find('FC6') != (-1) or os_str.find('Zod') != (-1) or os_str.find('Fedora release 6.9') != (-1): return 'rhel5' else: return 'rhel4' def add_commas(self, str1, str2): - return '%s; %s' % (str1, str2) - -def allowed_systems(self, user, systems): - allowed = [] - sm = getSecurityManager() - user = sm.getUser() - for system in systems: - #Does this take too long? - if user.has_permission('View', system[1]): - allowed.append(system) - return allowed - -def access_to_host_allowed(self, hostname, allowed_systems_list): - for system in allowed_systems_list: - if system[0] == hostname: - if len(self.allowed_systems(None, [system])) == 1: - return True - else: - return False - return False - - + return '%s; %s' % (str1, str2) class Worker(threading.Thread): - def __init__(self, - mutex, - hosts, - riccis): - threading.Thread.__init__(self) - self.mutex = mutex - self.hosts = hosts - self.riccis = riccis - return - def run(self): - while True: - self.mutex.acquire() - if len(self.hosts) == 0: - self.mutex.release() - return - host = self.hosts.pop() - self.mutex.release() - r = None - try: - r = RicciCommunicator(host) - #print host, 'done' - except Exception, e: - #print host, 'failed', str(e) - pass - except: - #print host, 'failed' - pass - self.mutex.acquire() - self.riccis[host] = r - self.mutex.release() - - - -# removes systems that user is not authorized access to -def get_systems_statuses(self, systems, from_cache=False): - CACHED_INDEX = '_get_systems_statuses()_cached_result_' - session = self.REQUEST.SESSION - if session.has_key(CACHED_INDEX): - res = session[CACHED_INDEX] - if res != None: - session.set(CACHED_INDEX, None) - if from_cache: - return res - pass - - ass = self.allowed_systems(self, None, systems) - - mutex = threading.RLock() - hive = [] # workers - ss = {} # storage systems (will store riccis, and then use them to retrieve real info) - hosts = [] # hostnames - for system in ass: - hosts.append(system[0]) - if len(hosts) < 10: - hive.append(Worker(mutex, hosts, ss)) - - for bee in hive: - bee.start() - for bee in hive: - bee.join() - - for hostname in ss.keys(): - OS = '' - cluname = '' - cluali = '' - authed = False - ricci = ss[hostname] - if ricci != None: - OS = ricci.os() - cluname = ricci.cluster_info()[0] - cluali = ricci.cluster_info()[1] - authed = ricci.authed() - else: - OS = 'System not accessible' - s = {'hostname' : hostname, - 'OS' : OS, - 'cluname' : cluname, - 'clualias' : cluali, - 'available': ricci != None, - 'authed' : authed} - # replace ricci with system's info - ss[hostname] = s - pass - ss_list = [] - sorted_keys = ss.keys() - sorted_keys.sort() - for name in sorted_keys: - ss_list.append(ss[name]) - - session.set(CACHED_INDEX, ss_list) - - return ss_list - - - - -def set_persistant_var(self, - var_name, - default_value): - request = self.REQUEST - response = request.RESPONSE - session = request.SESSION - - # TODO: add username into cookie_prefix, so users don't overwrite each other - cookie_prefix = '__luci_storage_cookie_' - - value = default_value - if request.has_key(var_name): - value = request[var_name] - elif session.has_key(var_name): - value = session[var_name] - elif request.cookies.has_key(cookie_prefix + var_name): - value = request.cookies[cookie_prefix + var_name] - - session.set(var_name, - value) - response.setCookie(cookie_prefix + var_name, - value, - expires='Tue, 30 Jun 2060 12:00:00 GMT') - return value - - + def __init__(self, mutex, hosts, riccis): + threading.Thread.__init__(self) + self.mutex = mutex + self.hosts = hosts + self.riccis = riccis + + def run(self): + while True: + self.mutex.acquire() + if len(self.hosts) == 0: + self.mutex.release() + return + host = self.hosts.pop() + self.mutex.release() + r = None + + try: + r = RicciCommunicator(host) + except Exception, e: + #print host, 'failed', str(e) + pass + except: + #print host, 'failed' + pass + + self.mutex.acquire() + self.riccis[host] = r + self.mutex.release() + +def get_system_info(self, system_list): + mutex = threading.RLock() + hive = [] # workers + ss = {} # storage systems (will store riccis and then use them to retrieve real info) + hosts = [] # hostnames + + for system in system_list: + hosts.append(system[0]) + if len(hosts) < 10: + hive.append(Worker(mutex, hosts, ss)) + + for bee in hive: + bee.start() + + for bee in hive: + bee.join() + + for hostname in ss.keys(): + OS = '' + cluname = '' + cluali = '' + key_fp = '' + trusted = '' + authed = False + trusted = False + ricci = ss[hostname] + + if ricci != None: + OS = ricci.os() + cluname = ricci.cluster_info()[0] + cluali = ricci.cluster_info()[1] + authed = ricci.authed() + key_fp = ricci.fingerprint() + trusted = ricci.trusted() + else: + OS = 'System not accessible' + + s = { + 'hostname' : hostname, + 'OS' : OS, + 'cluname' : cluname, + 'key_fp' : key_fp, + 'clualias' : cluali, + 'available' : ricci != None, + 'trusted' : trusted, + 'authed' : authed + } + + # replace ricci with system's info + ss[hostname] = s + + ss_list = [] + sorted_keys = ss.keys() + sorted_keys.sort() + + for name in sorted_keys: + ss_list.append(ss[name]) + return ss_list +def convert_bytes(bytes, units): + c = int(bytes) / get_units_multiplier(units) + c = repr(c) + c = c[:c.find('.') + 3] + return c # returns (str(float), units) that fits best, # ignores prefered units -def bytes_to_value_units(bytes): - units = 'TB' - if float(convert_bytes(bytes, 'GB')) < 1024: - units = 'GB' - if float(convert_bytes(bytes, 'MB')) < 1024: - units = 'MB' - if float(convert_bytes(bytes, 'KB')) < 1024: - units = 'KB' - if float(convert_bytes(bytes, 'bytes')) < 1024: - units = 'bytes' - return (convert_bytes(bytes, units), units) -# returns (str(float), units) that fits best, -# takes prefered units into account -def bytes_to_value_prefunits(self, bytes): - p_units = self.REQUEST.SESSION.get('preferred_size_units') - dummy, units = bytes_to_value_units(bytes) - if get_units_multiplier(units) > get_units_multiplier(p_units): - units = p_units - return (convert_bytes(bytes, units), units) +def bytes_to_value_units(bytes): + units = 'TB' + if float(convert_bytes(bytes, 'GB')) < 1024: + units = 'GB' + if float(convert_bytes(bytes, 'MB')) < 1024: + units = 'MB' + if float(convert_bytes(bytes, 'KB')) < 1024: + units = 'KB' + if float(convert_bytes(bytes, 'bytes')) < 1024: + units = 'bytes' + return (convert_bytes(bytes, units), units) def get_units_multiplier(units): - if units.lower() == 'bytes': - return 1.0 - elif units.lower() == 'kb': - return 1024.0 - elif units.lower() == 'mb': - return 1024*1024.0 - elif units.lower() == 'gb': - return 1024*1024*1024.0 - elif units.lower() == 'tb': - return 1024*1024*1024*1024.0 - else: - raise Exception, 'invalid size unit' - -def convert_bytes(bytes, units): - c = int(bytes) / get_units_multiplier(units) - c = repr(c) - c = c[:c.find('.') + 3] - return c + if units.lower() == 'bytes': + return 1.0 + elif units.lower() == 'kb': + return 1024.0 + elif units.lower() == 'mb': + return 1024*1024.0 + elif units.lower() == 'gb': + return 1024*1024*1024.0 + elif units.lower() == 'tb': + return 1024*1024*1024*1024.0 + else: + raise Exception, 'Invalid size unit: %s' % units --- conga/luci/site/luci/Extensions/Attic/LuciDB.py 2007/05/03 20:16:38 1.1.2.1 +++ conga/luci/site/luci/Extensions/Attic/LuciDB.py 2007/05/04 19:10:24 1.1.2.2 @@ -2,7 +2,7 @@ from conga_constants import CLUSTER_FOLDER_PATH, BATCH_ID, TASKTYPE, FLAG_DESC, CLUSTER_ADD, CLUSTER_NODE_ADDED, CLUSTER_NODE_NEED_AUTH, LAST_STATUS, PLONE_ROOT, STORAGE_FOLDER_PATH import RicciQuery as rq from ricci_communicator import RicciCommunicator -from HelperFunctions import isAdmin +from LuciZope import isAdmin from homebase_adapters import luci_log --- conga/luci/site/luci/Extensions/PropsObject.py 2006/10/16 04:26:19 1.2 +++ conga/luci/site/luci/Extensions/PropsObject.py 2007/05/04 19:10:24 1.2.8.1 @@ -18,7 +18,6 @@ return self.__vars[name].get_value() else: return None - pass def get_props(self): return self.__vars --- conga/luci/site/luci/Extensions/StorageReport.py 2007/05/03 20:16:38 1.23.2.1 +++ conga/luci/site/luci/Extensions/StorageReport.py 2007/05/04 19:10:24 1.23.2.2 @@ -9,6 +9,8 @@ from conga_storage_constants import * from HelperFunctions import * +from LuciZope import bytes_to_value_prefunits, get_systems_statuses + from ricci_communicator import get_ricci_communicator, batch_status, extract_module_status @@ -203,11 +205,6 @@ return None - pass - - - - def is_storage_report_cached(session, storagename): if session.has_key(SESSION_STORAGE_XML_REPORT): reports_dir = session[SESSION_STORAGE_XML_REPORT] @@ -594,12 +591,6 @@ mdata['targets_all'] = targets_all - pass - - - - - def get_mapper_template_data(self, storage_report, mapper_type): if self == None or storage_report == None or mapper_type == None: return None @@ -719,7 +710,6 @@ if sources_num < int(data['min_sources']) or sources_num > int(data['max_sources']): return 'BAD: Invalid number of %s selected' % data['pretty_sources_name'] props = data['props'] - pass elif object_type == 'add_sources': data = get_mapper_data(self, storage_report, mapper_id) # find sources @@ -730,8 +720,7 @@ sources_num += 1 if sources_num == 0 or sources_num > len(data['new_sources']): return 'BAD: Invalid number of %s selected' % data['pretty_sources_name'] - pass - + if props != None: res = check_props(self, props, request) if res[0] == False: @@ -863,7 +852,6 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass elif action_type == 'Apply': @@ -985,8 +973,6 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass - elif object_type == 'bd_template': @@ -1115,7 +1101,6 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass @@ -1142,7 +1127,6 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass elif action_type == 'Apply': @@ -1197,8 +1181,7 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass - + elif object_type == 'mapper_template': @@ -1270,7 +1253,6 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass @@ -1311,8 +1293,6 @@ res = ricci.process_batch(batch, True) batch_id = res.getAttribute('batch_id') - pass - pass if batch_id == '': @@ -1496,7 +1476,6 @@ vars[var.get_name()] = var except: pass - pass mapper_id = '' @@ -1724,7 +1703,6 @@ if not mutable and d_type != 'label' and d_type != 'hidden': d_type = 'label' d_value = str(value) - pass hidden = False if type == 'hidden' or name in ['partition_begin', 'snapshot']: @@ -1742,7 +1720,6 @@ 'units' : d_units, 'validation' : validation_data, 'hidden' : hidden} - pass return (removable, data) @@ -1915,7 +1892,6 @@ name = 'Extended Partition' else: name = "Unknown type" - pass return name, id --- conga/luci/site/luci/Extensions/Variable.py 2007/05/03 20:16:38 1.4.8.1 +++ conga/luci/site/luci/Extensions/Variable.py 2007/05/04 19:10:24 1.4.8.2 @@ -77,7 +77,6 @@ class Variable: - def __init__(self, name, value, mods={}): self.__name = str(name) self.__mods = mods @@ -88,6 +87,7 @@ def get_value(self): return self.__value + def set_value(self, value): if self.__is_bool(value): self.__type = VARIABLE_TYPE_BOOL @@ -223,8 +223,8 @@ class VariableList(Variable): - def __init__(self, name, value, mods, list_type): + Variable.__init__(name, value, mods=mods) if list_type != VARIABLE_TYPE_LIST_STR and list_type != VARIABLE_TYPE_LIST_XML: raise Exception, 'invalid list type' #if ! self.__is_list(value): @@ -233,7 +233,6 @@ self.__mods = mods self.__type = list_type self.__value = value - pass def get_name(self): return self.__name @@ -268,6 +267,3 @@ for mod in self.__mods: elem.setAttribute(str(mod), str(self.__mods[mod])) return elem - - - --- conga/luci/site/luci/Extensions/cluster_adapters.py 2007/05/03 20:16:38 1.255.2.1 +++ conga/luci/site/luci/Extensions/cluster_adapters.py 2007/05/04 19:10:24 1.255.2.2 @@ -25,7 +25,11 @@ from FenceHandler import validateNewFenceDevice, FENCE_OPTS, validateFenceDevice, validate_fenceinstance, FD_VAL_FAIL, FD_VAL_SUCCESS from system_adapters import validate_svc_update -from homebase_adapters import manageCluster, createClusterSystems, havePermCreateCluster, setNodeFlag, delNodeFlag, userAuthenticated, getStorageNode, getClusterNode, delCluster, parseHostForm + +from homebase_adapters import parseHostForm +from LuciZope import userAuthenticated + +from LuciDB import manageCluster, createClusterSystems, havePermCreateCluster, setNodeFlag, delNodeFlag, getStorageNode, getClusterNode, delCluster, buildClusterCreateFlags, noNodeFlagsPresent, resolve_nodename, set_node_flag try: luci_log = LuciSyslog() @@ -693,7 +697,6 @@ dummy_form[str(i.getAttribute('name'))] = str(i.getAttribute('value')) except Exception, e: luci_log.debug_verbose('vSA2: parsing XML: %s' % str(e)) - pass try: res_type = dummy_form['type'].strip() --- conga/luci/site/luci/Extensions/conga_constants.py 2007/05/03 20:16:38 1.39.2.1 +++ conga/luci/site/luci/Extensions/conga_constants.py 2007/05/04 19:10:24 1.39.2.2 @@ -142,5 +142,5 @@ # Debugging parameters. Set LUCI_DEBUG_MODE to 1 and LUCI_DEBUG_VERBOSITY # to >= 2 to get full debugging output in syslog (LOG_DAEMON/LOG_DEBUG). -LUCI_DEBUG_MODE = 0 -LUCI_DEBUG_VERBOSITY = 0 +LUCI_DEBUG_MODE = 1 +LUCI_DEBUG_VERBOSITY = 2 --- conga/luci/site/luci/Extensions/conga_ssl.py 2006/12/21 21:32:00 1.2 +++ conga/luci/site/luci/Extensions/conga_ssl.py 2007/05/04 19:10:24 1.2.4.1 @@ -1,17 +1,13 @@ - - import sys sys.path.append('/usr/lib/luci/ssl') sys.path.append('/usr/lib64/luci/ssl') + import conga_ssl_lib sys.path.remove('/usr/lib/luci/ssl') sys.path.remove('/usr/lib64/luci/ssl') - - # timeouts are in seconds (int) - class SSLSocket: def __init__(self, @@ -21,10 +17,10 @@ self.__id = -1 self.__id = conga_ssl_lib.connect(hostname, port, timeout) self.__hostname = hostname - pass + def __del__(self): self.disconnect() - pass + def disconnect(self): if self.__id != -1: conga_ssl_lib.disconnect(self.__id) @@ -42,7 +38,6 @@ def untrust(self): return conga_ssl_lib.untrust(self.__id) == 1 - def send(self, msg, timeout): conga_ssl_lib.send(self.__id, msg, timeout) def recv(self, timeout): --- conga/luci/site/luci/Extensions/homebase_adapters.py 2007/05/03 20:16:38 1.50.2.1 +++ conga/luci/site/luci/Extensions/homebase_adapters.py 2007/05/04 19:10:24 1.50.2.2 @@ -5,6 +5,15 @@ from LuciSyslog import LuciSyslog from HelperFunctions import resolveOSType +from LuciDB import delCluster, delNodeFlag, delSystem, getClusterNode, \ + getClusters, getStorage, getStorageNode, \ + havePermAddCluster, havePermAddUser, havePermDelUser, \ + havePermEditPerms, havePermRemCluster, havePermRemStorage, \ + havePermAddStorage, manageCluster + +from ricci_communicator import RicciCommunicator + + # Homebase area page types HOMEBASE_ADD_USER = '1' HOMEBASE_ADD_SYSTEM = '2' --- conga/luci/site/luci/Extensions/ricci_communicator.py 2007/05/03 20:16:38 1.25.2.1 +++ conga/luci/site/luci/Extensions/ricci_communicator.py 2007/05/04 19:10:24 1.25.2.2 @@ -56,8 +56,6 @@ self.__os = hello.firstChild.getAttribute('os') self.__dom0 = hello.firstChild.getAttribute('xen_host') == 'true' - pass - def hostname(self): luci_log.debug_verbose('RC:hostname: [auth %d] reported hostname = %s' \ % (self.__authed, self.__hostname)) @@ -344,8 +342,6 @@ except Exception, e: luci_log.debug_verbose('RC:GRC0: Error creating a ricci connection to %s: %s' % (hostname, str(e))) return None - pass - def ricci_get_called_hostname(self, ricci): return ricci.hostname() @@ -473,7 +469,6 @@ msg = var.getAttribute('value') return code, msg - pass elif status == '1': return -102, 'module scheduled for execution' elif status == '2': --- conga/luci/site/luci/Extensions/storage_adapters.py 2007/05/03 20:16:38 1.9.4.1 +++ conga/luci/site/luci/Extensions/storage_adapters.py 2007/05/04 19:10:24 1.9.4.2 @@ -8,7 +8,7 @@ from conga_storage_constants import * from ricci_defines import * from Variable import parse_variable -from HelperFunctions import get_systems_statuses +from LuciZope import get_systems_statuses