From: rmccabe@sourceware.org <rmccabe@sourceware.org>
To: cluster-devel.redhat.com
Subject: [Cluster-devel] conga/luci cluster/index_html site/luci/Extens ...
Date: 27 Jun 2007 06:19:23 -0000 [thread overview]
Message-ID: <20070627061923.32515.qmail@sourceware.org> (raw)
CVSROOT: /cvs/cluster
Module name: conga
Branch: RHEL5
Changes by: rmccabe at sourceware.org 2007-06-27 06:19:22
Modified files:
luci/cluster : index_html
luci/site/luci/Extensions: HelperFunctions.py LuciDB.py
RicciQueries.py cluster_adapters.py
ricci_communicator.py
luci/storage : index_html
Log message:
Add support for issuing ricci batch jobs on all cluster nodes simultaneously
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/cluster/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.20.2.10&r2=1.20.2.11
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.4.2.3&r2=1.4.2.4
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciDB.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/RicciQueries.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.120.2.31&r2=1.120.2.32
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/ricci_communicator.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.9.2.11&r2=1.9.2.12
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/storage/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.7.2.2&r2=1.7.2.3
--- conga/luci/cluster/index_html 2007/06/18 20:14:55 1.20.2.10
+++ conga/luci/cluster/index_html 2007/06/27 06:19:21 1.20.2.11
@@ -44,7 +44,7 @@
<tal:block tal:condition="firsttime">
<tal:block tal:define="global busywaiting python:True" />
<meta http-equiv="refresh"
- tal:attributes="content isBusy/refreshurl | string:." />
+ tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" />
</tal:block>
<tal:block tal:define="global busy isBusy/busy | nothing" />
@@ -52,7 +52,7 @@
<tal:block tal:condition="busy">
<tal:block tal:define="global busywaiting python:True" />
<meta http-equiv="refresh"
- tal:attributes="content isBusy/refreshurl | string:." />
+ tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" />
</tal:block>
</tal:block>
</metal:headslot>
--- conga/luci/site/luci/Extensions/HelperFunctions.py 2007/06/18 18:39:32 1.4.2.3
+++ conga/luci/site/luci/Extensions/HelperFunctions.py 2007/06/27 06:19:22 1.4.2.4
@@ -18,11 +18,13 @@
return '%s; %s' % (str1, str2)
class Worker(threading.Thread):
- def __init__(self, mutex, hosts, riccis):
+ def __init__(self, mutex, hosts, riccis, func=None, *args):
threading.Thread.__init__(self)
self.mutex = mutex
self.hosts = hosts
self.riccis = riccis
+ self.query_func = func
+ self.query_args = args
def run(self):
while True:
@@ -32,21 +34,45 @@
return
host = self.hosts.pop()
self.mutex.release()
- r = None
+
+ r = { 'ricci': None }
try:
- r = RicciCommunicator(host)
+ rc = RicciCommunicator(host)
+ r['ricci'] = rc
+
+ if self.query_func is not None:
+ if self.query_args:
+ args = list(self.query_args)
+ else:
+ args = list()
+ args.insert(0, rc)
+ r['batch_result'] = self.query_func(*args)
except Exception, e:
- #print host, 'failed', str(e)
- pass
- except:
- #print host, 'failed'
pass
self.mutex.acquire()
self.riccis[host] = r
self.mutex.release()
+def send_batch_to_hosts(system_list, max_threads, func, *args):
+ mutex = threading.RLock()
+ threads = list()
+ hosts = list()
+ num_hosts = 0
+ ret = {}
+
+ for host in system_list:
+ hosts.append(host)
+ num_hosts += 1
+ if num_hosts <= max_threads:
+ threads.append(Worker(mutex, hosts, ret, func, *args))
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ return ret
+
def get_system_info(self, system_list):
mutex = threading.RLock()
hive = [] # workers
@@ -72,7 +98,7 @@
trusted = ''
authed = False
trusted = False
- ricci = ss[hostname]
+ ricci = ss[hostname]['ricci']
if ricci is not None:
OS = ricci.os()
--- conga/luci/site/luci/Extensions/LuciDB.py 2007/06/18 18:39:32 1.1.4.1
+++ conga/luci/site/luci/Extensions/LuciDB.py 2007/06/27 06:19:22 1.1.4.2
@@ -6,7 +6,6 @@
# Free Software Foundation.
from AccessControl import getSecurityManager
-import RicciQueries as rq
from ricci_communicator import RicciCommunicator
from LuciZopePerm import isAdmin
from LuciSyslog import get_logger
@@ -158,7 +157,7 @@
return True
batch_id = item[1].getProperty(BATCH_ID)
- batch_ret = rq.checkBatch(rc, batch_id)
+ batch_ret = rc.batch_status(batch_id)
finished = batch_ret[0]
if finished is True or finished == -1:
if finished == -1:
--- conga/luci/site/luci/Extensions/RicciQueries.py 2007/06/18 18:39:32 1.1.4.1
+++ conga/luci/site/luci/Extensions/RicciQueries.py 2007/06/27 06:19:22 1.1.4.2
@@ -6,52 +6,12 @@
# Free Software Foundation.
from xml.dom import minidom
-from ricci_communicator import RicciCommunicator, extract_module_status
+from ricci_communicator import RicciCommunicator
from LuciSyslog import get_logger
from conga_constants import LUCI_DEBUG_MODE
luci_log = get_logger()
-def checkBatch(rc, batch_id):
- err_msg = 'An unknown Ricci error occurred on %s' % rc.hostname()
-
- try:
- batch = rc.batch_report(batch_id)
- if batch is None:
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch0: batch id %s not found' \
- % batch_id)
- return (True, 'batch id %s was not found' % batch_id)
- except Exception, e:
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch1: %s: %r %s' \
- % (rc.hostname(), e, str(e)))
- return (-1, err_msg)
-
- try:
- code, new_err_msg = extract_module_status(batch, 1)
- if new_err_msg:
- err_msg = 'A Ricci error occurred on %s: %s' \
- % (rc.hostname(), str(new_err_msg))
- except Exception, e:
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch2: %s: %r %s' \
- % (rc.hostname(), e, str(e)))
- return (-1, err_msg)
-
- # In progress.
- if code == -101 or code == -102:
- return (False, 'in progress')
-
- # Done successfully.
- if code == 0:
- return (True, 'completed sucessfully')
-
- # Error
- if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('checkBatch3: %d: %s' % (code, rc.hostname()))
- return (-1, err_msg)
-
def addClusterNodeBatch(cluster_name,
install_base,
install_services,
@@ -310,9 +270,9 @@
for i in batch:
try:
- batch_number = i.getAttribute('batch_id')
- result = i.getAttribute('status')
- return (str(batch_number), str(result))
+ batch_number = str(i.getAttribute('batch_id'))
+ result = str(i.getAttribute('status'))
+ return (batch_number, result)
except Exception, e:
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('BAR1: %s' % e)
--- conga/luci/site/luci/Extensions/cluster_adapters.py 2007/06/19 15:54:10 1.120.2.31
+++ conga/luci/site/luci/Extensions/cluster_adapters.py 2007/06/27 06:19:22 1.120.2.32
@@ -640,7 +640,7 @@
# abort the whole process.
try:
while True:
- batch_ret = rq.checkBatch(cluster_ricci, batch_number)
+ batch_ret = cluster_ricci.batch_status(batch_number)
code = batch_ret[0]
if code is True:
break
@@ -3106,7 +3106,7 @@
if rc is not None:
batch_num = item[1].getProperty(BATCH_ID)
- batch_res = rq.checkBatch(rc, batch_num)
+ batch_res = rc.batch_status(batch_num)
finished = batch_res[0]
err_msg = batch_res[1]
--- conga/luci/site/luci/Extensions/ricci_communicator.py 2007/06/18 18:39:33 1.9.2.11
+++ conga/luci/site/luci/Extensions/ricci_communicator.py 2007/06/27 06:19:22 1.9.2.12
@@ -53,7 +53,10 @@
% (self.__hostname, hello.toxml()))
except:
if LUCI_DEBUG_MODE is True:
- luci_log.debug_verbose('RC:init0: error receiving header from %s: %r %s' % (self.__hostname, e, str(e)))
+ luci_log.debug_verbose('RC:init0: error receiving header from %s:%d: %r %s' % (self.__hostname, self.__port, e, str(e)))
+ else:
+ luci_log.info('Error reading from %s:%d: %s' \
+ % (self.__hostname, self.__port, str(e)))
self.__authed = hello.firstChild.getAttribute('authenticated') == 'true'
self.__cluname = hello.firstChild.getAttribute('clustername')
@@ -231,8 +234,8 @@
if doc.firstChild.getAttribute('success') != '0':
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('RC:PB4: batch command failed')
- raise RicciError, 'The last ricci command to host %s failed' \
- % self.__hostname
+ raise RicciError, 'The last ricci command to host %s:%d failed' \
+ % (self.__hostname, self.__port)
batch_node = None
for node in doc.firstChild.childNodes:
@@ -243,8 +246,8 @@
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('RC:PB5: no <batch/> from %s' \
% self.__hostname)
- raise RicciError, 'missing <batch/> in ricci response from "%s"' \
- % self.__hostname
+ raise RicciError, 'missing <batch/> in ricci response from %s:%d' \
+ % (self.__hostname, self.__port)
return batch_node
@@ -264,7 +267,7 @@
ricci_xml = self.process_batch(batch_xml, async)
if LUCI_DEBUG_NET is True:
try:
- luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command.' % (ricci_xml.toxml(), self.__hostname))
+ luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command' % (ricci_xml.toxml(), self.__hostname))
except:
pass
except:
@@ -276,6 +279,48 @@
doc.appendChild(ricci_xml)
return doc
+ def batch_status(self, batch_id):
+ err_msg = 'An unknown ricci error occurred on %s:%d' \
+ % (self.__hostname, self.__port)
+
+ try:
+ batch = self.batch_report(batch_id)
+ if batch is None:
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB0: batch id %s not found' \
+ % batch_id)
+ return (True, 'batch id %s was not found' % batch_id)
+ except Exception, e:
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB1: %s:%d: %r %s' \
+ % (self.__hostname, self.__port, e, str(e)))
+ return (-1, err_msg)
+
+ try:
+ code, new_err_msg = extract_module_status(batch, 1)
+ if new_err_msg:
+ err_msg = 'A ricci error occurred on %s:%d: %s' \
+ % (self.__hostname, self.__port, str(new_err_msg))
+ except Exception, e:
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB2: %s:%d %r %s: %s' \
+ % (self.__hostname, self.__port, e, str(e), err_msg))
+ return (-1, err_msg)
+
+ # In progress.
+ if code == -101 or code == -102:
+ return (False, 'in progress')
+
+ # Done successfully.
+ if code == 0:
+ return (True, 'completed sucessfully')
+
+ # Error
+ if LUCI_DEBUG_MODE is True:
+ luci_log.debug_verbose('RCCB3: %s:%d code %d: %s' \
+ % (self.__hostname, self.__port, code, err_msg))
+ return (-1, err_msg)
+
def batch_report(self, batch_id):
if LUCI_DEBUG_NET is True:
luci_log.debug_net('RC:BRep0: [auth=%d] asking for batchid# %s for host %s' % (self.__authed, batch_id, self.__hostname))
@@ -374,6 +419,9 @@
return doc
def get_ricci_communicator(self, hostname, allowed_systems):
+ if not hostname:
+ return None
+
if not self.access_to_host_allowed(hostname, allowed_systems):
if LUCI_DEBUG_MODE is True:
luci_log.debug_verbose('GRC0: access to host %s is not allowed' % hostname)
--- conga/luci/storage/index_html 2007/06/26 17:00:53 1.7.2.2
+++ conga/luci/storage/index_html 2007/06/27 06:19:22 1.7.2.3
@@ -222,7 +222,7 @@
<h1>Future Site of Forms</h1>
</metal:main-form-content>
<div style="padding-top: 10px;"
- tal:condition="report_cached">
+ tal:condition="python: report_cached and not (request.has_key('pagetype') and request['pagetype']=='commit_changes')">
<form>
<input type="hidden" name="reprobe_url"
tal:attributes="value
reply other threads:[~2007-06-27 6:19 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20070627061923.32515.qmail@sourceware.org \
--to=rmccabe@sourceware.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.