* [PATCH V2 4/6] toaster: enable custom env support for shell calls
2017-11-30 8:55 [PATCH V2 0/6] toaster: cummulative 11202017 patch David Reyna
` (2 preceding siblings ...)
2017-11-30 8:55 ` [PATCH V2 3/6] toaster: some recipe events do not include packages David Reyna
@ 2017-11-30 8:55 ` David Reyna
2017-11-30 8:55 ` [PATCH V2 5/6] toaster: landing page show cmdline capture David Reyna
2017-11-30 8:55 ` [PATCH V2 6/6] toaster: add 'nobuild' option to Toaster David Reyna
5 siblings, 0 replies; 9+ messages in thread
From: David Reyna @ 2017-11-30 8:55 UTC (permalink / raw)
To: bitbake-devel
From: David Reyna <David.Reyna@windriver.com>
Allow for custom environment additions for git cloning, for example
for anspass support.
[YOCTO #12193]
Signed-off-by: David Reyna <David.Reyna@windriver.com>
---
lib/toaster/bldcontrol/localhostbecontroller.py | 26 ++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/lib/toaster/bldcontrol/localhostbecontroller.py b/lib/toaster/bldcontrol/localhostbecontroller.py
index 23b2792..4c17562 100644
--- a/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -52,12 +52,14 @@ class LocalhostBEController(BuildEnvironmentController):
self.pokydirname = None
self.islayerset = False
- def _shellcmd(self, command, cwd=None, nowait=False):
+ def _shellcmd(self, command, cwd=None, nowait=False,env=None):
if cwd is None:
cwd = self.be.sourcedir
+ if env is None:
+ env=os.environ.copy()
- logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
- p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ logger.debug("lbc_shellcmd: (%s) %s" % (cwd, command))
+ p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
if nowait:
return
(out,err) = p.communicate()
@@ -98,6 +100,8 @@ class LocalhostBEController(BuildEnvironmentController):
layerlist = []
nongitlayerlist = []
+ git_env = os.environ.copy()
+ # (note: add custom environment settings here)
# set layers in the layersource
@@ -138,7 +142,7 @@ class LocalhostBEController(BuildEnvironmentController):
cached_layers = {}
try:
- for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"):
+ for remotes in self._shellcmd("git remote -v", self.be.sourcedir,env=git_env).split("\n"):
try:
remote = remotes.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
@@ -167,7 +171,7 @@ class LocalhostBEController(BuildEnvironmentController):
if os.path.exists(localdirname):
try:
localremotes = self._shellcmd("git remote -v",
- localdirname)
+ localdirname,env=git_env)
if not giturl in localremotes and commit != 'HEAD':
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
except ShellCmdException:
@@ -177,18 +181,18 @@ class LocalhostBEController(BuildEnvironmentController):
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
- self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
- self._shellcmd("git remote remove origin", localdirname)
- self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
+ self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname),env=git_env)
+ self._shellcmd("git remote remove origin", localdirname,env=git_env)
+ self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname,env=git_env)
else:
logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
- self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
+ self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname),env=git_env)
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
- self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
+ self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname,env=git_env)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
@@ -198,7 +202,7 @@ class LocalhostBEController(BuildEnvironmentController):
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
- self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')))
+ self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
--
1.9.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH V2 6/6] toaster: add 'nobuild' option to Toaster
2017-11-30 8:55 [PATCH V2 0/6] toaster: cummulative 11202017 patch David Reyna
` (4 preceding siblings ...)
2017-11-30 8:55 ` [PATCH V2 5/6] toaster: landing page show cmdline capture David Reyna
@ 2017-11-30 8:55 ` David Reyna
5 siblings, 0 replies; 9+ messages in thread
From: David Reyna @ 2017-11-30 8:55 UTC (permalink / raw)
To: bitbake-devel
From: David Reyna <David.Reyna@windriver.com>
Add a 'nobuild' option for starting Toaster without the project
and hosted builds support. This allows a Toaster host to provide
local build statistics without opening the host to external users
building projects.
[YOCTO #12315]
Signed-off-by: David Reyna <David.Reyna@windriver.com>
---
bin/toaster | 19 ++++++---
lib/toaster/toastergui/templates/base.html | 4 ++
lib/toaster/toastergui/templates/landing.html | 2 +
lib/toaster/toastergui/views.py | 57 ++++++++++++++++-----------
lib/toaster/toastergui/widgets.py | 4 ++
5 files changed, 58 insertions(+), 28 deletions(-)
diff --git a/bin/toaster b/bin/toaster
index 762451d..4036f0a 100755
--- a/bin/toaster
+++ b/bin/toaster
@@ -18,9 +18,10 @@
# along with this program. If not, see http://www.gnu.org/licenses/.
HELP="
-Usage: source toaster start|stop [webport=<address:port>] [noweb]
+Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild]
Optional arguments:
- [noweb] Setup the environment for building with toaster but don't start the development server
+ [nobuild] Setup the environment for capturing builds with toaster but disable managed builds
+ [noweb] Setup the environment for capturing builds with toaster but don't start the web server
[webport] Set the development server (default: localhost:8000)
"
@@ -183,6 +184,7 @@ unset OE_ROOT
WEBSERVER=1
+export TOASTER_BUILDSERVER=1
ADDR_PORT="localhost:8000"
unset CMD
for param in $*; do
@@ -190,6 +192,9 @@ for param in $*; do
noweb )
WEBSERVER=0
;;
+ nobuild )
+ TOASTER_BUILDSERVER=0
+ ;;
start )
CMD=$param
;;
@@ -286,9 +291,13 @@ case $CMD in
return 4
fi
export BITBAKE_UI='toasterui'
- $MANAGE runbuilds \
- </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \
- & echo $! >${BUILDDIR}/.runbuilds.pid
+ if [ $TOASTER_BUILDSERVER -eq 1 ] ; then
+ $MANAGE runbuilds \
+ </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \
+ & echo $! >${BUILDDIR}/.runbuilds.pid
+ else
+ echo "Toaster build server not started."
+ fi
# set fail safe stop system on terminal exit
trap stop_system SIGHUP
diff --git a/lib/toaster/toastergui/templates/base.html b/lib/toaster/toastergui/templates/base.html
index edbd110..4f72064 100644
--- a/lib/toaster/toastergui/templates/base.html
+++ b/lib/toaster/toastergui/templates/base.html
@@ -110,6 +110,7 @@
All builds
</a>
</li>
+ {% if project_enable %}
<li id="navbar-all-projects"
{% if request.resolver_match.url_name == 'all-projects' %}
class="active"
@@ -119,6 +120,7 @@
All projects
</a>
</li>
+ {% endif %}
{% endif %}
<li id="navbar-docs">
<a target="_blank" href="http://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html">
@@ -127,7 +129,9 @@
</a>
</li>
</ul>
+ {% if project_enable %}
<a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a>
+ {% endif %}
</div>
</div>
</nav>
diff --git a/lib/toaster/toastergui/templates/landing.html b/lib/toaster/toastergui/templates/landing.html
index cf7516d..70c7359 100644
--- a/lib/toaster/toastergui/templates/landing.html
+++ b/lib/toaster/toastergui/templates/landing.html
@@ -21,11 +21,13 @@
</p>
{% if lvs_nos %}
+ {% if project_enable %}
<p class="top-air">
<a class="btn btn-primary btn-lg" href="{% url 'newproject' %}">
Create your first Toaster project to run manage builds
</a>
</p>
+ {% endif %}
{% else %}
<div class="alert alert-info lead top-air">
Toaster has no layer information. Without layer information, you cannot run builds. To generate layer information you can:
diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py
index 209b07d..34ed2b2 100755
--- a/lib/toaster/toastergui/views.py
+++ b/lib/toaster/toastergui/views.py
@@ -49,6 +49,8 @@ import logging
logger = logging.getLogger("toaster")
+# Project creation and managed build enable
+project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
class MimeTypeFinder(object):
# setting this to False enables additional non-standard mimetypes
@@ -65,6 +67,12 @@ class MimeTypeFinder(object):
guessed_type = 'application/octet-stream'
return guessed_type
+# single point to add global values into the context before rendering
+def toaster_render(request, page, context):
+ context['project_enable'] = project_enable
+ return render(request, page, context)
+
+
# all new sessions should come through the landing page;
# determine in which mode we are running in, and redirect appropriately
def landing(request):
@@ -86,7 +94,7 @@ def landing(request):
context = {'lvs_nos' : Layer_Version.objects.all().count()}
- return render(request, 'landing.html', context)
+ return toaster_render(request, 'landing.html', context)
def objtojson(obj):
from django.db.models.query import QuerySet
@@ -519,7 +527,7 @@ def builddashboard( request, build_id ):
'packagecount' : packageCount,
'logmessages' : logmessages,
}
- return render( request, template, context )
+ return toaster_render( request, template, context )
@@ -591,7 +599,7 @@ def task( request, build_id, task_id ):
build__completed_on__lt=task_object.build.completed_on).exclude(
order__isnull=True).exclude(outcome=Task.OUTCOME_NA).order_by('-build__completed_on')
- return render( request, template, context )
+ return toaster_render( request, template, context )
def recipe(request, build_id, recipe_id, active_tab="1"):
template = "recipe.html"
@@ -618,7 +626,7 @@ def recipe(request, build_id, recipe_id, active_tab="1"):
'package_count' : package_count,
'tab_states' : tab_states,
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def recipe_packages(request, build_id, recipe_id):
template = "recipe_packages.html"
@@ -663,7 +671,7 @@ def recipe_packages(request, build_id, recipe_id):
},
]
}
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -785,7 +793,7 @@ def dirinfo(request, build_id, target_id, file_path=None):
'dir_list': dir_list,
'file_path': file_path,
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def _find_task_dep(task_object):
tdeps = Task_Dependency.objects.filter(task=task_object).filter(depends_on__order__gt=0)
@@ -837,7 +845,7 @@ def configuration(request, build_id):
'build': build,
'project': build.project,
'targets': Target.objects.filter(build=build_id)})
- return render(request, template, context)
+ return toaster_render(request, template, context)
def configvars(request, build_id):
@@ -926,7 +934,7 @@ def configvars(request, build_id):
],
}
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -939,7 +947,7 @@ def bfile(request, build_id, package_id):
'project': build.project,
'objects' : files
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
# A set of dependency types valid for both included and built package views
@@ -1092,7 +1100,7 @@ def package_built_detail(request, build_id, package_id):
if paths.all().count() < 2:
context['disable_sort'] = True;
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -1111,7 +1119,7 @@ def package_built_dependencies(request, build_id, package_id):
'other_deps' : dependencies['other_deps'],
'dependency_count' : _get_package_dependency_count(package, -1, False)
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def package_included_detail(request, build_id, target_id, package_id):
@@ -1157,7 +1165,7 @@ def package_included_detail(request, build_id, target_id, package_id):
}
if paths.all().count() < 2:
context['disable_sort'] = True
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -1181,7 +1189,7 @@ def package_included_dependencies(request, build_id, target_id, package_id):
'reverse_count' : _get_package_reverse_dep_count(package, target_id),
'dependency_count' : _get_package_dependency_count(package, target_id, True)
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def package_included_reverse_dependencies(request, build_id, target_id, package_id):
template = "package_included_reverse_dependencies.html"
@@ -1232,7 +1240,7 @@ def package_included_reverse_dependencies(request, build_id, target_id, package_
}
if objects.all().count() < 2:
context['disable_sort'] = True
- response = render(request, template, context)
+ response = toaster_render(request, template, context)
_set_parameters_values(pagesize, orderby, request)
return response
@@ -1365,6 +1373,9 @@ if True:
# new project
def newproject(request):
+ if not project_enable:
+ return redirect( landing )
+
template = "newproject.html"
context = {
'email': request.user.email if request.user.is_authenticated() else '',
@@ -1379,7 +1390,7 @@ if True:
if request.method == "GET":
# render new project page
- return render(request, template, context)
+ return toaster_render(request, template, context)
elif request.method == "POST":
mandatory_fields = ['projectname', 'ptype']
try:
@@ -1419,7 +1430,7 @@ if True:
context['alert'] = "Your chosen username is already used"
else:
context['alert'] = str(e)
- return render(request, template, context)
+ return toaster_render(request, template, context)
raise Exception("Invalid HTTP method for this page")
@@ -1427,7 +1438,7 @@ if True:
def project(request, pid):
project = Project.objects.get(pk=pid)
context = {"project": project}
- return render(request, "project.html", context)
+ return toaster_render(request, "project.html", context)
def jsunittests(request):
""" Provides a page for the js unit tests """
@@ -1453,7 +1464,7 @@ if True:
name="MACHINE",
value="qemux86")
context = {'project': new_project}
- return render(request, "js-unit-tests.html", context)
+ return toaster_render(request, "js-unit-tests.html", context)
from django.views.decorators.csrf import csrf_exempt
@csrf_exempt
@@ -1588,7 +1599,7 @@ if True:
context = {
'project': Project.objects.get(id=pid),
}
- return render(request, template, context)
+ return toaster_render(request, template, context)
def layerdetails(request, pid, layerid):
project = Project.objects.get(pk=pid)
@@ -1617,7 +1628,7 @@ if True:
'projectlayers': list(project_layers)
}
- return render(request, 'layerdetails.html', context)
+ return toaster_render(request, 'layerdetails.html', context)
def get_project_configvars_context():
@@ -1707,7 +1718,7 @@ if True:
except (ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist):
pass
- return render(request, "projectconf.html", context)
+ return toaster_render(request, "projectconf.html", context)
def _file_names_for_artifact(build, artifact_type, artifact_id):
"""
@@ -1774,7 +1785,7 @@ if True:
return response
else:
- return render(request, "unavailable_artifact.html")
+ return toaster_render(request, "unavailable_artifact.html")
except (ObjectDoesNotExist, IOError):
- return render(request, "unavailable_artifact.html")
+ return toaster_render(request, "unavailable_artifact.html")
diff --git a/lib/toaster/toastergui/widgets.py b/lib/toaster/toastergui/widgets.py
index 67c1ff9..a1792d9 100644
--- a/lib/toaster/toastergui/widgets.py
+++ b/lib/toaster/toastergui/widgets.py
@@ -41,6 +41,7 @@ import types
import json
import collections
import re
+import os
from toastergui.tablefilter import TableFilterMap
@@ -86,6 +87,9 @@ class ToasterTable(TemplateView):
context['table_name'] = type(self).__name__.lower()
context['empty_state'] = self.empty_state
+ # global variables
+ context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
+
return context
def get(self, request, *args, **kwargs):
--
1.9.1
^ permalink raw reply related [flat|nested] 9+ messages in thread