* [layerindex-web][PATCH 1/5] Allow blanking out field values in bulk change
2016-05-30 4:31 [layerindex-web][PATCH 0/5] Layer index improvements Paul Eggleton
@ 2016-05-30 4:31 ` Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 2/5] update.py: refactor into two separate scripts Paul Eggleton
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Paul Eggleton @ 2016-05-30 4:31 UTC (permalink / raw)
To: yocto
If you're moving a short description value from DESCRIPTION to SUMMARY
then part of that is setting DESCRIPTION to blank, however that wasn't
possible - the code was assuming that a null value meant "keep the
original value". Change the logic so that the value in the bulk change
object is always set and is compared to the original value to see if it
is different. This provides less safety against bulk change data going
stale in the face of the metadata being updated, but without using an
additional "magic" field value that's the price we have to pay, and it's
unlikely to bother too many people I would imagine.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/bulkchange.py | 17 +++++++++--------
layerindex/forms.py | 19 -------------------
layerindex/models.py | 20 ++++++++++++--------
layerindex/views.py | 3 +--
4 files changed, 22 insertions(+), 37 deletions(-)
diff --git a/layerindex/bulkchange.py b/layerindex/bulkchange.py
index ed36859..c569b61 100644
--- a/layerindex/bulkchange.py
+++ b/layerindex/bulkchange.py
@@ -97,14 +97,15 @@ def patch_recipe(fn, relpath, values):
with tempfile.NamedTemporaryFile('w', delete=False) as tf:
def outputvalue(name):
- rawtext = '%s = "%s"\n' % (name, values[name])
- if name in nowrap_vars:
- tf.write(rawtext)
- else:
- wrapped = textwrap.wrap(rawtext)
- for wrapline in wrapped[:-1]:
- tf.write('%s \\\n' % wrapline)
- tf.write('%s\n' % wrapped[-1])
+ if values[name]:
+ rawtext = '%s = "%s"\n' % (name, values[name])
+ if name in nowrap_vars:
+ tf.write(rawtext)
+ else:
+ wrapped = textwrap.wrap(rawtext)
+ for wrapline in wrapped[:-1]:
+ tf.write('%s \\\n' % wrapline)
+ tf.write('%s\n' % wrapped[-1])
tfn = tf.name
with open(fn, 'r') as f:
diff --git a/layerindex/forms.py b/layerindex/forms.py
index 78711e5..60653cf 100644
--- a/layerindex/forms.py
+++ b/layerindex/forms.py
@@ -189,25 +189,6 @@ class BulkChangeEditForm(forms.ModelForm):
model = RecipeChange
fields = ('summary', 'description', 'homepage', 'bugtracker', 'section', 'license')
- def __init__(self, *args, **kwargs):
- instance = kwargs.get('instance', None)
- initial = kwargs.get('initial', {})
- if instance:
- recipe = instance.recipe
- if recipe:
- for fieldname in self._meta.fields:
- if not getattr(instance, fieldname):
- initial[fieldname] = getattr(recipe, fieldname)
- kwargs['initial'] = initial
- super(BulkChangeEditForm, self).__init__(*args, **kwargs)
-
- def clear_same_values(self):
- for fieldname in self._meta.fields:
- oldval = getattr(self.instance.recipe, fieldname)
- newval = getattr(self.instance, fieldname)
- if oldval == newval:
- setattr(self.instance, fieldname, '')
-
BulkChangeEditFormSet = modelformset_factory(RecipeChange, form=BulkChangeEditForm, extra=0)
diff --git a/layerindex/models.py b/layerindex/models.py
index cde8fe3..e0d85ea 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -415,12 +415,16 @@ class RecipeChange(models.Model):
def changed_fields(self, mapped = False):
res = {}
- for field in self._meta.fields:
- if not field.name in ['id', 'changeset', 'recipe']:
- value = getattr(self, field.name)
- if value:
- if mapped:
- res[self.RECIPE_VARIABLE_MAP[field.name]] = value
- else:
- res[field.name] = value
+ for fieldname in self.RECIPE_VARIABLE_MAP:
+ value = getattr(self, fieldname)
+ origvalue = getattr(self.recipe, fieldname)
+ if value != origvalue:
+ if mapped:
+ res[self.RECIPE_VARIABLE_MAP[fieldname]] = value
+ else:
+ res[fieldname] = value
return res
+
+ def reset_fields(self):
+ for fieldname in self.RECIPE_VARIABLE_MAP:
+ setattr(self, fieldname, getattr(self.recipe, fieldname))
diff --git a/layerindex/views.py b/layerindex/views.py
index 898a7c4..d033046 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -200,8 +200,6 @@ def bulk_change_edit_view(request, template_name, pk):
if request.method == 'POST':
formset = BulkChangeEditFormSet(request.POST, queryset=changeset.recipechange_set.all())
if formset.is_valid():
- for form in formset:
- form.clear_same_values()
formset.save()
return HttpResponseRedirect(reverse('bulk_change_review', args=(changeset.id,)))
else:
@@ -524,6 +522,7 @@ class BulkChangeSearchView(AdvancedRecipeSearchView):
change = RecipeChange()
change.changeset = changeset
change.recipe = recipe
+ change.reset_fields()
change.save()
if 'add_selected' in request.POST:
--
2.5.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [layerindex-web][PATCH 2/5] update.py: refactor into two separate scripts
2016-05-30 4:31 [layerindex-web][PATCH 0/5] Layer index improvements Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 1/5] Allow blanking out field values in bulk change Paul Eggleton
@ 2016-05-30 4:31 ` Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 3/5] update.py: allow updating all branches with one command Paul Eggleton
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Paul Eggleton @ 2016-05-30 4:31 UTC (permalink / raw)
To: yocto
In order to try to avoid problems with leaking memory, context bleeding
from one layer to another, and lay the groundwork for supporting
updating all branches in one operation, split the updating of a single
layer out to its own internal script. This means that the tinfoil
instantiation and metadata parsing is in a completely separate process
per layer.
Implements [YOCTO #9647].
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/update.py | 511 ++++--------------------------------------
layerindex/update_layer.py | 547 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 591 insertions(+), 467 deletions(-)
create mode 100644 layerindex/update_layer.py
diff --git a/layerindex/update.py b/layerindex/update.py
index 7daccef..f7cb25c 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -2,23 +2,20 @@
# Fetch layer repositories and update layer index database
#
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
-import os.path
+import os
import optparse
import logging
-from datetime import datetime
-import re
-import tempfile
-import shutil
+import subprocess
+import signal
from distutils.version import LooseVersion
import utils
-import recipeparse
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -33,79 +30,21 @@ except ImportError:
sys.exit(1)
-def check_machine_conf(path, subdir_start):
- subpath = path[len(subdir_start):]
- res = conf_re.match(subpath)
- if res:
- return res.group(1)
- return None
+def reenable_sigint():
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
-def split_recipe_fn(path):
- splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
- pn = splitfn[0]
- if len(splitfn) > 1:
- pv = splitfn[1]
- else:
- pv = "1.0"
- return (pn, pv)
-
-def update_recipe_file(data, path, recipe, layerdir_start, repodir):
- fn = str(os.path.join(path, recipe.filename))
+def run_command_interruptible(cmd):
+ """
+ Run a command with output displayed on the console, but ensure any Ctrl+C is
+ processed only by the child process.
+ """
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
try:
- logger.debug('Updating recipe %s' % fn)
- envdata = bb.cache.Cache.loadDataFull(fn, [], data)
- envdata.setVar('SRCPV', 'X')
- recipe.pn = envdata.getVar("PN", True)
- recipe.pv = envdata.getVar("PV", True)
- recipe.summary = envdata.getVar("SUMMARY", True)
- recipe.description = envdata.getVar("DESCRIPTION", True)
- recipe.section = envdata.getVar("SECTION", True)
- recipe.license = envdata.getVar("LICENSE", True)
- recipe.homepage = envdata.getVar("HOMEPAGE", True)
- recipe.bugtracker = envdata.getVar("BUGTRACKER", True) or ""
- recipe.provides = envdata.getVar("PROVIDES", True) or ""
- recipe.bbclassextend = envdata.getVar("BBCLASSEXTEND", True) or ""
- # Handle recipe inherits for this recipe
- gr = set(data.getVar("__inherit_cache", True) or [])
- lr = set(envdata.getVar("__inherit_cache", True) or [])
- recipe.inherits = ' '.join(sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}))
- recipe.blacklisted = envdata.getVarFlag('PNBLACKLIST', recipe.pn, True) or ""
- recipe.save()
-
- # Get file dependencies within this layer
- deps = envdata.getVar('__depends', True)
- filedeps = []
- for depstr, date in deps:
- found = False
- if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
- filedeps.append(os.path.relpath(depstr, repodir))
- from layerindex.models import RecipeFileDependency
- RecipeFileDependency.objects.filter(recipe=recipe).delete()
- for filedep in filedeps:
- recipedep = RecipeFileDependency()
- recipedep.layerbranch = recipe.layerbranch
- recipedep.recipe = recipe
- recipedep.path = filedep
- recipedep.save()
- except KeyboardInterrupt:
- raise
- except BaseException as e:
- if not recipe.pn:
- recipe.pn = recipe.filename[:-3].split('_')[0]
- logger.error("Unable to read %s: %s", fn, str(e))
-
-def update_machine_conf_file(path, machine):
- logger.debug('Updating machine %s' % path)
- desc = ""
- with open(path, 'r') as f:
- for line in f:
- if line.startswith('#@NAME:'):
- desc = line[7:].strip()
- if line.startswith('#@DESCRIPTION:'):
- desc = line[14:].strip()
- desc = re.sub(r'Machine configuration for( running)*( an)*( the)*', '', desc)
- break
- machine.description = desc
+ ret = subprocess.call(cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint)
+ finally:
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
+ return ret
+
def main():
if LooseVersion(git.__version__) < '0.3.1':
@@ -151,13 +90,9 @@ def main():
parser.print_help()
sys.exit(1)
- if options.fullreload:
- options.reload = True
-
utils.setup_django()
import settings
- from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
- from django.db import transaction
+ from layerindex.models import LayerItem
logger.setLevel(options.loglevel)
@@ -225,397 +160,39 @@ def main():
else:
out = utils.runcmd("git fetch", bitbakepath, logger=logger)
- try:
- (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
- except recipeparse.RecipeParseError as e:
- logger.error(str(e))
- sys.exit(1)
-
- # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
- tinfoil.config_data.setVar('SUMMARY', '')
- # Clear the default value of DESCRIPTION so that we can see where it's not set
- tinfoil.config_data.setVar('DESCRIPTION', '')
- # Clear the default value of HOMEPAGE ('unknown')
- tinfoil.config_data.setVar('HOMEPAGE', '')
- # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
- # why won't they just fix that?!)
- tinfoil.config_data.setVar('LICENSE', '')
-
# Process and extract data from each layer
+ # We now do this by calling out to a separate script; doing otherwise turned out to be
+ # unreliable due to leaking memory (we're using bitbake internals in a manner in which
+ # they never get used during normal operation).
for layer in layerquery:
- transaction.enter_transaction_management()
- transaction.managed(True)
- try:
- urldir = layer.get_fetch_dir()
- repodir = os.path.join(fetchdir, urldir)
- if layer.vcs_url in failedrepos:
- logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
- transaction.rollback()
- continue
-
- layerbranch = layer.get_layerbranch(options.branch)
-
- branchname = options.branch
- branchdesc = options.branch
- if layerbranch:
- if layerbranch.actual_branch:
- branchname = layerbranch.actual_branch
- branchdesc = "%s (%s)" % (options.branch, branchname)
-
- # Collect repo info
- repo = git.Repo(repodir)
- assert repo.bare == False
- try:
- if options.nocheckout:
- topcommit = repo.commit('HEAD')
- else:
- topcommit = repo.commit('origin/%s' % branchname)
- except:
- if layerbranch:
- logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
- else:
- logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
- transaction.rollback()
- continue
-
- newbranch = False
- if not layerbranch:
- # LayerBranch doesn't exist for this branch, create it
- newbranch = True
- layerbranch = LayerBranch()
- layerbranch.layer = layer
- layerbranch.branch = branch
- layerbranch_source = layer.get_layerbranch('master')
- if not layerbranch_source:
- layerbranch_source = layer.get_layerbranch(None)
- if layerbranch_source:
- layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
- layerbranch.save()
- if layerbranch_source:
- for maintainer in layerbranch_source.layermaintainer_set.all():
- maintainer.pk = None
- maintainer.id = None
- maintainer.layerbranch = layerbranch
- maintainer.save()
- for dep in layerbranch_source.dependencies_set.all():
- dep.pk = None
- dep.id = None
- dep.layerbranch = layerbranch
- dep.save()
-
- if layerbranch.vcs_subdir and not options.nocheckout:
- # Find latest commit in subdirectory
- # A bit odd to do it this way but apparently there's no other way in the GitPython API
- topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
- if not topcommit:
- # This will error out if the directory is completely invalid or had never existed at this point
- # If it previously existed but has since been deleted, you will get the revision where it was
- # deleted - so we need to handle that case separately later
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- elif layerbranch.vcs_subdir:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- else:
- logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
- transaction.rollback()
- continue
-
- layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
- layerdir_start = os.path.normpath(layerdir) + os.sep
- layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
- layermachines = Machine.objects.filter(layerbranch=layerbranch)
- layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
- layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
- if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
- # Check out appropriate branch
- if not options.nocheckout:
- out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
- out = utils.runcmd("git clean -f -x", repodir, logger=logger)
-
- if layerbranch.vcs_subdir and not os.path.exists(layerdir):
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- else:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- transaction.rollback()
- continue
-
- if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
- logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
- transaction.rollback()
- continue
-
- logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
-
- try:
- config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
- except recipeparse.RecipeParseError as e:
- logger.error(str(e))
- transaction.rollback()
- continue
-
- if layerbranch.vcs_last_rev and not options.reload:
- try:
- diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
- except Exception as e:
- logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
- diff = None
- else:
- diff = None
-
- # We handle recipes specially to try to preserve the same id
- # when recipe upgrades happen (so that if a user bookmarks a
- # recipe page it remains valid)
- layerrecipes_delete = []
- layerrecipes_add = []
-
- # Check if any paths should be ignored because there are layers within this layer
- removedirs = []
- for root, dirs, files in os.walk(layerdir):
- for d in dirs:
- if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
- removedirs.append(os.path.join(root, d) + os.sep)
-
- if diff:
- # Apply git changes to existing recipe list
-
- if layerbranch.vcs_subdir:
- subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
- else:
- subdir_start = ""
-
- updatedrecipes = set()
- for d in diff.iter_change_type('D'):
- path = d.a_blob.path
- if path.startswith(subdir_start):
- skip = False
- for d in removedirs:
- if path.startswith(d):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
- if len(values):
- layerrecipes_delete.append(values[0])
- logger.debug("Mark %s for deletion" % values[0])
- updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
- else:
- logger.warn("Deleted recipe %s could not be found" % path)
- elif typename == 'bbappend':
- layerappends.filter(filepath=filepath).filter(filename=filename).delete()
- elif typename == 'machine':
- layermachines.filter(name=filename).delete()
- elif typename == 'bbclass':
- layerclasses.filter(name=filename).delete()
-
- for d in diff.iter_change_type('A'):
- path = d.b_blob.path
- if path.startswith(subdir_start):
- skip = False
- for d in removedirs:
- if path.startswith(d):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- layerrecipes_add.append(os.path.join(repodir, path))
- logger.debug("Mark %s for addition" % path)
- updatedrecipes.add(os.path.join(filepath, filename))
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = filename
- append.filepath = filepath
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(os.path.join(repodir, path), machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- dirtyrecipes = set()
- for d in diff.iter_change_type('M'):
- path = d.a_blob.path
- if path.startswith(subdir_start):
- skip = False
- for d in removedirs:
- if path.startswith(d):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- logger.debug("Mark %s for update" % path)
- results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
- if results:
- recipe = results[0]
- update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
- recipe.save()
- updatedrecipes.add(recipe.full_path())
- elif typename == 'machine':
- results = layermachines.filter(name=filename)
- if results:
- machine = results[0]
- update_machine_conf_file(os.path.join(repodir, path), machine)
- machine.save()
-
- deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
- for dep in deps:
- dirtyrecipes.add(dep.recipe)
-
- for recipe in dirtyrecipes:
- if not recipe.full_path() in updatedrecipes:
- update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
- else:
- # Collect recipe data from scratch
-
- layerrecipe_fns = []
- if options.fullreload:
- layerrecipes.delete()
- else:
- # First, check which recipes still exist
- layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
- for v in layerrecipe_values:
- root = os.path.join(layerdir, v['filepath'])
- fullpath = os.path.join(root, v['filename'])
- preserve = True
- if os.path.exists(fullpath):
- for d in removedirs:
- if fullpath.startswith(d):
- preserve = False
- break
- else:
- preserve = False
-
- if preserve:
- # Recipe still exists, update it
- results = layerrecipes.filter(id=v['id'])[:1]
- recipe = results[0]
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- else:
- # Recipe no longer exists, mark it for later on
- layerrecipes_delete.append(v)
- layerrecipe_fns.append(fullpath)
-
- layermachines.delete()
- layerappends.delete()
- layerclasses.delete()
- for root, dirs, files in os.walk(layerdir):
- if '.git' in dirs:
- dirs.remove('.git')
- for d in dirs[:]:
- fullpath = os.path.join(root, d) + os.sep
- if fullpath in removedirs:
- dirs.remove(d)
- for f in files:
- fullpath = os.path.join(root, f)
- (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
- if typename == 'recipe':
- if fullpath not in layerrecipe_fns:
- layerrecipes_add.append(fullpath)
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = f
- append.filepath = os.path.relpath(root, layerdir)
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(fullpath, machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- for added in layerrecipes_add:
- # This is good enough without actually parsing the file
- (pn, pv) = split_recipe_fn(added)
- oldid = -1
- for deleted in layerrecipes_delete:
- if deleted['pn'] == pn:
- oldid = deleted['id']
- layerrecipes_delete.remove(deleted)
- break
- if oldid > -1:
- # Reclaim a record we would have deleted
- results = Recipe.objects.filter(id=oldid)[:1]
- recipe = results[0]
- logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
- else:
- # Create new record
- logger.debug("Add new recipe %s" % added)
- recipe = Recipe()
- recipe.layerbranch = layerbranch
- recipe.filename = os.path.basename(added)
- root = os.path.dirname(added)
- recipe.filepath = os.path.relpath(root, layerdir)
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- recipe.save()
-
- for deleted in layerrecipes_delete:
- logger.debug("Delete %s" % deleted)
- results = Recipe.objects.filter(id=deleted['id'])[:1]
- recipe = results[0]
- recipe.delete()
-
- # Save repo info
- layerbranch.vcs_last_rev = topcommit.hexsha
- layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
- else:
- logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
-
- layerbranch.vcs_last_fetch = datetime.now()
- layerbranch.save()
-
- if options.dryrun:
- transaction.rollback()
- else:
- transaction.commit()
-
- # Slightly hacky way of avoiding memory leaks
- bb.event.ui_queue = []
- bb.parse.parse_py.BBHandler.cached_statements = {}
- bb.codeparser.codeparsercache = bb.codeparser.CodeParserCache()
- if hasattr(bb.codeparser, 'codecache'):
- bb.codeparser.codecache = bb.codeparser.SetCache()
- bb.fetch._checksum_cache = bb.checksum.FileChecksumCache()
- bb.fetch.urldata_cache = {}
- bb.fetch.saved_headrevs = {}
- bb.parse.__pkgsplit_cache__={}
- bb.parse.__mtime_cache = {}
- bb.parse.init_parser(tinfoil.config_data)
-
- except KeyboardInterrupt:
- transaction.rollback()
- logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
+ if layer.vcs_url in failedrepos:
+ logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
+
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
+
+ cmd = 'python update_layer.py -l %s -b %s' % (layer.name, options.branch)
+ if options.reload:
+ cmd += ' --reload'
+ if options.fullreload:
+ cmd += ' --fullreload'
+ if options.nocheckout:
+ cmd += ' --nocheckout'
+ if options.dryrun:
+ cmd += ' -n'
+ if options.loglevel == logging.DEBUG:
+ cmd += ' -d'
+ elif options.loglevel == logging.ERROR:
+ cmd += ' -q'
+ logger.debug('Running layer update command: %s' % cmd)
+ ret = run_command_interruptible(cmd)
+ if ret == 254:
+ # Interrupted by user, break out of loop
break
- except:
- import traceback
- traceback.print_exc()
- transaction.rollback()
- finally:
- transaction.leave_transaction_management()
finally:
utils.unlock_file(lockfile)
- shutil.rmtree(tempdir)
sys.exit(0)
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
new file mode 100644
index 0000000..bf1dbb2
--- /dev/null
+++ b/layerindex/update_layer.py
@@ -0,0 +1,547 @@
+#!/usr/bin/env python
+
+# Update layer index database for a single layer
+#
+# Copyright (C) 2013-2016 Intel Corporation
+# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+import optparse
+import logging
+from datetime import datetime
+import re
+import tempfile
+import shutil
+from distutils.version import LooseVersion
+import utils
+import recipeparse
+
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+logger = utils.logger_create('LayerIndexUpdate')
+
+# Ensure PythonGit is installed (buildhistory_analysis needs it)
+try:
+ import git
+except ImportError:
+ logger.error("Please install PythonGit 0.3.1 or later in order to use this script")
+ sys.exit(1)
+
+
+def check_machine_conf(path, subdir_start):
+ subpath = path[len(subdir_start):]
+ res = conf_re.match(subpath)
+ if res:
+ return res.group(1)
+ return None
+
+def split_recipe_fn(path):
+ splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
+ pn = splitfn[0]
+ if len(splitfn) > 1:
+ pv = splitfn[1]
+ else:
+ pv = "1.0"
+ return (pn, pv)
+
+def update_recipe_file(data, path, recipe, layerdir_start, repodir):
+ fn = str(os.path.join(path, recipe.filename))
+ try:
+ logger.debug('Updating recipe %s' % fn)
+ envdata = bb.cache.Cache.loadDataFull(fn, [], data)
+ envdata.setVar('SRCPV', 'X')
+ recipe.pn = envdata.getVar("PN", True)
+ recipe.pv = envdata.getVar("PV", True)
+ recipe.summary = envdata.getVar("SUMMARY", True)
+ recipe.description = envdata.getVar("DESCRIPTION", True)
+ recipe.section = envdata.getVar("SECTION", True)
+ recipe.license = envdata.getVar("LICENSE", True)
+ recipe.homepage = envdata.getVar("HOMEPAGE", True)
+ recipe.bugtracker = envdata.getVar("BUGTRACKER", True) or ""
+ recipe.provides = envdata.getVar("PROVIDES", True) or ""
+ recipe.bbclassextend = envdata.getVar("BBCLASSEXTEND", True) or ""
+ # Handle recipe inherits for this recipe
+ gr = set(data.getVar("__inherit_cache", True) or [])
+ lr = set(envdata.getVar("__inherit_cache", True) or [])
+ recipe.inherits = ' '.join(sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}))
+ recipe.blacklisted = envdata.getVarFlag('PNBLACKLIST', recipe.pn, True) or ""
+ recipe.save()
+
+ # Get file dependencies within this layer
+ deps = envdata.getVar('__depends', True)
+ filedeps = []
+ for depstr, date in deps:
+ found = False
+ if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
+ filedeps.append(os.path.relpath(depstr, repodir))
+ from layerindex.models import RecipeFileDependency
+ RecipeFileDependency.objects.filter(recipe=recipe).delete()
+ for filedep in filedeps:
+ recipedep = RecipeFileDependency()
+ recipedep.layerbranch = recipe.layerbranch
+ recipedep.recipe = recipe
+ recipedep.path = filedep
+ recipedep.save()
+ except KeyboardInterrupt:
+ raise
+ except BaseException as e:
+ if not recipe.pn:
+ recipe.pn = recipe.filename[:-3].split('_')[0]
+ logger.error("Unable to read %s: %s", fn, str(e))
+
+def update_machine_conf_file(path, machine):
+ logger.debug('Updating machine %s' % path)
+ desc = ""
+ with open(path, 'r') as f:
+ for line in f:
+ if line.startswith('#@NAME:'):
+ desc = line[7:].strip()
+ if line.startswith('#@DESCRIPTION:'):
+ desc = line[14:].strip()
+ desc = re.sub(r'Machine configuration for( running)*( an)*( the)*', '', desc)
+ break
+ machine.description = desc
+
+def main():
+ if LooseVersion(git.__version__) < '0.3.1':
+ logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
+ sys.exit(1)
+
+
+ parser = optparse.OptionParser(
+ usage = """
+ %prog [options]""")
+
+ parser.add_option("-b", "--branch",
+ help = "Specify branch to update",
+ action="store", dest="branch", default='master')
+ parser.add_option("-l", "--layer",
+ help = "Layer to update",
+ action="store", dest="layer")
+ parser.add_option("-r", "--reload",
+ help = "Reload recipe data instead of updating since last update",
+ action="store_true", dest="reload")
+ parser.add_option("", "--fullreload",
+ help = "Discard existing recipe data and fetch it from scratch",
+ action="store_true", dest="fullreload")
+ parser.add_option("-n", "--dry-run",
+ help = "Don't write any data back to the database",
+ action="store_true", dest="dryrun")
+ parser.add_option("", "--nocheckout",
+ help = "Don't check out branches",
+ action="store_true", dest="nocheckout")
+ parser.add_option("-d", "--debug",
+ help = "Enable debug output",
+ action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+ parser.add_option("-q", "--quiet",
+ help = "Hide all output except error messages",
+ action="store_const", const=logging.ERROR, dest="loglevel")
+
+ options, args = parser.parse_args(sys.argv)
+ if len(args) > 1:
+ logger.error('unexpected argument "%s"' % args[1])
+ parser.print_help()
+ sys.exit(1)
+
+ if options.fullreload:
+ options.reload = True
+
+ utils.setup_django()
+ import settings
+ from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
+ from django.db import transaction
+
+ logger.setLevel(options.loglevel)
+
+ branch = utils.get_branch(options.branch)
+ if not branch:
+ logger.error("Specified branch %s is not valid" % options.branch)
+ sys.exit(1)
+
+ fetchdir = settings.LAYER_FETCH_DIR
+ if not fetchdir:
+ logger.error("Please set LAYER_FETCH_DIR in settings.py")
+ sys.exit(1)
+
+ bitbakepath = os.path.join(fetchdir, 'bitbake')
+
+ try:
+ (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
+ except recipeparse.RecipeParseError as e:
+ logger.error(str(e))
+ sys.exit(1)
+
+ # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
+ tinfoil.config_data.setVar('SUMMARY', '')
+ # Clear the default value of DESCRIPTION so that we can see where it's not set
+ tinfoil.config_data.setVar('DESCRIPTION', '')
+ # Clear the default value of HOMEPAGE ('unknown')
+ tinfoil.config_data.setVar('HOMEPAGE', '')
+ # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
+ # why won't they just fix that?!)
+ tinfoil.config_data.setVar('LICENSE', '')
+
+ transaction.enter_transaction_management()
+ transaction.managed(True)
+ try:
+ layer = utils.get_layer(options.layer)
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
+
+ layerbranch = layer.get_layerbranch(options.branch)
+
+ branchname = options.branch
+ branchdesc = options.branch
+ if layerbranch:
+ if layerbranch.actual_branch:
+ branchname = layerbranch.actual_branch
+ branchdesc = "%s (%s)" % (options.branch, branchname)
+
+ # Collect repo info
+ repo = git.Repo(repodir)
+ assert repo.bare == False
+ try:
+ if options.nocheckout:
+ topcommit = repo.commit('HEAD')
+ else:
+ topcommit = repo.commit('origin/%s' % branchname)
+ except:
+ if layerbranch:
+ logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
+ else:
+ logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
+ transaction.rollback()
+ sys.exit(1)
+
+ newbranch = False
+ if not layerbranch:
+ # LayerBranch doesn't exist for this branch, create it
+ newbranch = True
+ layerbranch = LayerBranch()
+ layerbranch.layer = layer
+ layerbranch.branch = branch
+ layerbranch_source = layer.get_layerbranch('master')
+ if not layerbranch_source:
+ layerbranch_source = layer.get_layerbranch(None)
+ if layerbranch_source:
+ layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
+ layerbranch.save()
+ if layerbranch_source:
+ for maintainer in layerbranch_source.layermaintainer_set.all():
+ maintainer.pk = None
+ maintainer.id = None
+ maintainer.layerbranch = layerbranch
+ maintainer.save()
+ for dep in layerbranch_source.dependencies_set.all():
+ dep.pk = None
+ dep.id = None
+ dep.layerbranch = layerbranch
+ dep.save()
+
+ if layerbranch.vcs_subdir and not options.nocheckout:
+ # Find latest commit in subdirectory
+ # A bit odd to do it this way but apparently there's no other way in the GitPython API
+ topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
+ if not topcommit:
+ # This will error out if the directory is completely invalid or had never existed at this point
+ # If it previously existed but has since been deleted, you will get the revision where it was
+ # deleted - so we need to handle that case separately later
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ elif layerbranch.vcs_subdir:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ else:
+ logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
+ transaction.rollback()
+ sys.exit(1)
+
+ layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+ layerdir_start = os.path.normpath(layerdir) + os.sep
+ layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
+ layermachines = Machine.objects.filter(layerbranch=layerbranch)
+ layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
+ layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
+ if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
+ # Check out appropriate branch
+ if not options.nocheckout:
+ out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
+ out = utils.runcmd("git clean -f -x", repodir, logger=logger)
+
+ if layerbranch.vcs_subdir and not os.path.exists(layerdir):
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ else:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ transaction.rollback()
+ sys.exit(1)
+
+ if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
+ logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
+ transaction.rollback()
+ sys.exit(1)
+
+ logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+
+ try:
+ config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
+ except recipeparse.RecipeParseError as e:
+ logger.error(str(e))
+ transaction.rollback()
+ sys.exit(1)
+
+ if layerbranch.vcs_last_rev and not options.reload:
+ try:
+ diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
+ except Exception as e:
+ logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
+ diff = None
+ else:
+ diff = None
+
+ # We handle recipes specially to try to preserve the same id
+ # when recipe upgrades happen (so that if a user bookmarks a
+ # recipe page it remains valid)
+ layerrecipes_delete = []
+ layerrecipes_add = []
+
+ # Check if any paths should be ignored because there are layers within this layer
+ removedirs = []
+ for root, dirs, files in os.walk(layerdir):
+ for d in dirs:
+ if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
+ removedirs.append(os.path.join(root, d) + os.sep)
+
+ if diff:
+ # Apply git changes to existing recipe list
+
+ if layerbranch.vcs_subdir:
+ subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+ else:
+ subdir_start = ""
+
+ updatedrecipes = set()
+ for d in diff.iter_change_type('D'):
+ path = d.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for d in removedirs:
+ if path.startswith(d):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
+ if len(values):
+ layerrecipes_delete.append(values[0])
+ logger.debug("Mark %s for deletion" % values[0])
+ updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+ else:
+ logger.warn("Deleted recipe %s could not be found" % path)
+ elif typename == 'bbappend':
+ layerappends.filter(filepath=filepath).filter(filename=filename).delete()
+ elif typename == 'machine':
+ layermachines.filter(name=filename).delete()
+ elif typename == 'bbclass':
+ layerclasses.filter(name=filename).delete()
+
+ for d in diff.iter_change_type('A'):
+ path = d.b_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for d in removedirs:
+ if path.startswith(d):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ layerrecipes_add.append(os.path.join(repodir, path))
+ logger.debug("Mark %s for addition" % path)
+ updatedrecipes.add(os.path.join(filepath, filename))
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = filename
+ append.filepath = filepath
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ dirtyrecipes = set()
+ for d in diff.iter_change_type('M'):
+ path = d.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for d in removedirs:
+ if path.startswith(d):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ logger.debug("Mark %s for update" % path)
+ results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
+ if results:
+ recipe = results[0]
+ update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
+ recipe.save()
+ updatedrecipes.add(recipe.full_path())
+ elif typename == 'machine':
+ results = layermachines.filter(name=filename)
+ if results:
+ machine = results[0]
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+ for recipe in dirtyrecipes:
+ if not recipe.full_path() in updatedrecipes:
+ update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+ else:
+ # Collect recipe data from scratch
+
+ layerrecipe_fns = []
+ if options.fullreload:
+ layerrecipes.delete()
+ else:
+ # First, check which recipes still exist
+ layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
+ for v in layerrecipe_values:
+ root = os.path.join(layerdir, v['filepath'])
+ fullpath = os.path.join(root, v['filename'])
+ preserve = True
+ if os.path.exists(fullpath):
+ for d in removedirs:
+ if fullpath.startswith(d):
+ preserve = False
+ break
+ else:
+ preserve = False
+
+ if preserve:
+ # Recipe still exists, update it
+ results = layerrecipes.filter(id=v['id'])[:1]
+ recipe = results[0]
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ else:
+ # Recipe no longer exists, mark it for later on
+ layerrecipes_delete.append(v)
+ layerrecipe_fns.append(fullpath)
+
+ layermachines.delete()
+ layerappends.delete()
+ layerclasses.delete()
+ for root, dirs, files in os.walk(layerdir):
+ if '.git' in dirs:
+ dirs.remove('.git')
+ for d in dirs[:]:
+ fullpath = os.path.join(root, d) + os.sep
+ if fullpath in removedirs:
+ dirs.remove(d)
+ for f in files:
+ fullpath = os.path.join(root, f)
+ (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
+ if typename == 'recipe':
+ if fullpath not in layerrecipe_fns:
+ layerrecipes_add.append(fullpath)
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = f
+ append.filepath = os.path.relpath(root, layerdir)
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(fullpath, machine)
+ machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ for added in layerrecipes_add:
+ # This is good enough without actually parsing the file
+ (pn, pv) = split_recipe_fn(added)
+ oldid = -1
+ for deleted in layerrecipes_delete:
+ if deleted['pn'] == pn:
+ oldid = deleted['id']
+ layerrecipes_delete.remove(deleted)
+ break
+ if oldid > -1:
+ # Reclaim a record we would have deleted
+ results = Recipe.objects.filter(id=oldid)[:1]
+ recipe = results[0]
+ logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
+ else:
+ # Create new record
+ logger.debug("Add new recipe %s" % added)
+ recipe = Recipe()
+ recipe.layerbranch = layerbranch
+ recipe.filename = os.path.basename(added)
+ root = os.path.dirname(added)
+ recipe.filepath = os.path.relpath(root, layerdir)
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ recipe.save()
+
+ for deleted in layerrecipes_delete:
+ logger.debug("Delete %s" % deleted)
+ results = Recipe.objects.filter(id=deleted['id'])[:1]
+ recipe = results[0]
+ recipe.delete()
+
+ # Save repo info
+ layerbranch.vcs_last_rev = topcommit.hexsha
+ layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
+ else:
+ logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
+
+ layerbranch.vcs_last_fetch = datetime.now()
+ layerbranch.save()
+
+ if options.dryrun:
+ transaction.rollback()
+ else:
+ transaction.commit()
+
+ except KeyboardInterrupt:
+ transaction.rollback()
+ logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
+ sys.exit(254)
+ except:
+ import traceback
+ traceback.print_exc()
+ transaction.rollback()
+ finally:
+ transaction.leave_transaction_management()
+
+ shutil.rmtree(tempdir)
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ main()
--
2.5.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [layerindex-web][PATCH 3/5] update.py: allow updating all branches with one command
2016-05-30 4:31 [layerindex-web][PATCH 0/5] Layer index improvements Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 1/5] Allow blanking out field values in bulk change Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 2/5] update.py: refactor into two separate scripts Paul Eggleton
@ 2016-05-30 4:31 ` Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 4/5] Fix listing *_git.bbappend as appends for git recipe Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 5/5] Increase size of Recipe provides and license fields Paul Eggleton
4 siblings, 0 replies; 6+ messages in thread
From: Paul Eggleton @ 2016-05-30 4:31 UTC (permalink / raw)
To: yocto
Allow updating multiple branches, and if no branches are specified,
update all branches that have a new "updates_enabled" flag field set to
True. This avoids the need to have a separate shell script which runs
update.py for each branch (and thus has hardcoded knowledge of each
active branch in the index, i.e. it needs to be kept up-to-date in
addition to the database.)
The migration will default updates_enabled to True for all branches so
if you wish to take advantage of this functionality, the flag will need
to be set to False for any branches that shouldn't be updated.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
README | 7 +-
.../0011_auto__add_field_branch_updates_enabled.py | 198 +++++++++++++++++++++
layerindex/models.py | 1 +
layerindex/update.py | 70 ++++----
4 files changed, 239 insertions(+), 37 deletions(-)
create mode 100644 layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py
diff --git a/README b/README
index 84edee2..ff0a7a5 100644
--- a/README
+++ b/README
@@ -119,11 +119,8 @@ On a regular basis you need to run the update script:
path/to/layerindex/update.py
This will fetch all of the layer repositories, analyse their contents
-and update the database with the results. Note that if you set up more
-than just the master branch in the database, you will need to run the
-script once for each branch using -b (or --branch) to specify the
-branch name. Run the script with --help for further information on
-available options.
+and update the database with the results. Run the script with --help for
+further information on available options.
Maintenance
diff --git a/layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py b/layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py
new file mode 100644
index 0000000..5e2d4bf
--- /dev/null
+++ b/layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py
@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding field 'Branch.updates_enabled'
+ db.add_column('layerindex_branch', 'updates_enabled',
+ self.gf('django.db.models.fields.BooleanField')(default=True),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ # Deleting field 'Branch.updates_enabled'
+ db.delete_column('layerindex_branch', 'updates_enabled')
+
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ 'auth.permission': {
+ 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.bbappend': {
+ 'Meta': {'object_name': 'BBAppend'},
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.bbclass': {
+ 'Meta': {'object_name': 'BBClass'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.branch': {
+ 'Meta': {'object_name': 'Branch'},
+ 'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+ 'sort_priority': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'auto_now': 'True', 'blank': 'True'}),
+ 'updates_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+ },
+ 'layerindex.classicrecipe': {
+ 'Meta': {'object_name': 'ClassicRecipe', '_ormbases': ['layerindex.Recipe']},
+ 'classic_category': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'cover_layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
+ 'cover_pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_status': ('django.db.models.fields.CharField', [], {'default': "'U'", 'max_length': '1'}),
+ 'cover_verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'recipe_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['layerindex.Recipe']", 'unique': 'True', 'primary_key': 'True'})
+ },
+ 'layerindex.layerbranch': {
+ 'Meta': {'object_name': 'LayerBranch'},
+ 'actual_branch': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_rev': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'vcs_subdir': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'})
+ },
+ 'layerindex.layerdependency': {
+ 'Meta': {'object_name': 'LayerDependency'},
+ 'dependency': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependents_set'", 'to': "orm['layerindex.LayerItem']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependencies_set'", 'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.layeritem': {
+ 'Meta': {'object_name': 'LayerItem'},
+ 'classic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'description': ('django.db.models.fields.TextField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+ 'mailing_list_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'usage_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+ },
+ 'layerindex.layermaintainer': {
+ 'Meta': {'object_name': 'LayerMaintainer'},
+ 'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'responsibility': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
+ },
+ 'layerindex.layernote': {
+ 'Meta': {'object_name': 'LayerNote'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'text': ('django.db.models.fields.TextField', [], {})
+ },
+ 'layerindex.machine': {
+ 'Meta': {'object_name': 'Machine'},
+ 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipe': {
+ 'Meta': {'object_name': 'Recipe'},
+ 'bbclassextend': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'blacklisted': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'inherits': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'provides': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'pv': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipechange': {
+ 'Meta': {'object_name': 'RecipeChange'},
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'changeset': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.RecipeChangeset']"}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.Recipe']"}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
+ },
+ 'layerindex.recipechangeset': {
+ 'Meta': {'object_name': 'RecipeChangeset'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+ },
+ 'layerindex.recipefiledependency': {
+ 'Meta': {'object_name': 'RecipeFileDependency'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.LayerBranch']"}),
+ 'path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
+ }
+ }
+
+ complete_apps = ['layerindex']
\ No newline at end of file
diff --git a/layerindex/models.py b/layerindex/models.py
index e0d85ea..6c5f65f 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -19,6 +19,7 @@ class Branch(models.Model):
bitbake_branch = models.CharField(max_length=50)
short_description = models.CharField(max_length=50, blank=True)
sort_priority = models.IntegerField(blank=True, null=True)
+ updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
updated = models.DateTimeField(auto_now = True, default = datetime.now)
diff --git a/layerindex/update.py b/layerindex/update.py
index f7cb25c..ee4138d 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -57,8 +57,8 @@ def main():
%prog [options]""")
parser.add_option("-b", "--branch",
- help = "Specify branch to update",
- action="store", dest="branch", default='master')
+ help = "Specify branch(es) to update (use commas to separate multiple). Default is all enabled branches.",
+ action="store", dest="branch", default='')
parser.add_option("-l", "--layer",
help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
action="store", dest="layers")
@@ -92,14 +92,19 @@ def main():
utils.setup_django()
import settings
- from layerindex.models import LayerItem
+ from layerindex.models import Branch, LayerItem
logger.setLevel(options.loglevel)
- branch = utils.get_branch(options.branch)
- if not branch:
- logger.error("Specified branch %s is not valid" % options.branch)
- sys.exit(1)
+ if options.branch:
+ branches = options.branch.split(',')
+ for branch in branches:
+ if not utils.get_branch(branch):
+ logger.error("Specified branch %s is not valid" % branch)
+ sys.exit(1)
+ else:
+ branchquery = Branch.objects.filter(updates_enabled=True)
+ branches = [branch.name for branch in branchquery]
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
@@ -164,31 +169,32 @@ def main():
# We now do this by calling out to a separate script; doing otherwise turned out to be
# unreliable due to leaking memory (we're using bitbake internals in a manner in which
# they never get used during normal operation).
- for layer in layerquery:
- if layer.vcs_url in failedrepos:
- logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
-
- urldir = layer.get_fetch_dir()
- repodir = os.path.join(fetchdir, urldir)
-
- cmd = 'python update_layer.py -l %s -b %s' % (layer.name, options.branch)
- if options.reload:
- cmd += ' --reload'
- if options.fullreload:
- cmd += ' --fullreload'
- if options.nocheckout:
- cmd += ' --nocheckout'
- if options.dryrun:
- cmd += ' -n'
- if options.loglevel == logging.DEBUG:
- cmd += ' -d'
- elif options.loglevel == logging.ERROR:
- cmd += ' -q'
- logger.debug('Running layer update command: %s' % cmd)
- ret = run_command_interruptible(cmd)
- if ret == 254:
- # Interrupted by user, break out of loop
- break
+ for branch in branches:
+ for layer in layerquery:
+ if layer.vcs_url in failedrepos:
+ logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
+
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
+
+ cmd = 'python update_layer.py -l %s -b %s' % (layer.name, branch)
+ if options.reload:
+ cmd += ' --reload'
+ if options.fullreload:
+ cmd += ' --fullreload'
+ if options.nocheckout:
+ cmd += ' --nocheckout'
+ if options.dryrun:
+ cmd += ' -n'
+ if options.loglevel == logging.DEBUG:
+ cmd += ' -d'
+ elif options.loglevel == logging.ERROR:
+ cmd += ' -q'
+ logger.debug('Running layer update command: %s' % cmd)
+ ret = run_command_interruptible(cmd)
+ if ret == 254:
+ # Interrupted by user, break out of loop
+ break
finally:
utils.unlock_file(lockfile)
--
2.5.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [layerindex-web][PATCH 4/5] Fix listing *_git.bbappend as appends for git recipe
2016-05-30 4:31 [layerindex-web][PATCH 0/5] Layer index improvements Paul Eggleton
` (2 preceding siblings ...)
2016-05-30 4:31 ` [layerindex-web][PATCH 3/5] update.py: allow updating all branches with one command Paul Eggleton
@ 2016-05-30 4:31 ` Paul Eggleton
2016-05-30 4:31 ` [layerindex-web][PATCH 5/5] Increase size of Recipe provides and license fields Paul Eggleton
4 siblings, 0 replies; 6+ messages in thread
From: Paul Eggleton @ 2016-05-30 4:31 UTC (permalink / raw)
To: yocto
Fix a greedy regex in the recipe detail view resulting in the git recipe
listing all bbappends named *_git.bbappend as its bbappends (quite a few
in the public instance).
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
TODO | 1 -
layerindex/views.py | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/TODO b/TODO
index 2d5a53d..9ab7aa7 100644
--- a/TODO
+++ b/TODO
@@ -10,7 +10,6 @@ TODO:
* Make it easy to update people's email addresses
Bugs
-* git recipe shows up all _git bbappends: http://layers.openembedded.org/layerindex/recipe/5542/
* Duplication of first maintainer when editing to add a second?
* PROVIDES column appears to be too short for some recipes
diff --git a/layerindex/views.py b/layerindex/views.py
index d033046..d9480c9 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -672,7 +672,7 @@ class RecipeDetailView(DetailView):
verappendprefix = recipe.filename.split('.bb')[0]
appendprefix = verappendprefix.split('_')[0]
#context['verappends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename='%s.bbappend' % verappendprefix)
- context['appends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename__regex=r'%s(_[^_]*)?\.bbappend' % appendprefix)
+ context['appends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename__regex=r'^%s(_[^_]*)?\.bbappend' % appendprefix)
verappends = []
for append in context['appends']:
if append.matches_recipe(recipe):
--
2.5.5
^ permalink raw reply related [flat|nested] 6+ messages in thread* [layerindex-web][PATCH 5/5] Increase size of Recipe provides and license fields
2016-05-30 4:31 [layerindex-web][PATCH 0/5] Layer index improvements Paul Eggleton
` (3 preceding siblings ...)
2016-05-30 4:31 ` [layerindex-web][PATCH 4/5] Fix listing *_git.bbappend as appends for git recipe Paul Eggleton
@ 2016-05-30 4:31 ` Paul Eggleton
4 siblings, 0 replies; 6+ messages in thread
From: Paul Eggleton @ 2016-05-30 4:31 UTC (permalink / raw)
To: yocto
Fix "data truncated" warnings/errors when loading data for OE-Core:
* PROVIDES for recipe packagegroup-base is ~1452 characters
* LICENSE for recipe linux-firmware is ~1053 characters
(These aren't shown with SQLite, only with something like MariaDB where
column sizes are enforced.)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
TODO | 1 -
...ld_recipe_license__chg_field_recipe_provides.py | 202 +++++++++++++++++++++
layerindex/models.py | 4 +-
3 files changed, 204 insertions(+), 3 deletions(-)
create mode 100644 layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py
diff --git a/TODO b/TODO
index 9ab7aa7..b5e8974 100644
--- a/TODO
+++ b/TODO
@@ -11,7 +11,6 @@ TODO:
Bugs
* Duplication of first maintainer when editing to add a second?
-* PROVIDES column appears to be too short for some recipes
Other
* Full-text search on layer contents
diff --git a/layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py b/layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py
new file mode 100644
index 0000000..52cc264
--- /dev/null
+++ b/layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py
@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Changing field 'Recipe.license'
+ db.alter_column('layerindex_recipe', 'license', self.gf('django.db.models.fields.CharField')(max_length=2048))
+
+ # Changing field 'Recipe.provides'
+ db.alter_column('layerindex_recipe', 'provides', self.gf('django.db.models.fields.CharField')(max_length=2048))
+
+ def backwards(self, orm):
+
+ # Changing field 'Recipe.license'
+ db.alter_column('layerindex_recipe', 'license', self.gf('django.db.models.fields.CharField')(max_length=100))
+
+ # Changing field 'Recipe.provides'
+ db.alter_column('layerindex_recipe', 'provides', self.gf('django.db.models.fields.CharField')(max_length=255))
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ 'auth.permission': {
+ 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.bbappend': {
+ 'Meta': {'object_name': 'BBAppend'},
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.bbclass': {
+ 'Meta': {'object_name': 'BBClass'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.branch': {
+ 'Meta': {'object_name': 'Branch'},
+ 'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+ 'sort_priority': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'auto_now': 'True', 'blank': 'True'}),
+ 'updates_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+ },
+ 'layerindex.classicrecipe': {
+ 'Meta': {'object_name': 'ClassicRecipe', '_ormbases': ['layerindex.Recipe']},
+ 'classic_category': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'cover_layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
+ 'cover_pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_status': ('django.db.models.fields.CharField', [], {'default': "'U'", 'max_length': '1'}),
+ 'cover_verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'recipe_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['layerindex.Recipe']", 'unique': 'True', 'primary_key': 'True'})
+ },
+ 'layerindex.layerbranch': {
+ 'Meta': {'object_name': 'LayerBranch'},
+ 'actual_branch': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_rev': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'vcs_subdir': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'})
+ },
+ 'layerindex.layerdependency': {
+ 'Meta': {'object_name': 'LayerDependency'},
+ 'dependency': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependents_set'", 'to': "orm['layerindex.LayerItem']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependencies_set'", 'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.layeritem': {
+ 'Meta': {'object_name': 'LayerItem'},
+ 'classic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'description': ('django.db.models.fields.TextField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+ 'mailing_list_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'usage_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+ },
+ 'layerindex.layermaintainer': {
+ 'Meta': {'object_name': 'LayerMaintainer'},
+ 'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'responsibility': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
+ },
+ 'layerindex.layernote': {
+ 'Meta': {'object_name': 'LayerNote'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'text': ('django.db.models.fields.TextField', [], {})
+ },
+ 'layerindex.machine': {
+ 'Meta': {'object_name': 'Machine'},
+ 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipe': {
+ 'Meta': {'object_name': 'Recipe'},
+ 'bbclassextend': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'blacklisted': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'inherits': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}),
+ 'pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'provides': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}),
+ 'pv': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipechange': {
+ 'Meta': {'object_name': 'RecipeChange'},
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'changeset': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.RecipeChangeset']"}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.Recipe']"}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
+ },
+ 'layerindex.recipechangeset': {
+ 'Meta': {'object_name': 'RecipeChangeset'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+ },
+ 'layerindex.recipefiledependency': {
+ 'Meta': {'object_name': 'RecipeFileDependency'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.LayerBranch']"}),
+ 'path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
+ }
+ }
+
+ complete_apps = ['layerindex']
\ No newline at end of file
diff --git a/layerindex/models.py b/layerindex/models.py
index 6c5f65f..8d82846 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -238,10 +238,10 @@ class Recipe(models.Model):
summary = models.CharField(max_length=200, blank=True)
description = models.TextField(blank=True)
section = models.CharField(max_length=100, blank=True)
- license = models.CharField(max_length=100, blank=True)
+ license = models.CharField(max_length=2048, blank=True)
homepage = models.URLField(blank=True)
bugtracker = models.URLField(blank=True)
- provides = models.CharField(max_length=255, blank=True)
+ provides = models.CharField(max_length=2048, blank=True)
bbclassextend = models.CharField(max_length=100, blank=True)
inherits = models.CharField(max_length=255, blank=True)
updated = models.DateTimeField(auto_now = True)
--
2.5.5
^ permalink raw reply related [flat|nested] 6+ messages in thread