* [layerindex-web][PATCH v2 01/15] Allow blanking out field values in bulk change
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
@ 2016-06-08 13:19 ` Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 02/15] update.py: refactor into two separate scripts Paul Eggleton
` (13 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:19 UTC (permalink / raw)
To: yocto
If you're moving a short description value from DESCRIPTION to SUMMARY
then part of that is setting DESCRIPTION to blank, however that wasn't
possible - the code was assuming that a null value meant "keep the
original value". Change the logic so that the value in the bulk change
object is always set and is compared to the original value to see if it
is different. This provides less safety against bulk change data going
stale in the face of the metadata being updated, but without using an
additional "magic" field value that's the price we have to pay, and it's
unlikely to bother too many people I would imagine.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/bulkchange.py | 17 +++++++++--------
layerindex/forms.py | 19 -------------------
layerindex/models.py | 20 ++++++++++++--------
layerindex/views.py | 3 +--
4 files changed, 22 insertions(+), 37 deletions(-)
diff --git a/layerindex/bulkchange.py b/layerindex/bulkchange.py
index ed36859..c569b61 100644
--- a/layerindex/bulkchange.py
+++ b/layerindex/bulkchange.py
@@ -97,14 +97,15 @@ def patch_recipe(fn, relpath, values):
with tempfile.NamedTemporaryFile('w', delete=False) as tf:
def outputvalue(name):
- rawtext = '%s = "%s"\n' % (name, values[name])
- if name in nowrap_vars:
- tf.write(rawtext)
- else:
- wrapped = textwrap.wrap(rawtext)
- for wrapline in wrapped[:-1]:
- tf.write('%s \\\n' % wrapline)
- tf.write('%s\n' % wrapped[-1])
+ if values[name]:
+ rawtext = '%s = "%s"\n' % (name, values[name])
+ if name in nowrap_vars:
+ tf.write(rawtext)
+ else:
+ wrapped = textwrap.wrap(rawtext)
+ for wrapline in wrapped[:-1]:
+ tf.write('%s \\\n' % wrapline)
+ tf.write('%s\n' % wrapped[-1])
tfn = tf.name
with open(fn, 'r') as f:
diff --git a/layerindex/forms.py b/layerindex/forms.py
index 78711e5..60653cf 100644
--- a/layerindex/forms.py
+++ b/layerindex/forms.py
@@ -189,25 +189,6 @@ class BulkChangeEditForm(forms.ModelForm):
model = RecipeChange
fields = ('summary', 'description', 'homepage', 'bugtracker', 'section', 'license')
- def __init__(self, *args, **kwargs):
- instance = kwargs.get('instance', None)
- initial = kwargs.get('initial', {})
- if instance:
- recipe = instance.recipe
- if recipe:
- for fieldname in self._meta.fields:
- if not getattr(instance, fieldname):
- initial[fieldname] = getattr(recipe, fieldname)
- kwargs['initial'] = initial
- super(BulkChangeEditForm, self).__init__(*args, **kwargs)
-
- def clear_same_values(self):
- for fieldname in self._meta.fields:
- oldval = getattr(self.instance.recipe, fieldname)
- newval = getattr(self.instance, fieldname)
- if oldval == newval:
- setattr(self.instance, fieldname, '')
-
BulkChangeEditFormSet = modelformset_factory(RecipeChange, form=BulkChangeEditForm, extra=0)
diff --git a/layerindex/models.py b/layerindex/models.py
index cde8fe3..e0d85ea 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -415,12 +415,16 @@ class RecipeChange(models.Model):
def changed_fields(self, mapped = False):
res = {}
- for field in self._meta.fields:
- if not field.name in ['id', 'changeset', 'recipe']:
- value = getattr(self, field.name)
- if value:
- if mapped:
- res[self.RECIPE_VARIABLE_MAP[field.name]] = value
- else:
- res[field.name] = value
+ for fieldname in self.RECIPE_VARIABLE_MAP:
+ value = getattr(self, fieldname)
+ origvalue = getattr(self.recipe, fieldname)
+ if value != origvalue:
+ if mapped:
+ res[self.RECIPE_VARIABLE_MAP[fieldname]] = value
+ else:
+ res[fieldname] = value
return res
+
+ def reset_fields(self):
+ for fieldname in self.RECIPE_VARIABLE_MAP:
+ setattr(self, fieldname, getattr(self.recipe, fieldname))
diff --git a/layerindex/views.py b/layerindex/views.py
index 898a7c4..d033046 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -200,8 +200,6 @@ def bulk_change_edit_view(request, template_name, pk):
if request.method == 'POST':
formset = BulkChangeEditFormSet(request.POST, queryset=changeset.recipechange_set.all())
if formset.is_valid():
- for form in formset:
- form.clear_same_values()
formset.save()
return HttpResponseRedirect(reverse('bulk_change_review', args=(changeset.id,)))
else:
@@ -524,6 +522,7 @@ class BulkChangeSearchView(AdvancedRecipeSearchView):
change = RecipeChange()
change.changeset = changeset
change.recipe = recipe
+ change.reset_fields()
change.save()
if 'add_selected' in request.POST:
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 02/15] update.py: refactor into two separate scripts
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 01/15] Allow blanking out field values in bulk change Paul Eggleton
@ 2016-06-08 13:19 ` Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 03/15] update.py: allow updating all branches with one command Paul Eggleton
` (12 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:19 UTC (permalink / raw)
To: yocto
In order to try to avoid problems with leaking memory, context bleeding
from one layer to another, and lay the groundwork for supporting
updating all branches in one operation, split the updating of a single
layer out to its own internal script. This means that the tinfoil
instantiation and metadata parsing is in a completely separate process
per layer.
Implements [YOCTO #9647].
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/update.py | 511 ++++--------------------------------------
layerindex/update_layer.py | 547 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 591 insertions(+), 467 deletions(-)
create mode 100644 layerindex/update_layer.py
diff --git a/layerindex/update.py b/layerindex/update.py
index 7daccef..f7cb25c 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -2,23 +2,20 @@
# Fetch layer repositories and update layer index database
#
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
-import os.path
+import os
import optparse
import logging
-from datetime import datetime
-import re
-import tempfile
-import shutil
+import subprocess
+import signal
from distutils.version import LooseVersion
import utils
-import recipeparse
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -33,79 +30,21 @@ except ImportError:
sys.exit(1)
-def check_machine_conf(path, subdir_start):
- subpath = path[len(subdir_start):]
- res = conf_re.match(subpath)
- if res:
- return res.group(1)
- return None
+def reenable_sigint():
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
-def split_recipe_fn(path):
- splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
- pn = splitfn[0]
- if len(splitfn) > 1:
- pv = splitfn[1]
- else:
- pv = "1.0"
- return (pn, pv)
-
-def update_recipe_file(data, path, recipe, layerdir_start, repodir):
- fn = str(os.path.join(path, recipe.filename))
+def run_command_interruptible(cmd):
+ """
+ Run a command with output displayed on the console, but ensure any Ctrl+C is
+ processed only by the child process.
+ """
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
try:
- logger.debug('Updating recipe %s' % fn)
- envdata = bb.cache.Cache.loadDataFull(fn, [], data)
- envdata.setVar('SRCPV', 'X')
- recipe.pn = envdata.getVar("PN", True)
- recipe.pv = envdata.getVar("PV", True)
- recipe.summary = envdata.getVar("SUMMARY", True)
- recipe.description = envdata.getVar("DESCRIPTION", True)
- recipe.section = envdata.getVar("SECTION", True)
- recipe.license = envdata.getVar("LICENSE", True)
- recipe.homepage = envdata.getVar("HOMEPAGE", True)
- recipe.bugtracker = envdata.getVar("BUGTRACKER", True) or ""
- recipe.provides = envdata.getVar("PROVIDES", True) or ""
- recipe.bbclassextend = envdata.getVar("BBCLASSEXTEND", True) or ""
- # Handle recipe inherits for this recipe
- gr = set(data.getVar("__inherit_cache", True) or [])
- lr = set(envdata.getVar("__inherit_cache", True) or [])
- recipe.inherits = ' '.join(sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}))
- recipe.blacklisted = envdata.getVarFlag('PNBLACKLIST', recipe.pn, True) or ""
- recipe.save()
-
- # Get file dependencies within this layer
- deps = envdata.getVar('__depends', True)
- filedeps = []
- for depstr, date in deps:
- found = False
- if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
- filedeps.append(os.path.relpath(depstr, repodir))
- from layerindex.models import RecipeFileDependency
- RecipeFileDependency.objects.filter(recipe=recipe).delete()
- for filedep in filedeps:
- recipedep = RecipeFileDependency()
- recipedep.layerbranch = recipe.layerbranch
- recipedep.recipe = recipe
- recipedep.path = filedep
- recipedep.save()
- except KeyboardInterrupt:
- raise
- except BaseException as e:
- if not recipe.pn:
- recipe.pn = recipe.filename[:-3].split('_')[0]
- logger.error("Unable to read %s: %s", fn, str(e))
-
-def update_machine_conf_file(path, machine):
- logger.debug('Updating machine %s' % path)
- desc = ""
- with open(path, 'r') as f:
- for line in f:
- if line.startswith('#@NAME:'):
- desc = line[7:].strip()
- if line.startswith('#@DESCRIPTION:'):
- desc = line[14:].strip()
- desc = re.sub(r'Machine configuration for( running)*( an)*( the)*', '', desc)
- break
- machine.description = desc
+ ret = subprocess.call(cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint)
+ finally:
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
+ return ret
+
def main():
if LooseVersion(git.__version__) < '0.3.1':
@@ -151,13 +90,9 @@ def main():
parser.print_help()
sys.exit(1)
- if options.fullreload:
- options.reload = True
-
utils.setup_django()
import settings
- from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
- from django.db import transaction
+ from layerindex.models import LayerItem
logger.setLevel(options.loglevel)
@@ -225,397 +160,39 @@ def main():
else:
out = utils.runcmd("git fetch", bitbakepath, logger=logger)
- try:
- (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
- except recipeparse.RecipeParseError as e:
- logger.error(str(e))
- sys.exit(1)
-
- # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
- tinfoil.config_data.setVar('SUMMARY', '')
- # Clear the default value of DESCRIPTION so that we can see where it's not set
- tinfoil.config_data.setVar('DESCRIPTION', '')
- # Clear the default value of HOMEPAGE ('unknown')
- tinfoil.config_data.setVar('HOMEPAGE', '')
- # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
- # why won't they just fix that?!)
- tinfoil.config_data.setVar('LICENSE', '')
-
# Process and extract data from each layer
+ # We now do this by calling out to a separate script; doing otherwise turned out to be
+ # unreliable due to leaking memory (we're using bitbake internals in a manner in which
+ # they never get used during normal operation).
for layer in layerquery:
- transaction.enter_transaction_management()
- transaction.managed(True)
- try:
- urldir = layer.get_fetch_dir()
- repodir = os.path.join(fetchdir, urldir)
- if layer.vcs_url in failedrepos:
- logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
- transaction.rollback()
- continue
-
- layerbranch = layer.get_layerbranch(options.branch)
-
- branchname = options.branch
- branchdesc = options.branch
- if layerbranch:
- if layerbranch.actual_branch:
- branchname = layerbranch.actual_branch
- branchdesc = "%s (%s)" % (options.branch, branchname)
-
- # Collect repo info
- repo = git.Repo(repodir)
- assert repo.bare == False
- try:
- if options.nocheckout:
- topcommit = repo.commit('HEAD')
- else:
- topcommit = repo.commit('origin/%s' % branchname)
- except:
- if layerbranch:
- logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
- else:
- logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
- transaction.rollback()
- continue
-
- newbranch = False
- if not layerbranch:
- # LayerBranch doesn't exist for this branch, create it
- newbranch = True
- layerbranch = LayerBranch()
- layerbranch.layer = layer
- layerbranch.branch = branch
- layerbranch_source = layer.get_layerbranch('master')
- if not layerbranch_source:
- layerbranch_source = layer.get_layerbranch(None)
- if layerbranch_source:
- layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
- layerbranch.save()
- if layerbranch_source:
- for maintainer in layerbranch_source.layermaintainer_set.all():
- maintainer.pk = None
- maintainer.id = None
- maintainer.layerbranch = layerbranch
- maintainer.save()
- for dep in layerbranch_source.dependencies_set.all():
- dep.pk = None
- dep.id = None
- dep.layerbranch = layerbranch
- dep.save()
-
- if layerbranch.vcs_subdir and not options.nocheckout:
- # Find latest commit in subdirectory
- # A bit odd to do it this way but apparently there's no other way in the GitPython API
- topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
- if not topcommit:
- # This will error out if the directory is completely invalid or had never existed at this point
- # If it previously existed but has since been deleted, you will get the revision where it was
- # deleted - so we need to handle that case separately later
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- elif layerbranch.vcs_subdir:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- else:
- logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
- transaction.rollback()
- continue
-
- layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
- layerdir_start = os.path.normpath(layerdir) + os.sep
- layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
- layermachines = Machine.objects.filter(layerbranch=layerbranch)
- layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
- layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
- if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
- # Check out appropriate branch
- if not options.nocheckout:
- out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
- out = utils.runcmd("git clean -f -x", repodir, logger=logger)
-
- if layerbranch.vcs_subdir and not os.path.exists(layerdir):
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- else:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- transaction.rollback()
- continue
-
- if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
- logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
- transaction.rollback()
- continue
-
- logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
-
- try:
- config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
- except recipeparse.RecipeParseError as e:
- logger.error(str(e))
- transaction.rollback()
- continue
-
- if layerbranch.vcs_last_rev and not options.reload:
- try:
- diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
- except Exception as e:
- logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
- diff = None
- else:
- diff = None
-
- # We handle recipes specially to try to preserve the same id
- # when recipe upgrades happen (so that if a user bookmarks a
- # recipe page it remains valid)
- layerrecipes_delete = []
- layerrecipes_add = []
-
- # Check if any paths should be ignored because there are layers within this layer
- removedirs = []
- for root, dirs, files in os.walk(layerdir):
- for d in dirs:
- if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
- removedirs.append(os.path.join(root, d) + os.sep)
-
- if diff:
- # Apply git changes to existing recipe list
-
- if layerbranch.vcs_subdir:
- subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
- else:
- subdir_start = ""
-
- updatedrecipes = set()
- for d in diff.iter_change_type('D'):
- path = d.a_blob.path
- if path.startswith(subdir_start):
- skip = False
- for d in removedirs:
- if path.startswith(d):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
- if len(values):
- layerrecipes_delete.append(values[0])
- logger.debug("Mark %s for deletion" % values[0])
- updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
- else:
- logger.warn("Deleted recipe %s could not be found" % path)
- elif typename == 'bbappend':
- layerappends.filter(filepath=filepath).filter(filename=filename).delete()
- elif typename == 'machine':
- layermachines.filter(name=filename).delete()
- elif typename == 'bbclass':
- layerclasses.filter(name=filename).delete()
-
- for d in diff.iter_change_type('A'):
- path = d.b_blob.path
- if path.startswith(subdir_start):
- skip = False
- for d in removedirs:
- if path.startswith(d):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- layerrecipes_add.append(os.path.join(repodir, path))
- logger.debug("Mark %s for addition" % path)
- updatedrecipes.add(os.path.join(filepath, filename))
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = filename
- append.filepath = filepath
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(os.path.join(repodir, path), machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- dirtyrecipes = set()
- for d in diff.iter_change_type('M'):
- path = d.a_blob.path
- if path.startswith(subdir_start):
- skip = False
- for d in removedirs:
- if path.startswith(d):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- logger.debug("Mark %s for update" % path)
- results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
- if results:
- recipe = results[0]
- update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
- recipe.save()
- updatedrecipes.add(recipe.full_path())
- elif typename == 'machine':
- results = layermachines.filter(name=filename)
- if results:
- machine = results[0]
- update_machine_conf_file(os.path.join(repodir, path), machine)
- machine.save()
-
- deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
- for dep in deps:
- dirtyrecipes.add(dep.recipe)
-
- for recipe in dirtyrecipes:
- if not recipe.full_path() in updatedrecipes:
- update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
- else:
- # Collect recipe data from scratch
-
- layerrecipe_fns = []
- if options.fullreload:
- layerrecipes.delete()
- else:
- # First, check which recipes still exist
- layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
- for v in layerrecipe_values:
- root = os.path.join(layerdir, v['filepath'])
- fullpath = os.path.join(root, v['filename'])
- preserve = True
- if os.path.exists(fullpath):
- for d in removedirs:
- if fullpath.startswith(d):
- preserve = False
- break
- else:
- preserve = False
-
- if preserve:
- # Recipe still exists, update it
- results = layerrecipes.filter(id=v['id'])[:1]
- recipe = results[0]
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- else:
- # Recipe no longer exists, mark it for later on
- layerrecipes_delete.append(v)
- layerrecipe_fns.append(fullpath)
-
- layermachines.delete()
- layerappends.delete()
- layerclasses.delete()
- for root, dirs, files in os.walk(layerdir):
- if '.git' in dirs:
- dirs.remove('.git')
- for d in dirs[:]:
- fullpath = os.path.join(root, d) + os.sep
- if fullpath in removedirs:
- dirs.remove(d)
- for f in files:
- fullpath = os.path.join(root, f)
- (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
- if typename == 'recipe':
- if fullpath not in layerrecipe_fns:
- layerrecipes_add.append(fullpath)
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = f
- append.filepath = os.path.relpath(root, layerdir)
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(fullpath, machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- for added in layerrecipes_add:
- # This is good enough without actually parsing the file
- (pn, pv) = split_recipe_fn(added)
- oldid = -1
- for deleted in layerrecipes_delete:
- if deleted['pn'] == pn:
- oldid = deleted['id']
- layerrecipes_delete.remove(deleted)
- break
- if oldid > -1:
- # Reclaim a record we would have deleted
- results = Recipe.objects.filter(id=oldid)[:1]
- recipe = results[0]
- logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
- else:
- # Create new record
- logger.debug("Add new recipe %s" % added)
- recipe = Recipe()
- recipe.layerbranch = layerbranch
- recipe.filename = os.path.basename(added)
- root = os.path.dirname(added)
- recipe.filepath = os.path.relpath(root, layerdir)
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- recipe.save()
-
- for deleted in layerrecipes_delete:
- logger.debug("Delete %s" % deleted)
- results = Recipe.objects.filter(id=deleted['id'])[:1]
- recipe = results[0]
- recipe.delete()
-
- # Save repo info
- layerbranch.vcs_last_rev = topcommit.hexsha
- layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
- else:
- logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
-
- layerbranch.vcs_last_fetch = datetime.now()
- layerbranch.save()
-
- if options.dryrun:
- transaction.rollback()
- else:
- transaction.commit()
-
- # Slightly hacky way of avoiding memory leaks
- bb.event.ui_queue = []
- bb.parse.parse_py.BBHandler.cached_statements = {}
- bb.codeparser.codeparsercache = bb.codeparser.CodeParserCache()
- if hasattr(bb.codeparser, 'codecache'):
- bb.codeparser.codecache = bb.codeparser.SetCache()
- bb.fetch._checksum_cache = bb.checksum.FileChecksumCache()
- bb.fetch.urldata_cache = {}
- bb.fetch.saved_headrevs = {}
- bb.parse.__pkgsplit_cache__={}
- bb.parse.__mtime_cache = {}
- bb.parse.init_parser(tinfoil.config_data)
-
- except KeyboardInterrupt:
- transaction.rollback()
- logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
+ if layer.vcs_url in failedrepos:
+ logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
+
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
+
+ cmd = 'python update_layer.py -l %s -b %s' % (layer.name, options.branch)
+ if options.reload:
+ cmd += ' --reload'
+ if options.fullreload:
+ cmd += ' --fullreload'
+ if options.nocheckout:
+ cmd += ' --nocheckout'
+ if options.dryrun:
+ cmd += ' -n'
+ if options.loglevel == logging.DEBUG:
+ cmd += ' -d'
+ elif options.loglevel == logging.ERROR:
+ cmd += ' -q'
+ logger.debug('Running layer update command: %s' % cmd)
+ ret = run_command_interruptible(cmd)
+ if ret == 254:
+ # Interrupted by user, break out of loop
break
- except:
- import traceback
- traceback.print_exc()
- transaction.rollback()
- finally:
- transaction.leave_transaction_management()
finally:
utils.unlock_file(lockfile)
- shutil.rmtree(tempdir)
sys.exit(0)
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
new file mode 100644
index 0000000..bf1dbb2
--- /dev/null
+++ b/layerindex/update_layer.py
@@ -0,0 +1,547 @@
+#!/usr/bin/env python
+
+# Update layer index database for a single layer
+#
+# Copyright (C) 2013-2016 Intel Corporation
+# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+import optparse
+import logging
+from datetime import datetime
+import re
+import tempfile
+import shutil
+from distutils.version import LooseVersion
+import utils
+import recipeparse
+
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+logger = utils.logger_create('LayerIndexUpdate')
+
+# Ensure PythonGit is installed (buildhistory_analysis needs it)
+try:
+ import git
+except ImportError:
+ logger.error("Please install PythonGit 0.3.1 or later in order to use this script")
+ sys.exit(1)
+
+
+def check_machine_conf(path, subdir_start):
+ subpath = path[len(subdir_start):]
+ res = conf_re.match(subpath)
+ if res:
+ return res.group(1)
+ return None
+
+def split_recipe_fn(path):
+ splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
+ pn = splitfn[0]
+ if len(splitfn) > 1:
+ pv = splitfn[1]
+ else:
+ pv = "1.0"
+ return (pn, pv)
+
+def update_recipe_file(data, path, recipe, layerdir_start, repodir):
+ fn = str(os.path.join(path, recipe.filename))
+ try:
+ logger.debug('Updating recipe %s' % fn)
+ envdata = bb.cache.Cache.loadDataFull(fn, [], data)
+ envdata.setVar('SRCPV', 'X')
+ recipe.pn = envdata.getVar("PN", True)
+ recipe.pv = envdata.getVar("PV", True)
+ recipe.summary = envdata.getVar("SUMMARY", True)
+ recipe.description = envdata.getVar("DESCRIPTION", True)
+ recipe.section = envdata.getVar("SECTION", True)
+ recipe.license = envdata.getVar("LICENSE", True)
+ recipe.homepage = envdata.getVar("HOMEPAGE", True)
+ recipe.bugtracker = envdata.getVar("BUGTRACKER", True) or ""
+ recipe.provides = envdata.getVar("PROVIDES", True) or ""
+ recipe.bbclassextend = envdata.getVar("BBCLASSEXTEND", True) or ""
+ # Handle recipe inherits for this recipe
+ gr = set(data.getVar("__inherit_cache", True) or [])
+ lr = set(envdata.getVar("__inherit_cache", True) or [])
+ recipe.inherits = ' '.join(sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}))
+ recipe.blacklisted = envdata.getVarFlag('PNBLACKLIST', recipe.pn, True) or ""
+ recipe.save()
+
+ # Get file dependencies within this layer
+ deps = envdata.getVar('__depends', True)
+ filedeps = []
+ for depstr, date in deps:
+ found = False
+ if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
+ filedeps.append(os.path.relpath(depstr, repodir))
+ from layerindex.models import RecipeFileDependency
+ RecipeFileDependency.objects.filter(recipe=recipe).delete()
+ for filedep in filedeps:
+ recipedep = RecipeFileDependency()
+ recipedep.layerbranch = recipe.layerbranch
+ recipedep.recipe = recipe
+ recipedep.path = filedep
+ recipedep.save()
+ except KeyboardInterrupt:
+ raise
+ except BaseException as e:
+ if not recipe.pn:
+ recipe.pn = recipe.filename[:-3].split('_')[0]
+ logger.error("Unable to read %s: %s", fn, str(e))
+
+def update_machine_conf_file(path, machine):
+ logger.debug('Updating machine %s' % path)
+ desc = ""
+ with open(path, 'r') as f:
+ for line in f:
+ if line.startswith('#@NAME:'):
+ desc = line[7:].strip()
+ if line.startswith('#@DESCRIPTION:'):
+ desc = line[14:].strip()
+ desc = re.sub(r'Machine configuration for( running)*( an)*( the)*', '', desc)
+ break
+ machine.description = desc
+
+def main():
+ if LooseVersion(git.__version__) < '0.3.1':
+ logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
+ sys.exit(1)
+
+
+ parser = optparse.OptionParser(
+ usage = """
+ %prog [options]""")
+
+ parser.add_option("-b", "--branch",
+ help = "Specify branch to update",
+ action="store", dest="branch", default='master')
+ parser.add_option("-l", "--layer",
+ help = "Layer to update",
+ action="store", dest="layer")
+ parser.add_option("-r", "--reload",
+ help = "Reload recipe data instead of updating since last update",
+ action="store_true", dest="reload")
+ parser.add_option("", "--fullreload",
+ help = "Discard existing recipe data and fetch it from scratch",
+ action="store_true", dest="fullreload")
+ parser.add_option("-n", "--dry-run",
+ help = "Don't write any data back to the database",
+ action="store_true", dest="dryrun")
+ parser.add_option("", "--nocheckout",
+ help = "Don't check out branches",
+ action="store_true", dest="nocheckout")
+ parser.add_option("-d", "--debug",
+ help = "Enable debug output",
+ action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+ parser.add_option("-q", "--quiet",
+ help = "Hide all output except error messages",
+ action="store_const", const=logging.ERROR, dest="loglevel")
+
+ options, args = parser.parse_args(sys.argv)
+ if len(args) > 1:
+ logger.error('unexpected argument "%s"' % args[1])
+ parser.print_help()
+ sys.exit(1)
+
+ if options.fullreload:
+ options.reload = True
+
+ utils.setup_django()
+ import settings
+ from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
+ from django.db import transaction
+
+ logger.setLevel(options.loglevel)
+
+ branch = utils.get_branch(options.branch)
+ if not branch:
+ logger.error("Specified branch %s is not valid" % options.branch)
+ sys.exit(1)
+
+ fetchdir = settings.LAYER_FETCH_DIR
+ if not fetchdir:
+ logger.error("Please set LAYER_FETCH_DIR in settings.py")
+ sys.exit(1)
+
+ bitbakepath = os.path.join(fetchdir, 'bitbake')
+
+ try:
+ (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
+ except recipeparse.RecipeParseError as e:
+ logger.error(str(e))
+ sys.exit(1)
+
+ # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
+ tinfoil.config_data.setVar('SUMMARY', '')
+ # Clear the default value of DESCRIPTION so that we can see where it's not set
+ tinfoil.config_data.setVar('DESCRIPTION', '')
+ # Clear the default value of HOMEPAGE ('unknown')
+ tinfoil.config_data.setVar('HOMEPAGE', '')
+ # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
+ # why won't they just fix that?!)
+ tinfoil.config_data.setVar('LICENSE', '')
+
+ transaction.enter_transaction_management()
+ transaction.managed(True)
+ try:
+ layer = utils.get_layer(options.layer)
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
+
+ layerbranch = layer.get_layerbranch(options.branch)
+
+ branchname = options.branch
+ branchdesc = options.branch
+ if layerbranch:
+ if layerbranch.actual_branch:
+ branchname = layerbranch.actual_branch
+ branchdesc = "%s (%s)" % (options.branch, branchname)
+
+ # Collect repo info
+ repo = git.Repo(repodir)
+ assert repo.bare == False
+ try:
+ if options.nocheckout:
+ topcommit = repo.commit('HEAD')
+ else:
+ topcommit = repo.commit('origin/%s' % branchname)
+ except:
+ if layerbranch:
+ logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
+ else:
+ logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
+ transaction.rollback()
+ sys.exit(1)
+
+ newbranch = False
+ if not layerbranch:
+ # LayerBranch doesn't exist for this branch, create it
+ newbranch = True
+ layerbranch = LayerBranch()
+ layerbranch.layer = layer
+ layerbranch.branch = branch
+ layerbranch_source = layer.get_layerbranch('master')
+ if not layerbranch_source:
+ layerbranch_source = layer.get_layerbranch(None)
+ if layerbranch_source:
+ layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
+ layerbranch.save()
+ if layerbranch_source:
+ for maintainer in layerbranch_source.layermaintainer_set.all():
+ maintainer.pk = None
+ maintainer.id = None
+ maintainer.layerbranch = layerbranch
+ maintainer.save()
+ for dep in layerbranch_source.dependencies_set.all():
+ dep.pk = None
+ dep.id = None
+ dep.layerbranch = layerbranch
+ dep.save()
+
+ if layerbranch.vcs_subdir and not options.nocheckout:
+ # Find latest commit in subdirectory
+ # A bit odd to do it this way but apparently there's no other way in the GitPython API
+ topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
+ if not topcommit:
+ # This will error out if the directory is completely invalid or had never existed at this point
+ # If it previously existed but has since been deleted, you will get the revision where it was
+ # deleted - so we need to handle that case separately later
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ elif layerbranch.vcs_subdir:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ else:
+ logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
+ transaction.rollback()
+ sys.exit(1)
+
+ layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+ layerdir_start = os.path.normpath(layerdir) + os.sep
+ layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
+ layermachines = Machine.objects.filter(layerbranch=layerbranch)
+ layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
+ layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
+ if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
+ # Check out appropriate branch
+ if not options.nocheckout:
+ out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
+ out = utils.runcmd("git clean -f -x", repodir, logger=logger)
+
+ if layerbranch.vcs_subdir and not os.path.exists(layerdir):
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ else:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ transaction.rollback()
+ sys.exit(1)
+
+ if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
+ logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
+ transaction.rollback()
+ sys.exit(1)
+
+ logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+
+ try:
+ config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
+ except recipeparse.RecipeParseError as e:
+ logger.error(str(e))
+ transaction.rollback()
+ sys.exit(1)
+
+ if layerbranch.vcs_last_rev and not options.reload:
+ try:
+ diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
+ except Exception as e:
+ logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
+ diff = None
+ else:
+ diff = None
+
+ # We handle recipes specially to try to preserve the same id
+ # when recipe upgrades happen (so that if a user bookmarks a
+ # recipe page it remains valid)
+ layerrecipes_delete = []
+ layerrecipes_add = []
+
+ # Check if any paths should be ignored because there are layers within this layer
+ removedirs = []
+ for root, dirs, files in os.walk(layerdir):
+ for d in dirs:
+ if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
+ removedirs.append(os.path.join(root, d) + os.sep)
+
+ if diff:
+ # Apply git changes to existing recipe list
+
+ if layerbranch.vcs_subdir:
+ subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+ else:
+ subdir_start = ""
+
+ updatedrecipes = set()
+ for d in diff.iter_change_type('D'):
+ path = d.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for d in removedirs:
+ if path.startswith(d):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
+ if len(values):
+ layerrecipes_delete.append(values[0])
+ logger.debug("Mark %s for deletion" % values[0])
+ updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+ else:
+ logger.warn("Deleted recipe %s could not be found" % path)
+ elif typename == 'bbappend':
+ layerappends.filter(filepath=filepath).filter(filename=filename).delete()
+ elif typename == 'machine':
+ layermachines.filter(name=filename).delete()
+ elif typename == 'bbclass':
+ layerclasses.filter(name=filename).delete()
+
+ for d in diff.iter_change_type('A'):
+ path = d.b_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for d in removedirs:
+ if path.startswith(d):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ layerrecipes_add.append(os.path.join(repodir, path))
+ logger.debug("Mark %s for addition" % path)
+ updatedrecipes.add(os.path.join(filepath, filename))
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = filename
+ append.filepath = filepath
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ dirtyrecipes = set()
+ for d in diff.iter_change_type('M'):
+ path = d.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for d in removedirs:
+ if path.startswith(d):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ logger.debug("Mark %s for update" % path)
+ results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
+ if results:
+ recipe = results[0]
+ update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
+ recipe.save()
+ updatedrecipes.add(recipe.full_path())
+ elif typename == 'machine':
+ results = layermachines.filter(name=filename)
+ if results:
+ machine = results[0]
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+ for recipe in dirtyrecipes:
+ if not recipe.full_path() in updatedrecipes:
+ update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+ else:
+ # Collect recipe data from scratch
+
+ layerrecipe_fns = []
+ if options.fullreload:
+ layerrecipes.delete()
+ else:
+ # First, check which recipes still exist
+ layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
+ for v in layerrecipe_values:
+ root = os.path.join(layerdir, v['filepath'])
+ fullpath = os.path.join(root, v['filename'])
+ preserve = True
+ if os.path.exists(fullpath):
+ for d in removedirs:
+ if fullpath.startswith(d):
+ preserve = False
+ break
+ else:
+ preserve = False
+
+ if preserve:
+ # Recipe still exists, update it
+ results = layerrecipes.filter(id=v['id'])[:1]
+ recipe = results[0]
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ else:
+ # Recipe no longer exists, mark it for later on
+ layerrecipes_delete.append(v)
+ layerrecipe_fns.append(fullpath)
+
+ layermachines.delete()
+ layerappends.delete()
+ layerclasses.delete()
+ for root, dirs, files in os.walk(layerdir):
+ if '.git' in dirs:
+ dirs.remove('.git')
+ for d in dirs[:]:
+ fullpath = os.path.join(root, d) + os.sep
+ if fullpath in removedirs:
+ dirs.remove(d)
+ for f in files:
+ fullpath = os.path.join(root, f)
+ (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
+ if typename == 'recipe':
+ if fullpath not in layerrecipe_fns:
+ layerrecipes_add.append(fullpath)
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = f
+ append.filepath = os.path.relpath(root, layerdir)
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(fullpath, machine)
+ machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ for added in layerrecipes_add:
+ # This is good enough without actually parsing the file
+ (pn, pv) = split_recipe_fn(added)
+ oldid = -1
+ for deleted in layerrecipes_delete:
+ if deleted['pn'] == pn:
+ oldid = deleted['id']
+ layerrecipes_delete.remove(deleted)
+ break
+ if oldid > -1:
+ # Reclaim a record we would have deleted
+ results = Recipe.objects.filter(id=oldid)[:1]
+ recipe = results[0]
+ logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
+ else:
+ # Create new record
+ logger.debug("Add new recipe %s" % added)
+ recipe = Recipe()
+ recipe.layerbranch = layerbranch
+ recipe.filename = os.path.basename(added)
+ root = os.path.dirname(added)
+ recipe.filepath = os.path.relpath(root, layerdir)
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ recipe.save()
+
+ for deleted in layerrecipes_delete:
+ logger.debug("Delete %s" % deleted)
+ results = Recipe.objects.filter(id=deleted['id'])[:1]
+ recipe = results[0]
+ recipe.delete()
+
+ # Save repo info
+ layerbranch.vcs_last_rev = topcommit.hexsha
+ layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
+ else:
+ logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
+
+ layerbranch.vcs_last_fetch = datetime.now()
+ layerbranch.save()
+
+ if options.dryrun:
+ transaction.rollback()
+ else:
+ transaction.commit()
+
+ except KeyboardInterrupt:
+ transaction.rollback()
+ logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
+ sys.exit(254)
+ except:
+ import traceback
+ traceback.print_exc()
+ transaction.rollback()
+ finally:
+ transaction.leave_transaction_management()
+
+ shutil.rmtree(tempdir)
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ main()
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 03/15] update.py: allow updating all branches with one command
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 01/15] Allow blanking out field values in bulk change Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 02/15] update.py: refactor into two separate scripts Paul Eggleton
@ 2016-06-08 13:19 ` Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 04/15] Fix listing *_git.bbappend as appends for git recipe Paul Eggleton
` (11 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:19 UTC (permalink / raw)
To: yocto
Allow updating multiple branches, and if no branches are specified,
update all branches that have a new "updates_enabled" flag field set to
True. This avoids the need to have a separate shell script which runs
update.py for each branch (and thus has hardcoded knowledge of each
active branch in the index, i.e. it needs to be kept up-to-date in
addition to the database.)
The migration will default updates_enabled to True for all branches so
if you wish to take advantage of this functionality, the flag will need
to be set to False for any branches that shouldn't be updated.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
README | 7 +-
.../0011_auto__add_field_branch_updates_enabled.py | 198 +++++++++++++++++++++
layerindex/models.py | 1 +
layerindex/update.py | 70 ++++----
4 files changed, 239 insertions(+), 37 deletions(-)
create mode 100644 layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py
diff --git a/README b/README
index 84edee2..ff0a7a5 100644
--- a/README
+++ b/README
@@ -119,11 +119,8 @@ On a regular basis you need to run the update script:
path/to/layerindex/update.py
This will fetch all of the layer repositories, analyse their contents
-and update the database with the results. Note that if you set up more
-than just the master branch in the database, you will need to run the
-script once for each branch using -b (or --branch) to specify the
-branch name. Run the script with --help for further information on
-available options.
+and update the database with the results. Run the script with --help for
+further information on available options.
Maintenance
diff --git a/layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py b/layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py
new file mode 100644
index 0000000..5e2d4bf
--- /dev/null
+++ b/layerindex/migrations/0011_auto__add_field_branch_updates_enabled.py
@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding field 'Branch.updates_enabled'
+ db.add_column('layerindex_branch', 'updates_enabled',
+ self.gf('django.db.models.fields.BooleanField')(default=True),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ # Deleting field 'Branch.updates_enabled'
+ db.delete_column('layerindex_branch', 'updates_enabled')
+
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ 'auth.permission': {
+ 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.bbappend': {
+ 'Meta': {'object_name': 'BBAppend'},
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.bbclass': {
+ 'Meta': {'object_name': 'BBClass'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.branch': {
+ 'Meta': {'object_name': 'Branch'},
+ 'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+ 'sort_priority': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'auto_now': 'True', 'blank': 'True'}),
+ 'updates_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+ },
+ 'layerindex.classicrecipe': {
+ 'Meta': {'object_name': 'ClassicRecipe', '_ormbases': ['layerindex.Recipe']},
+ 'classic_category': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'cover_layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
+ 'cover_pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_status': ('django.db.models.fields.CharField', [], {'default': "'U'", 'max_length': '1'}),
+ 'cover_verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'recipe_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['layerindex.Recipe']", 'unique': 'True', 'primary_key': 'True'})
+ },
+ 'layerindex.layerbranch': {
+ 'Meta': {'object_name': 'LayerBranch'},
+ 'actual_branch': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_rev': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'vcs_subdir': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'})
+ },
+ 'layerindex.layerdependency': {
+ 'Meta': {'object_name': 'LayerDependency'},
+ 'dependency': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependents_set'", 'to': "orm['layerindex.LayerItem']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependencies_set'", 'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.layeritem': {
+ 'Meta': {'object_name': 'LayerItem'},
+ 'classic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'description': ('django.db.models.fields.TextField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+ 'mailing_list_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'usage_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+ },
+ 'layerindex.layermaintainer': {
+ 'Meta': {'object_name': 'LayerMaintainer'},
+ 'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'responsibility': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
+ },
+ 'layerindex.layernote': {
+ 'Meta': {'object_name': 'LayerNote'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'text': ('django.db.models.fields.TextField', [], {})
+ },
+ 'layerindex.machine': {
+ 'Meta': {'object_name': 'Machine'},
+ 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipe': {
+ 'Meta': {'object_name': 'Recipe'},
+ 'bbclassextend': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'blacklisted': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'inherits': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'provides': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'pv': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipechange': {
+ 'Meta': {'object_name': 'RecipeChange'},
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'changeset': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.RecipeChangeset']"}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.Recipe']"}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
+ },
+ 'layerindex.recipechangeset': {
+ 'Meta': {'object_name': 'RecipeChangeset'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+ },
+ 'layerindex.recipefiledependency': {
+ 'Meta': {'object_name': 'RecipeFileDependency'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.LayerBranch']"}),
+ 'path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
+ }
+ }
+
+ complete_apps = ['layerindex']
\ No newline at end of file
diff --git a/layerindex/models.py b/layerindex/models.py
index e0d85ea..6c5f65f 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -19,6 +19,7 @@ class Branch(models.Model):
bitbake_branch = models.CharField(max_length=50)
short_description = models.CharField(max_length=50, blank=True)
sort_priority = models.IntegerField(blank=True, null=True)
+ updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
updated = models.DateTimeField(auto_now = True, default = datetime.now)
diff --git a/layerindex/update.py b/layerindex/update.py
index f7cb25c..ee4138d 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -57,8 +57,8 @@ def main():
%prog [options]""")
parser.add_option("-b", "--branch",
- help = "Specify branch to update",
- action="store", dest="branch", default='master')
+ help = "Specify branch(es) to update (use commas to separate multiple). Default is all enabled branches.",
+ action="store", dest="branch", default='')
parser.add_option("-l", "--layer",
help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
action="store", dest="layers")
@@ -92,14 +92,19 @@ def main():
utils.setup_django()
import settings
- from layerindex.models import LayerItem
+ from layerindex.models import Branch, LayerItem
logger.setLevel(options.loglevel)
- branch = utils.get_branch(options.branch)
- if not branch:
- logger.error("Specified branch %s is not valid" % options.branch)
- sys.exit(1)
+ if options.branch:
+ branches = options.branch.split(',')
+ for branch in branches:
+ if not utils.get_branch(branch):
+ logger.error("Specified branch %s is not valid" % branch)
+ sys.exit(1)
+ else:
+ branchquery = Branch.objects.filter(updates_enabled=True)
+ branches = [branch.name for branch in branchquery]
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
@@ -164,31 +169,32 @@ def main():
# We now do this by calling out to a separate script; doing otherwise turned out to be
# unreliable due to leaking memory (we're using bitbake internals in a manner in which
# they never get used during normal operation).
- for layer in layerquery:
- if layer.vcs_url in failedrepos:
- logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
-
- urldir = layer.get_fetch_dir()
- repodir = os.path.join(fetchdir, urldir)
-
- cmd = 'python update_layer.py -l %s -b %s' % (layer.name, options.branch)
- if options.reload:
- cmd += ' --reload'
- if options.fullreload:
- cmd += ' --fullreload'
- if options.nocheckout:
- cmd += ' --nocheckout'
- if options.dryrun:
- cmd += ' -n'
- if options.loglevel == logging.DEBUG:
- cmd += ' -d'
- elif options.loglevel == logging.ERROR:
- cmd += ' -q'
- logger.debug('Running layer update command: %s' % cmd)
- ret = run_command_interruptible(cmd)
- if ret == 254:
- # Interrupted by user, break out of loop
- break
+ for branch in branches:
+ for layer in layerquery:
+ if layer.vcs_url in failedrepos:
+ logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
+
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
+
+ cmd = 'python update_layer.py -l %s -b %s' % (layer.name, branch)
+ if options.reload:
+ cmd += ' --reload'
+ if options.fullreload:
+ cmd += ' --fullreload'
+ if options.nocheckout:
+ cmd += ' --nocheckout'
+ if options.dryrun:
+ cmd += ' -n'
+ if options.loglevel == logging.DEBUG:
+ cmd += ' -d'
+ elif options.loglevel == logging.ERROR:
+ cmd += ' -q'
+ logger.debug('Running layer update command: %s' % cmd)
+ ret = run_command_interruptible(cmd)
+ if ret == 254:
+ # Interrupted by user, break out of loop
+ break
finally:
utils.unlock_file(lockfile)
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 04/15] Fix listing *_git.bbappend as appends for git recipe
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (2 preceding siblings ...)
2016-06-08 13:19 ` [layerindex-web][PATCH v2 03/15] update.py: allow updating all branches with one command Paul Eggleton
@ 2016-06-08 13:19 ` Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 05/15] Increase size of Recipe provides and license fields Paul Eggleton
` (10 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:19 UTC (permalink / raw)
To: yocto
Fix a greedy regex in the recipe detail view resulting in the git recipe
listing all bbappends named *_git.bbappend as its bbappends (quite a few
in the public instance).
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
TODO | 1 -
layerindex/views.py | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/TODO b/TODO
index 2d5a53d..9ab7aa7 100644
--- a/TODO
+++ b/TODO
@@ -10,7 +10,6 @@ TODO:
* Make it easy to update people's email addresses
Bugs
-* git recipe shows up all _git bbappends: http://layers.openembedded.org/layerindex/recipe/5542/
* Duplication of first maintainer when editing to add a second?
* PROVIDES column appears to be too short for some recipes
diff --git a/layerindex/views.py b/layerindex/views.py
index d033046..d9480c9 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -672,7 +672,7 @@ class RecipeDetailView(DetailView):
verappendprefix = recipe.filename.split('.bb')[0]
appendprefix = verappendprefix.split('_')[0]
#context['verappends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename='%s.bbappend' % verappendprefix)
- context['appends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename__regex=r'%s(_[^_]*)?\.bbappend' % appendprefix)
+ context['appends'] = BBAppend.objects.filter(layerbranch__branch=recipe.layerbranch.branch).filter(filename__regex=r'^%s(_[^_]*)?\.bbappend' % appendprefix)
verappends = []
for append in context['appends']:
if append.matches_recipe(recipe):
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 05/15] Increase size of Recipe provides and license fields
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (3 preceding siblings ...)
2016-06-08 13:19 ` [layerindex-web][PATCH v2 04/15] Fix listing *_git.bbappend as appends for git recipe Paul Eggleton
@ 2016-06-08 13:19 ` Paul Eggleton
2016-06-08 13:19 ` [layerindex-web][PATCH v2 06/15] bulkchange: drop temp paths in multi-patch tarball Paul Eggleton
` (9 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:19 UTC (permalink / raw)
To: yocto
Fix "data truncated" warnings/errors when loading data for OE-Core:
* PROVIDES for recipe packagegroup-base is ~1452 characters
* LICENSE for recipe linux-firmware is ~1053 characters
(These aren't shown with SQLite, only with something like MariaDB where
column sizes are enforced.)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
TODO | 1 -
...ld_recipe_license__chg_field_recipe_provides.py | 202 +++++++++++++++++++++
layerindex/models.py | 4 +-
3 files changed, 204 insertions(+), 3 deletions(-)
create mode 100644 layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py
diff --git a/TODO b/TODO
index 9ab7aa7..b5e8974 100644
--- a/TODO
+++ b/TODO
@@ -11,7 +11,6 @@ TODO:
Bugs
* Duplication of first maintainer when editing to add a second?
-* PROVIDES column appears to be too short for some recipes
Other
* Full-text search on layer contents
diff --git a/layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py b/layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py
new file mode 100644
index 0000000..52cc264
--- /dev/null
+++ b/layerindex/migrations/0012_auto__chg_field_recipe_license__chg_field_recipe_provides.py
@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Changing field 'Recipe.license'
+ db.alter_column('layerindex_recipe', 'license', self.gf('django.db.models.fields.CharField')(max_length=2048))
+
+ # Changing field 'Recipe.provides'
+ db.alter_column('layerindex_recipe', 'provides', self.gf('django.db.models.fields.CharField')(max_length=2048))
+
+ def backwards(self, orm):
+
+ # Changing field 'Recipe.license'
+ db.alter_column('layerindex_recipe', 'license', self.gf('django.db.models.fields.CharField')(max_length=100))
+
+ # Changing field 'Recipe.provides'
+ db.alter_column('layerindex_recipe', 'provides', self.gf('django.db.models.fields.CharField')(max_length=255))
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+ },
+ 'auth.permission': {
+ 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.bbappend': {
+ 'Meta': {'object_name': 'BBAppend'},
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.bbclass': {
+ 'Meta': {'object_name': 'BBClass'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.branch': {
+ 'Meta': {'object_name': 'Branch'},
+ 'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+ 'sort_priority': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'auto_now': 'True', 'blank': 'True'}),
+ 'updates_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+ },
+ 'layerindex.classicrecipe': {
+ 'Meta': {'object_name': 'ClassicRecipe', '_ormbases': ['layerindex.Recipe']},
+ 'classic_category': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'cover_layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}),
+ 'cover_pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'cover_status': ('django.db.models.fields.CharField', [], {'default': "'U'", 'max_length': '1'}),
+ 'cover_verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'recipe_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['layerindex.Recipe']", 'unique': 'True', 'primary_key': 'True'})
+ },
+ 'layerindex.layerbranch': {
+ 'Meta': {'object_name': 'LayerBranch'},
+ 'actual_branch': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+ 'vcs_last_rev': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+ 'vcs_subdir': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'})
+ },
+ 'layerindex.layerdependency': {
+ 'Meta': {'object_name': 'LayerDependency'},
+ 'dependency': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependents_set'", 'to': "orm['layerindex.LayerItem']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependencies_set'", 'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.layeritem': {
+ 'Meta': {'object_name': 'LayerItem'},
+ 'classic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'description': ('django.db.models.fields.TextField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+ 'mailing_list_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+ 'usage_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'vcs_web_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+ },
+ 'layerindex.layermaintainer': {
+ 'Meta': {'object_name': 'LayerMaintainer'},
+ 'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'responsibility': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
+ },
+ 'layerindex.layernote': {
+ 'Meta': {'object_name': 'LayerNote'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'text': ('django.db.models.fields.TextField', [], {})
+ },
+ 'layerindex.machine': {
+ 'Meta': {'object_name': 'Machine'},
+ 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipe': {
+ 'Meta': {'object_name': 'Recipe'},
+ 'bbclassextend': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'blacklisted': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'inherits': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}),
+ 'pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'provides': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'blank': 'True'}),
+ 'pv': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+ },
+ 'layerindex.recipechange': {
+ 'Meta': {'object_name': 'RecipeChange'},
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'changeset': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.RecipeChangeset']"}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.Recipe']"}),
+ 'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
+ },
+ 'layerindex.recipechangeset': {
+ 'Meta': {'object_name': 'RecipeChangeset'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+ },
+ 'layerindex.recipefiledependency': {
+ 'Meta': {'object_name': 'RecipeFileDependency'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.LayerBranch']"}),
+ 'path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
+ }
+ }
+
+ complete_apps = ['layerindex']
\ No newline at end of file
diff --git a/layerindex/models.py b/layerindex/models.py
index 6c5f65f..8d82846 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -238,10 +238,10 @@ class Recipe(models.Model):
summary = models.CharField(max_length=200, blank=True)
description = models.TextField(blank=True)
section = models.CharField(max_length=100, blank=True)
- license = models.CharField(max_length=100, blank=True)
+ license = models.CharField(max_length=2048, blank=True)
homepage = models.URLField(blank=True)
bugtracker = models.URLField(blank=True)
- provides = models.CharField(max_length=255, blank=True)
+ provides = models.CharField(max_length=2048, blank=True)
bbclassextend = models.CharField(max_length=100, blank=True)
inherits = models.CharField(max_length=255, blank=True)
updated = models.DateTimeField(auto_now = True)
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 06/15] bulkchange: drop temp paths in multi-patch tarball
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (4 preceding siblings ...)
2016-06-08 13:19 ` [layerindex-web][PATCH v2 05/15] Increase size of Recipe provides and license fields Paul Eggleton
@ 2016-06-08 13:19 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 07/15] Upgrade to Django 1.6+ Paul Eggleton
` (8 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:19 UTC (permalink / raw)
To: yocto
We were getting temporary paths appearing in the tarball that gets
created to allow you to download the patches for a bulk change operation
that crosses multiple layers - we don't need those.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/bulkchange.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/layerindex/bulkchange.py b/layerindex/bulkchange.py
index c569b61..08d6d78 100644
--- a/layerindex/bulkchange.py
+++ b/layerindex/bulkchange.py
@@ -70,7 +70,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
tar = tarfile.open(None, "w:gz", tmptarfile)
for patch in patches:
patchfn = os.path.join(tmpoutdir, patch)
- tar.add(patchfn)
+ tar.add(patchfn, arcname=patch)
tar.close()
ret = tmptarname
elif len(patches) == 1:
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 07/15] Upgrade to Django 1.6+
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (5 preceding siblings ...)
2016-06-08 13:19 ` [layerindex-web][PATCH v2 06/15] bulkchange: drop temp paths in multi-patch tarball Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 08/15] Fix for changes in modern django-reversion Paul Eggleton
` (7 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
I'd like to be upgrading to 1.8 but that causes problems with South, and
we're not quite ready to dispense with our existing migrations yet.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
README | 18 +++++-----
layerindex/forms.py | 13 ++++----
layerindex/restviews.py | 4 +--
layerindex/urls.py | 16 ++++-----
layerindex/urls_branch.py | 3 +-
layerindex/utils.py | 6 ----
manage.py | 20 ++++--------
requirements.txt | 38 +++++++++++-----------
settings.py | 2 +-
templates/404.html | 2 +-
templates/base.html | 22 ++++++-------
templates/base_toplevel.html | 3 +-
templates/layerindex/about.html | 2 +-
templates/layerindex/bulkchange.html | 2 +-
templates/layerindex/bulkchangereview.html | 8 ++---
templates/layerindex/bulkchangesearch.html | 8 ++---
templates/layerindex/classic_base.html | 10 +++---
templates/layerindex/classicrecipedetail.html | 2 +-
templates/layerindex/classicrecipes.html | 10 +++---
templates/layerindex/classicstats.html | 4 +--
templates/layerindex/detail.html | 14 ++++----
templates/layerindex/duplicates.html | 1 -
templates/layerindex/editlayernote.html | 2 +-
templates/layerindex/layers.html | 8 ++---
templates/layerindex/machines.html | 10 +++---
templates/layerindex/profile.html | 2 +-
templates/layerindex/recipedetail.html | 12 +++----
templates/layerindex/recipes.html | 12 +++----
templates/layerindex/reviewdetail.html | 14 ++++----
templates/layerindex/reviewlist.html | 2 +-
templates/registration/activate.html | 2 +-
templates/registration/activation_email.txt | 2 +-
templates/registration/login.html | 4 +--
.../registration/password_reset_complete.html | 2 +-
templates/registration/password_reset_email.html | 2 +-
urls.py | 6 ++--
36 files changed, 136 insertions(+), 152 deletions(-)
diff --git a/README b/README
index ff0a7a5..db788df 100644
--- a/README
+++ b/README
@@ -11,22 +11,22 @@ Setup
In order to make use of this application you will need:
-* Django 1.4.x - tested with 1.4.1-1.4.10; newer versions may work, but
- the application has not been tested with 1.5 or newer.
+* Django 1.6.x - tested with 1.6.10; newer versions may work, but
+ the application has not been tested with 1.7 or newer.
* For production usage, a web server set up to host Django applications
(not needed for local-only testing)
* A database supported by Django (SQLite, MySQL, etc.). Django takes
care of creating the database itself, you just need to ensure that the
database server (if not using SQLite) is configured and running.
* The following third-party Django modules (tested versions listed):
- * django-south (0.8.4)
+ * django-south (1.0.2)
* django-registration (1.0)
- * django-reversion (1.7.1)
- * django-reversion-compare (0.3.5)
- * django-simple-captcha (0.4.1)
- * django-nvd3 (0.6.0)
- * djangorestframework (2.3.14)
- * django-cors-headers (0.12)
+ * django-reversion (1.8.7)
+ * django-reversion-compare (0.4.0)
+ * django-simple-captcha (0.4.6)
+ * django-nvd3 (0.9.7)
+ * djangorestframework (3.2.5)
+ * django-cors-headers (1.1.0)
* On the machine that will run the backend update script (which does not
have to be the same machine as the web server, however it does still
have to have Django installed, have the same or similar configuration
diff --git a/layerindex/forms.py b/layerindex/forms.py
index 60653cf..e15dbeb 100644
--- a/layerindex/forms.py
+++ b/layerindex/forms.py
@@ -6,7 +6,7 @@
from layerindex.models import LayerItem, LayerBranch, LayerMaintainer, LayerNote, RecipeChangeset, RecipeChange, ClassicRecipe
from django import forms
-from django.core.validators import URLValidator, RegexValidator, email_re
+from django.core.validators import URLValidator, RegexValidator, EmailValidator
from django.forms.models import inlineformset_factory, modelformset_factory
from captcha.fields import CaptchaField
from django.contrib.auth.models import User
@@ -29,9 +29,8 @@ class LayerMaintainerForm(forms.ModelForm):
if email:
if len(email) < 7:
raise forms.ValidationError('%s is not a valid email address' % email)
- reg = re.compile(email_re)
- if not reg.match(email):
- raise forms.ValidationError('%s is not a valid email address' % email)
+ val = EmailValidator()
+ val(email)
return email
@@ -115,21 +114,21 @@ class EditLayerForm(forms.ModelForm):
def clean_vcs_web_tree_base_url(self):
url = self.cleaned_data['vcs_web_tree_base_url'].strip()
if url:
- val = URLValidator(verify_exists=False)
+ val = URLValidator()
val(url)
return url
def clean_vcs_web_file_base_url(self):
url = self.cleaned_data['vcs_web_file_base_url'].strip()
if url:
- val = URLValidator(verify_exists=False)
+ val = URLValidator()
val(url)
return url
def clean_usage_url(self):
usage = self.cleaned_data['usage_url'].strip()
if usage.startswith('http'):
- val = URLValidator(verify_exists=False)
+ val = URLValidator()
val(usage)
return usage
diff --git a/layerindex/restviews.py b/layerindex/restviews.py
index 61698a9..b33d3d1 100644
--- a/layerindex/restviews.py
+++ b/layerindex/restviews.py
@@ -1,6 +1,6 @@
-from models import Branch, LayerItem, LayerNote, LayerBranch, LayerDependency, Recipe, Machine
+from layerindex.models import Branch, LayerItem, LayerNote, LayerBranch, LayerDependency, Recipe, Machine
from rest_framework import viewsets, serializers
-from querysethelper import params_to_queryset, get_search_tuple
+from layerindex.querysethelper import params_to_queryset, get_search_tuple
class ParametricSearchableModelViewSet(viewsets.ModelViewSet):
def get_queryset(self):
diff --git a/layerindex/urls.py b/layerindex/urls.py
index 1bd4f0b..e7808e6 100644
--- a/layerindex/urls.py
+++ b/layerindex/urls.py
@@ -4,9 +4,8 @@
#
# Licensed under the MIT license, see COPYING.MIT for details
-from django.conf.urls.defaults import *
-from django.views.generic import TemplateView, DetailView, ListView
-from django.views.generic.simple import redirect_to
+from django.conf.urls import *
+from django.views.generic import TemplateView, DetailView, ListView, RedirectView
from django.views.defaults import page_not_found
from django.core.urlresolvers import reverse_lazy
from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView
@@ -24,19 +23,20 @@ router.register(r'recipes', restviews.RecipeViewSet)
router.register(r'machines', restviews.MachineViewSet)
urlpatterns = patterns('',
- url(r'^$', redirect_to, {'url' : reverse_lazy('layer_list', args=('master',))},
+ url(r'^$',
+ RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',))),
name='frontpage'),
url(r'^api/', include(router.urls)),
url(r'^layers/$',
- redirect_to, {'url' : reverse_lazy('layer_list', args=('master',))}),
+ RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)))),
url(r'^layer/(?P<slug>[-\w]+)/$',
RedirectParamsView.as_view(), {'redirect_name': 'layer_item', 'branch':'master'}),
url(r'^recipes/$',
- redirect_to, {'url' : reverse_lazy('recipe_search', args=('master',))}),
+ RedirectView.as_view(url=reverse_lazy('recipe_search', args=('master',)))),
url(r'^machines/$',
- redirect_to, {'url' : reverse_lazy('machine_search', args=('master',))}),
+ RedirectView.as_view(url=reverse_lazy('machine_search', args=('master',)))),
url(r'^submit/$', edit_layer_view, {'template_name': 'layerindex/submitlayer.html'}, name="submit_layer"),
url(r'^submit/thanks$',
@@ -107,7 +107,7 @@ urlpatterns = patterns('',
template_name='layerindex/about.html'),
name="about"),
url(r'^oe-classic/$',
- redirect_to, {'url' : reverse_lazy('classic_recipe_search')},
+ RedirectView.as_view(url=reverse_lazy('classic_recipe_search')),
name='classic'),
url(r'^oe-classic/recipes/$',
ClassicRecipeSearchView.as_view(
diff --git a/layerindex/urls_branch.py b/layerindex/urls_branch.py
index b0f577c..ab5e2d5 100644
--- a/layerindex/urls_branch.py
+++ b/layerindex/urls_branch.py
@@ -4,8 +4,7 @@
#
# Licensed under the MIT license, see COPYING.MIT for details
-from django.conf.urls.defaults import *
-from django.views.generic.simple import redirect_to
+from django.conf.urls import *
from django.views.defaults import page_not_found
from django.core.urlresolvers import reverse_lazy
from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView
diff --git a/layerindex/utils.py b/layerindex/utils.py
index 440cc88..ebb89d3 100644
--- a/layerindex/utils.py
+++ b/layerindex/utils.py
@@ -57,12 +57,6 @@ def setup_django():
sys.path.append(newpath)
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
- from django.core.management import setup_environ
- from django.conf import settings
- import settings
-
- setup_environ(settings)
-
def logger_create(name):
logger = logging.getLogger(name)
loggerhandler = logging.StreamHandler()
diff --git a/manage.py b/manage.py
index 0b219fb..0a64562 100644
--- a/manage.py
+++ b/manage.py
@@ -6,20 +6,14 @@
#
# Copyright (c) Django Software Foundation and individual contributors.
# All rights reserved.
-
+#!/usr/bin/env python
import os
-
-from django.core.management import execute_manager
-import imp
-try:
- imp.find_module('settings') # Assumed to be in the same directory.
-except ImportError:
- import sys
- sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__)
- sys.exit(1)
-
-import settings
+import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
- execute_manager(settings)
+
+ from django.core.management import execute_from_command_line
+
+ execute_from_command_line(sys.argv)
+
diff --git a/requirements.txt b/requirements.txt
index cde88a0..ea5beee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,20 @@
-Django>=1.4,<1.5
-GitPython>=0.3.7
-Jinja2==2.7.3
+Django==1.6.11
+django-cors-headers==1.1.0
+django-nvd3==0.9.7
+django-registration==1.0
+django-reversion==1.8.7
+django-reversion-compare==0.4.0
+django-simple-captcha==0.4.6
+djangorestframework==3.2.5
+gitdb==0.6.4
+GitPython==2.0.5
+Jinja2==2.8
MarkupSafe==0.23
-Pillow>=2.4.0
-South==0.8.4
-Unidecode==0.04.16
-argparse==1.2.1
-awesome-slugify==1.5
-django-cors-headers==0.12
-django-nvd3==0.7.4
-django-registration==0.8
-django-reversion==1.6.6
-django-reversion-compare==0.3.5
-django-simple-captcha==0.4.2
-djangorestframework==2.3.14
-python-nvd3==0.12.2
-regex==2014.06.28
-six==1.7.3
-wsgiref==0.1.2
+mysqlclient==1.3.7
+Pillow==3.2.0
+python-nvd3==0.14.2
+python-slugify==1.1.4
+six==1.10.0
+smmap==0.9.0
+South==1.0.2
+Unidecode==0.4.19
diff --git a/settings.py b/settings.py
index b21a5b4..b731a6b 100644
--- a/settings.py
+++ b/settings.py
@@ -131,7 +131,7 @@ TEMPLATE_DIRS = (
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
- BASE_DIR + "/templates"
+ BASE_DIR + "/templates",
)
INSTALLED_APPS = (
diff --git a/templates/404.html b/templates/404.html
index 843d5bc..5bd5f75 100644
--- a/templates/404.html
+++ b/templates/404.html
@@ -22,7 +22,7 @@
<p>The page you requested was not found.</p>
-<p><a href="{% url frontpage %}">Return to the front page</a></p>
+<p><a href="{% url 'frontpage' %}">Return to the front page</a></p>
{% endautoescape %}
{% endblock %}
diff --git a/templates/base.html b/templates/base.html
index c872383..fedcfe2 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -28,13 +28,13 @@
<div class="navbar navbar-fixed-top">
<div class="navbar-inner">
<div class="container-fluid">
- <a class="brand" href="{% url frontpage %}">{{ site_name }}</a>
+ <a class="brand" href="{% url 'frontpage' %}">{{ site_name }}</a>
{% if user.is_authenticated %}
<div class="btn-group pull-right">
{% if perms.layerindex.publish_layer %}
{% if unpublished_count > 0 %}
- <a class="btn" href="{% url layer_list_review %}?branch=master">
+ <a class="btn" href="{% url 'layer_list_review' %}?branch=master">
<span class="badge badge-warning review-notification">{{ unpublished_count }}</span>
</a>
{% endif %}
@@ -45,14 +45,14 @@
<b class="caret"></b>
</button>
<ul class="dropdown-menu">
- <li><a href="{% url auth_logout %}">{% trans "Log out" %}</a></li>
- <li><a href="{% url auth_password_change %}">{% trans "Change password" %}</a></li>
- <li><a href="{% url profile %}">{% trans "Edit profile" %}</a></li>
+ <li><a href="{% url 'auth_logout' %}">{% trans "Log out" %}</a></li>
+ <li><a href="{% url 'auth_password_change' %}">{% trans "Change password" %}</a></li>
+ <li><a href="{% url 'profile' %}">{% trans "Edit profile" %}</a></li>
</ul>
</div>
{% else %}
<div class="pull-right">
- <a class="btn" href="{% url auth_login %}">{% trans "Log in" %}</a>
+ <a class="btn" href="{% url 'auth_login' %}">{% trans "Log in" %}</a>
</div>
{% endif %}
<ul class="nav pull-right">
@@ -60,7 +60,7 @@
</ul>
{% block topfunctions %}
<div class="pull-right nav-spacer">
- <a class="btn btn-info" href="{% url submit_layer %}">Submit layer</a>
+ <a class="btn btn-info" href="{% url 'submit_layer' %}">Submit layer</a>
</div>
<ul class="nav pull-right">
{% if user.is_authenticated %}
@@ -70,8 +70,8 @@
<b class="caret"></b>
</a>
<ul class="dropdown-menu">
- <li><a href="{% url bulk_change %}">Bulk Change</a></li>
- <li><a href="{% url duplicates 'master' %}">Duplicates</a></li>
+ <li><a href="{% url 'bulk_change' %}">Bulk Change</a></li>
+ <li><a href="{% url 'duplicates' 'master' %}">Duplicates</a></li>
</ul>
</li>
{% endif %}
@@ -96,8 +96,8 @@
{% block footer %}
<hr />
<div class="footer">
- <a href="{% url history_list %}">change history</a>
- • <a href="{% url about %}">about this site</a>
+ <a href="{% url 'history_list' %}">change history</a>
+ • <a href="{% url 'about' %}">about this site</a>
• <a href="http://www.openembedded.org/Layers_FAQ">FAQ</a>
</div>
diff --git a/templates/base_toplevel.html b/templates/base_toplevel.html
index d9ea7bb..c86ccc2 100644
--- a/templates/base_toplevel.html
+++ b/templates/base_toplevel.html
@@ -1,7 +1,6 @@
{% extends "base.html" %}
{% load i18n %}
-{% load url from future %}
{% comment %}
layerindex-web - top level page template
@@ -56,4 +55,4 @@
</div>
</div>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/templates/layerindex/about.html b/templates/layerindex/about.html
index 67c646a..3e98825 100644
--- a/templates/layerindex/about.html
+++ b/templates/layerindex/about.html
@@ -11,7 +11,7 @@
<p>This website indexes layers for the <a href="http://www.openembedded.org">OpenEmbedded</a> build system provided by members of the OpenEmbedded / Yocto Project community, suitable for use on top of OpenEmbedded-Core and compatible systems, providing additional recipes, machine support and/or distro policy configuration.</p>
-<p>If you have a layer for use with OpenEmbedded that you wish to share with others, please <a href="{% url submit_layer %}">submit it</a>!</p>
+<p>If you have a layer for use with OpenEmbedded that you wish to share with others, please <a href="{% url 'submit_layer' %}">submit it</a>!</p>
<h3>Technologies</h3>
diff --git a/templates/layerindex/bulkchange.html b/templates/layerindex/bulkchange.html
index e7fc270..e8936de 100644
--- a/templates/layerindex/bulkchange.html
+++ b/templates/layerindex/bulkchange.html
@@ -29,7 +29,7 @@ generate a patch for these changes which can be submitted for merging.</p>
<h3>Select an existing changeset</h3>
<ul>
{% for changeset in changesets %}
- <li><a href="{% url bulk_change_search changeset.id %}">{{ changeset.name }}</a></li>
+ <li><a href="{% url 'bulk_change_search' changeset.id %}">{{ changeset.name }}</a></li>
{% endfor %}
</ul>
{% endif %}
diff --git a/templates/layerindex/bulkchangereview.html b/templates/layerindex/bulkchangereview.html
index 5fb971e..a95bb5d 100644
--- a/templates/layerindex/bulkchangereview.html
+++ b/templates/layerindex/bulkchangereview.html
@@ -37,10 +37,10 @@
</li>
{% endfor %}
</ul>
-<a href="{% url bulk_change_search changeset.id %}" class="btn">Add recipes</a>
-<a href="{% url bulk_change_edit changeset.id %}" class="btn">Edit</a>
-<a href="{% url bulk_change_patches changeset.id %}" class="btn">Get patches</a>
-<a href="{% url bulk_change_delete changeset.id %}?cancel=bulk_change_review" class="btn">Delete</a>
+<a href="{% url 'bulk_change_search' changeset.id %}" class="btn">Add recipes</a>
+<a href="{% url 'bulk_change_edit' changeset.id %}" class="btn">Edit</a>
+<a href="{% url 'bulk_change_patches' changeset.id %}" class="btn">Get patches</a>
+<a href="{% url 'bulk_change_delete' changeset.id %}?cancel=bulk_change_review" class="btn">Delete</a>
{% endautoescape %}
diff --git a/templates/layerindex/bulkchangesearch.html b/templates/layerindex/bulkchangesearch.html
index 134f9a3..0cb83ff 100644
--- a/templates/layerindex/bulkchangesearch.html
+++ b/templates/layerindex/bulkchangesearch.html
@@ -78,10 +78,10 @@
{% for recipe in recipe_list %}
<tr>
<td><input type="checkbox" name="selecteditems" value="{{ recipe.id }}"></input></td>
- <td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a></td>
+ <td><a href="{% url 'recipe' recipe.id %}">{{ recipe.name }}</a></td>
<td>{{ recipe.pv }}</td>
<td>{{ recipe.short_desc }}</td>
- <td><a href="{% url layer_item current_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
+ <td><a href="{% url 'layer_item' current_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
</tr>
{% endfor %}
</tbody>
@@ -120,9 +120,9 @@
</ul>
</small>
<input type="submit" class="btn" name="remove_all" value="Remove all"></input>
- <a href="{% url bulk_change_edit changeset.id %}" class="btn">Edit</a>
+ <a href="{% url 'bulk_change_edit' changeset.id %}" class="btn">Edit</a>
{% endif %}
- <a href="{% url bulk_change_delete changeset.id %}?cancel=bulk_change_search" class="btn">Delete</a>
+ <a href="{% url 'bulk_change_delete' changeset.id %}?cancel=bulk_change_search" class="btn">Delete</a>
</div>
{% endif %}
</form>
diff --git a/templates/layerindex/classic_base.html b/templates/layerindex/classic_base.html
index eab56f2..7ba63f5 100644
--- a/templates/layerindex/classic_base.html
+++ b/templates/layerindex/classic_base.html
@@ -24,7 +24,7 @@
</a>
<ul class="dropdown-menu">
{% for branch in all_branches %}
- <li><a href="{% url layer_list branch.name %}">
+ <li><a href="{% url 'layer_list' branch.name %}">
{{ branch.name }}
{% if branch.short_description %}
({{ branch.short_description }})
@@ -32,7 +32,7 @@
</a></li>
{% endfor %}
<li class="divider"></li>
- <li><a href="{% url classic %}"><b>OE-Classic</b></a></li>
+ <li><a href="{% url 'classic' %}"><b>OE-Classic</b></a></li>
</ul>
</li>
{% endautoescape %}
@@ -48,9 +48,9 @@
<p>OpenEmbedded-Classic (OE-Classic) is the name for the old monolithic version of OpenEmbedded. It contained a number of recipes some of which have not yet been migrated on top of OE-Core. To help people to find and migrate these recipes we provide an index here as well as some statistics to get an idea of the migration.</p>
- <a class="btn btn-large btn-primary" href="{% url classic_recipe_search %}">Recipes</a>
- <a class="btn btn-large" href="{% url classic_recipe_search %}?q=&cover_status=!">Unmigrated Recipes</a>
- <a class="btn btn-large btn-primary" href="{% url classic_recipe_stats %}">Stats</a>
+ <a class="btn btn-large btn-primary" href="{% url 'classic_recipe_search' %}">Recipes</a>
+ <a class="btn btn-large" href="{% url 'classic_recipe_search' %}?q=&cover_status=!">Unmigrated Recipes</a>
+ <a class="btn btn-large btn-primary" href="{% url 'classic_recipe_stats' %}">Stats</a>
</div>
</div>
diff --git a/templates/layerindex/classicrecipedetail.html b/templates/layerindex/classicrecipedetail.html
index 9df9a84..45d845f 100644
--- a/templates/layerindex/classicrecipedetail.html
+++ b/templates/layerindex/classicrecipedetail.html
@@ -21,7 +21,7 @@
{% autoescape on %}
<ul class="breadcrumb">
- <li><a href="{% url classic_recipe_search %}">OE-Classic</a> <span class="divider">→</span></li>
+ <li><a href="{% url 'classic_recipe_search' %}">OE-Classic</a> <span class="divider">→</span></li>
<li class="active">{{ recipe.name }}</li>
</ul>
diff --git a/templates/layerindex/classicrecipes.html b/templates/layerindex/classicrecipes.html
index 7fb62be..799dbb4 100644
--- a/templates/layerindex/classicrecipes.html
+++ b/templates/layerindex/classicrecipes.html
@@ -17,8 +17,8 @@
{% block navs %}
{% autoescape on %}
- <li class="active"><a href="{% url classic_recipe_search %}">Recipes</a></li>
- <li><a href="{% url classic_recipe_stats %}">Stats</a></li>
+ <li class="active"><a href="{% url 'classic_recipe_search' %}">Recipes</a></li>
+ <li><a href="{% url 'classic_recipe_stats' %}">Stats</a></li>
{% endautoescape %}
{% endblock %}
@@ -31,7 +31,7 @@
<h2>OE-Classic recipes</h2>
<div class="alert alert-warning">
- <b>NOTE:</b> This is the recipe search for OE-Classic, the older monolithic version of OpenEmbedded which is no longer actively developed. <a href="{% url recipe_search 'master' %}">Click here</a> to search current recipes.
+ <b>NOTE:</b> This is the recipe search for OE-Classic, the older monolithic version of OpenEmbedded which is no longer actively developed. <a href="{% url 'recipe_search' 'master' %}">Click here</a> to search current recipes.
</div>
<div class="row-fluid">
@@ -72,13 +72,13 @@
<tbody>
{% for recipe in recipe_list %}
<tr {% if recipe.preferred_count > 0 %}class="muted"{% endif %}>
- <td><a href="{% url classic_recipe recipe.id %}">{{ recipe.name }}</a></td>
+ <td><a href="{% url 'classic_recipe' recipe.id %}">{{ recipe.name }}</a></td>
<td>{{ recipe.pv }}</td>
<td>{{ recipe.short_desc }}</td>
<td>{{ recipe.get_cover_desc }}</td>
<td>{{ recipe.classic_category }}</td>
{% if multi_classic_layers %}
- <td><a href="{% url layer_item recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
+ <td><a href="{% url 'layer_item' recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
{% endif %}
</tr>
{% endfor %}
diff --git a/templates/layerindex/classicstats.html b/templates/layerindex/classicstats.html
index 2fde01b..63d0b43 100644
--- a/templates/layerindex/classicstats.html
+++ b/templates/layerindex/classicstats.html
@@ -19,8 +19,8 @@
{% block navs %}
{% autoescape on %}
- <li><a href="{% url classic_recipe_search %}">Recipes</a></li>
- <li class="active"><a href="{% url classic_recipe_stats %}">Stats</a></li>
+ <li><a href="{% url 'classic_recipe_search' %}">Recipes</a></li>
+ <li class="active"><a href="{% url 'classic_recipe_stats' %}">Stats</a></li>
{% endautoescape %}
{% endblock %}
diff --git a/templates/layerindex/detail.html b/templates/layerindex/detail.html
index feb2c6c..d0d11c0 100644
--- a/templates/layerindex/detail.html
+++ b/templates/layerindex/detail.html
@@ -22,7 +22,7 @@
{% autoescape on %}
<ul class="breadcrumb">
- <li><a href="{% url layer_list url_branch %}">{{ layerbranch.branch.name }}</a> <span class="divider">→</span></li>
+ <li><a href="{% url 'layer_list' url_branch %}">{{ layerbranch.branch.name }}</a> <span class="divider">→</span></li>
<li class="active">{{ layeritem.name }}</li>
</ul>
@@ -36,9 +36,9 @@
{% if user.is_authenticated %}
<span class="pull-right">
{% if perms.layerindex.publish_layer or useredit %}
- <a href="{% url edit_layer url_branch layeritem.name %}" class="btn">Edit layer</a>
+ <a href="{% url 'edit_layer' url_branch layeritem.name %}" class="btn">Edit layer</a>
{% if layeritem.layernote_set.count = 0 %}
- <a href="{% url add_layernote layeritem.name %}" class="btn">Add note</a>
+ <a href="{% url 'add_layernote' layeritem.name %}" class="btn">Add note</a>
{% endif %}
{% endif %}
</span>
@@ -63,8 +63,8 @@
{% if perms.layerindex.publish_layer or useredit %}
<br><br>
<p>
- <a href="{% url edit_layernote layeritem.name note.pk %}" class="btn">Edit note</a>
- <a href="{% url delete_layernote layeritem.name note.pk %}" class='btn'>Delete note</a>
+ <a href="{% url 'edit_layernote' layeritem.name note.pk %}" class="btn">Edit note</a>
+ <a href="{% url 'delete_layernote' layeritem.name note.pk %}" class='btn'>Delete note</a>
</p>
{% endif %}
</div>
@@ -144,7 +144,7 @@
<p>The {{ layeritem.name }} layer depends upon:</p>
<ul>
{% for dep in layerbranch.dependencies_set.all %}
- <li><a href="{% url layer_item url_branch dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
+ <li><a href="{% url 'layer_item' url_branch dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
{% endfor %}
</ul>
</div> <!-- end of well -->
@@ -198,7 +198,7 @@
<tbody>
{% for recipe in layerbranch.sorted_recipes %}
<tr>
- <td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
+ <td><a href="{% url 'recipe' recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
<td>{{ recipe.pv }}</td>
<td class="span8">{{ recipe.short_desc }}</td>
</tr>
diff --git a/templates/layerindex/duplicates.html b/templates/layerindex/duplicates.html
index 116aab9..0ebfe11 100644
--- a/templates/layerindex/duplicates.html
+++ b/templates/layerindex/duplicates.html
@@ -1,6 +1,5 @@
{% extends "base_toplevel.html" %}
{% load i18n %}
-{% load url from future %}
{% comment %}
diff --git a/templates/layerindex/editlayernote.html b/templates/layerindex/editlayernote.html
index 1c7693c..abf8299 100644
--- a/templates/layerindex/editlayernote.html
+++ b/templates/layerindex/editlayernote.html
@@ -25,7 +25,7 @@
{% csrf_token %}
{{ form.as_p }}
<input type="submit" value="Save" class='btn' />
-<a href="{% url layer_item 'master' form.instance.layer.name %}" class='btn'>Cancel</a>
+<a href="{% url 'layer_item' 'master' form.instance.layer.name %}" class='btn'>Cancel</a>
</form>
{% endautoescape %}
diff --git a/templates/layerindex/layers.html b/templates/layerindex/layers.html
index 4ca43fc..cfc4ebd 100644
--- a/templates/layerindex/layers.html
+++ b/templates/layerindex/layers.html
@@ -18,9 +18,9 @@
{% block navs %}
{% autoescape on %}
- <li class="active"><a href="{% url layer_list url_branch %}">Layers</a></li>
- <li><a href="{% url recipe_search url_branch %}">Recipes</a></li>
- <li><a href="{% url machine_search url_branch %}">Machines</a></li>
+ <li class="active"><a href="{% url 'layer_list' url_branch %}">Layers</a></li>
+ <li><a href="{% url 'recipe_search' url_branch %}">Recipes</a></li>
+ <li><a href="{% url 'machine_search' url_branch %}">Machines</a></li>
{% endautoescape %}
{% endblock %}
@@ -71,7 +71,7 @@
<tbody>
{% for layerbranch in layerbranch_list %}
<tr class="layertype_{{ layerbranch.layer.layer_type }}">
- <td><a href="{% url layer_item url_branch layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
+ <td><a href="{% url 'layer_item' url_branch layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
<td>{{ layerbranch.layer.summary }}</td>
<td>{{ layerbranch.layer.get_layer_type_display }}</td>
<td class="showRollie">
diff --git a/templates/layerindex/machines.html b/templates/layerindex/machines.html
index 1851937..e31433c 100644
--- a/templates/layerindex/machines.html
+++ b/templates/layerindex/machines.html
@@ -17,9 +17,9 @@
{% block navs %}
{% autoescape on %}
- <li><a href="{% url layer_list url_branch %}">Layers</a></li>
- <li><a href="{% url recipe_search url_branch %}">Recipes</a></li>
- <li class="active"><a href="{% url machine_search url_branch %}">Machines</a></li>
+ <li><a href="{% url 'layer_list' url_branch %}">Layers</a></li>
+ <li><a href="{% url 'recipe_search' url_branch %}">Recipes</a></li>
+ <li class="active"><a href="{% url 'machine_search' url_branch %}">Machines</a></li>
{% endautoescape %}
{% endblock %}
@@ -30,7 +30,7 @@
<div class="row-fluid">
<div class="input-append">
- <form id="filter-form" action="{% url machine_search url_branch %}" method="get">
+ <form id="filter-form" action="{% url 'machine_search' url_branch %}" method="get">
<input type="text" class="input-xxlarge" id="appendedInputButtons" placeholder="Search machines" name="q" value="{{ search_keyword }}" />
<button class="btn" type="submit">search</button>
</form>
@@ -52,7 +52,7 @@
<tr>
<td><a href="{{ machine.vcs_web_url }}">{{ machine.name }}</a></td>
<td>{{ machine.description }}</td>
- <td><a href="{% url layer_item url_branch machine.layerbranch.layer.name %}">{{ machine.layerbranch.layer.name }}</a></td>
+ <td><a href="{% url 'layer_item' url_branch machine.layerbranch.layer.name %}">{{ machine.layerbranch.layer.name }}</a></td>
</tr>
{% endfor %}
</tbody>
diff --git a/templates/layerindex/profile.html b/templates/layerindex/profile.html
index 8edc451..d6d25ce 100644
--- a/templates/layerindex/profile.html
+++ b/templates/layerindex/profile.html
@@ -42,7 +42,7 @@
{% endfor %}
<input type="submit" class="btn" value="{% trans 'Save' %}" />
- <a class="btn" href="{% url frontpage %}">{% trans 'Cancel' %}</a>
+ <a class="btn" href="{% url 'frontpage' %}">{% trans 'Cancel' %}</a>
{% csrf_token %}
</form>
diff --git a/templates/layerindex/recipedetail.html b/templates/layerindex/recipedetail.html
index 7dbb362..c0199a1 100644
--- a/templates/layerindex/recipedetail.html
+++ b/templates/layerindex/recipedetail.html
@@ -21,8 +21,8 @@
{% autoescape on %}
<ul class="breadcrumb">
- <li><a href="{% url layer_list recipe.layerbranch.branch.name %}">{{ recipe.layerbranch.branch.name }}</a> <span class="divider">→</span></li>
- <li><a href="{% url layer_item recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> <span class="divider">→</span></li>
+ <li><a href="{% url 'layer_list' recipe.layerbranch.branch.name %}">{{ recipe.layerbranch.branch.name }}</a> <span class="divider">→</span></li>
+ <li><a href="{% url 'layer_item' recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> <span class="divider">→</span></li>
<li class="active">{{ recipe.name }}</li>
</ul>
@@ -36,7 +36,7 @@
{% if recipe.blacklisted %}
<div class="alert">
<div class="row-fluid">
- <p>This recipe is <strong>blacklisted</strong> by the <a href="{% url layer_item recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> layer. The reason provided is:</p>
+ <p>This recipe is <strong>blacklisted</strong> by the <a href="{% url 'layer_item' recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> layer. The reason provided is:</p>
<blockquote class="span7 warn">
<p>{{ recipe.blacklisted }}</p>
</blockquote>
@@ -92,7 +92,7 @@
</tr>
<tr>
<th>Layer</th>
- <td><a href="{% url layer_item recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> ({{ recipe.layerbranch.branch.name}} branch)</td>
+ <td><a href="{% url 'layer_item' recipe.layerbranch.branch.name recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a> ({{ recipe.layerbranch.branch.name}} branch)</td>
</tr>
<tr>
<th>Inherits</th>
@@ -116,7 +116,7 @@
{% for append in verappends %}
<tr>
<td>
- <a href="{% url layer_item append.layerbranch.branch.name append.layerbranch.layer.name %}">{{ append.layerbranch.layer.name }}</a>
+ <a href="{% url 'layer_item' append.layerbranch.branch.name append.layerbranch.layer.name %}">{{ append.layerbranch.layer.name }}</a>
</td>
<td>
<a href="{{ append.vcs_web_url }}">{{ append.filename }}</a>
@@ -127,7 +127,7 @@
{% if not append in verappends %}
<tr>
<td>
- <a href="{% url layer_item append.layerbranch.branch.name append.layerbranch.layer.name %}" class="muted">{{ append.layerbranch.layer.name }}</a>
+ <a href="{% url 'layer_item' append.layerbranch.branch.name append.layerbranch.layer.name %}" class="muted">{{ append.layerbranch.layer.name }}</a>
</td>
<td>
<a href="{{ append.vcs_web_url }}" class="muted">{{ append.filename }}</a>
diff --git a/templates/layerindex/recipes.html b/templates/layerindex/recipes.html
index c2141a4..74f3bb4 100644
--- a/templates/layerindex/recipes.html
+++ b/templates/layerindex/recipes.html
@@ -17,9 +17,9 @@
{% block navs %}
{% autoescape on %}
- <li><a href="{% url layer_list url_branch %}">Layers</a></li>
- <li class="active"><a href="{% url recipe_search url_branch %}">Recipes</a></li>
- <li><a href="{% url machine_search url_branch %}">Machines</a></li>
+ <li><a href="{% url 'layer_list' url_branch %}">Layers</a></li>
+ <li class="active"><a href="{% url 'recipe_search' url_branch %}">Recipes</a></li>
+ <li><a href="{% url 'machine_search' url_branch %}">Machines</a></li>
{% endautoescape %}
{% endblock %}
@@ -31,7 +31,7 @@
<div class="row-fluid">
<div class="input-append">
- <form id="filter-form" action="{% url recipe_search url_branch %}" method="get">
+ <form id="filter-form" action="{% url 'recipe_search' url_branch %}" method="get">
<input type="text" class="input-xxlarge" id="appendedInputButtons" placeholder="Search recipes" name="q" value="{{ search_keyword }}" />
<button class="btn" type="submit">search</button>
</form>
@@ -52,10 +52,10 @@
<tbody>
{% for recipe in recipe_list %}
<tr {% if recipe.preferred_count > 0 %}class="muted"{% endif %}>
- <td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
+ <td><a href="{% url 'recipe' recipe.id %}">{{ recipe.name }}</a>{% if 'image' in recipe.inherits.split %}<i class="icon-hdd"></i>{% endif %}{% if recipe.blacklisted %}<span class="label label-inverse" title="{{ recipe.blacklisted }}">blacklisted</span>{% endif %}</td>
<td>{{ recipe.pv }}</td>
<td>{{ recipe.short_desc }}</td>
- <td><a href="{% url layer_item url_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
+ <td><a href="{% url 'layer_item' url_branch recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
</tr>
{% endfor %}
</tbody>
diff --git a/templates/layerindex/reviewdetail.html b/templates/layerindex/reviewdetail.html
index 5840e25..cff1731 100644
--- a/templates/layerindex/reviewdetail.html
+++ b/templates/layerindex/reviewdetail.html
@@ -35,14 +35,14 @@
{% if user.is_authenticated %}
<span class="pull-right">
{% if perms.layerindex.publish_layer or useredit %}
- <a href="{% url edit_layer 'master' layeritem.name %}?returnto=layer_review" class="btn">Edit layer</a>
+ <a href="{% url 'edit_layer' 'master' layeritem.name %}?returnto=layer_review" class="btn">Edit layer</a>
{% if layeritem.layernote_set.count = 0 %}
- <a href="{% url add_layernote layeritem.name %}" class="btn">Add note</a>
+ <a href="{% url 'add_layernote' layeritem.name %}" class="btn">Add note</a>
{% endif %}
{% endif %}
{% if layeritem.status = "N" and perms.layerindex.publish_layer %}
- <a href="{% url delete_layer layeritem.name %}" class="btn btn-warning">Delete layer</a>
- <a href="{% url publish layeritem.name %}" class="btn btn-primary">Publish layer</a>
+ <a href="{% url 'delete_layer' layeritem.name %}" class="btn btn-warning">Delete layer</a>
+ <a href="{% url 'publish' layeritem.name %}" class="btn btn-primary">Publish layer</a>
{% endif %}
</span>
{% endif %}
@@ -58,8 +58,8 @@
<p>{{ note.text }}</p>
{% if perms.layerindex.publish_layer or useredit %}
<p>
- <a href="{% url edit_layernote layeritem.name note.pk %}" class="btn">Edit note</a>
- <a href="{% url delete_layernote layeritem.name note.pk %}" class='btn'>Delete note</a>
+ <a href="{% url 'edit_layernote' layeritem.name note.pk %}" class="btn">Edit note</a>
+ <a href="{% url 'delete_layernote' layeritem.name note.pk %}" class='btn'>Delete note</a>
</p>
{% endif %}
</div>
@@ -163,7 +163,7 @@
<td>
<ul class="unstyled">
{% for dep in layerbranch.dependencies_set.all %}
- <li><a href="{% url layer_item 'master' dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
+ <li><a href="{% url 'layer_item' 'master' dep.dependency.name %}">{{ dep.dependency.name }}</a></li>
{% endfor %}
</ul>
</td>
diff --git a/templates/layerindex/reviewlist.html b/templates/layerindex/reviewlist.html
index d20001a..eaa24e4 100644
--- a/templates/layerindex/reviewlist.html
+++ b/templates/layerindex/reviewlist.html
@@ -36,7 +36,7 @@
<tbody>
{% for layerbranch in layerbranch_list %}
<tr class="layertype_{{ layerbranch.layer.layer_type }}">
- <td><a href="{% url layer_review layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
+ <td><a href="{% url 'layer_review' layerbranch.layer.name %}">{{ layerbranch.layer.name }}</a></td>
<td>{{ layerbranch.layer.summary }}</td>
<td>{{ layerbranch.layer.get_layer_type_display }}</td>
<td class="showRollie">
diff --git a/templates/registration/activate.html b/templates/registration/activate.html
index e85121e..6e24890 100644
--- a/templates/registration/activate.html
+++ b/templates/registration/activate.html
@@ -7,7 +7,7 @@
<p>{% trans "Account successfully activated" %}</p>
-<p><a href="{% url auth_login %}">{% trans "Log in" %}</a></p>
+<p><a href="{% url "auth_login" %}">{% trans "Log in" %}</a></p>
{% else %}
diff --git a/templates/registration/activation_email.txt b/templates/registration/activation_email.txt
index 0820fce..736ea8a 100644
--- a/templates/registration/activation_email.txt
+++ b/templates/registration/activation_email.txt
@@ -7,7 +7,7 @@ link is valid for {{ expiration_days }} days.
{% endblocktrans %}
-http://{{ site.domain }}{% url registration_activate activation_key %}
+http://{{ site.domain }}{% url "registration_activate" activation_key %}
{% blocktrans %}
If you did not make this request, please ignore this message.
diff --git a/templates/registration/login.html b/templates/registration/login.html
index 7e3d615..7f8a153 100644
--- a/templates/registration/login.html
+++ b/templates/registration/login.html
@@ -10,8 +10,8 @@
{% csrf_token %}
</form>
-<p>{% trans "Forgot password" %}? <a href="{% url auth_password_reset %}">{% trans "Reset it" %}</a>!</p>
-<p>{% trans "Don't have an account" %}? <a href="{% url registration_register %}">{% trans "Create one now" %}</a>!</p>
+<p>{% trans "Forgot password" %}? <a href="{% url "auth_password_reset" %}">{% trans "Reset it" %}</a>!</p>
+<p>{% trans "Don't have an account" %}? <a href="{% url "registration_register" %}">{% trans "Create one now" %}</a>!</p>
{% endblock %}
diff --git a/templates/registration/password_reset_complete.html b/templates/registration/password_reset_complete.html
index ef3637c..d9f3879 100644
--- a/templates/registration/password_reset_complete.html
+++ b/templates/registration/password_reset_complete.html
@@ -5,6 +5,6 @@
<p>{% trans "Password reset successfully" %}</p>
-<p><a href="{% url auth_login %}">{% trans "Log in" %}</a></p>
+<p><a href="{% url "auth_login" %}">{% trans "Log in" %}</a></p>
{% endblock %}
diff --git a/templates/registration/password_reset_email.html b/templates/registration/password_reset_email.html
index a55c869..f219c68 100644
--- a/templates/registration/password_reset_email.html
+++ b/templates/registration/password_reset_email.html
@@ -1,5 +1,5 @@
{% load i18n %}
{% blocktrans %}Reset password at {{ site_name }}{% endblocktrans %}:
{% block reset_link %}
-{{ protocol }}://{{ domain }}{% url auth_password_reset_confirm uidb36=uid, token=token %}
+{{ protocol }}://{{ domain }}{% url "auth_password_reset_confirm" uidb36=uid, token=token %}
{% endblock %}
diff --git a/urls.py b/urls.py
index aa4f8c5..98fc734 100644
--- a/urls.py
+++ b/urls.py
@@ -5,8 +5,8 @@
# Copyright (c) Django Software Foundation and individual contributors.
# All rights reserved.
-from django.conf.urls.defaults import patterns, include, url
-from django.views.generic.simple import redirect_to
+from django.conf.urls import patterns, include, url
+from django.views.generic import RedirectView
from django.contrib import admin
admin.autodiscover()
@@ -16,6 +16,6 @@ urlpatterns = patterns('',
url(r'^admin/', include(admin.site.urls)),
url(r'^accounts/', include('registration.backends.default.urls')),
url(r'^captcha/', include('captcha.urls')),
- url(r'.*', redirect_to, {'url' : '/layerindex/'})
+ url(r'.*', RedirectView.as_view(url='/layerindex/')),
)
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 08/15] Fix for changes in modern django-reversion
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (6 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 07/15] Upgrade to Django 1.6+ Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 09/15] Support (and require) Python 3 Paul Eggleton
` (6 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
We have to upgrade django-reversion to 1.8 due to upgrading Django, but
unfortunately in that same version the author has removed the type field
on Version model, without a particularly good explanation as to why. This
is really annoying as we were using it to provide a reasonable audit
including for deletes. I suspect we'll need to move away from
django-reversion and do our own thing in future, but for now at least
allow the layer index to keep working.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/views.py | 45 +++++++++++++++++++++++----------------------
1 file changed, 23 insertions(+), 22 deletions(-)
diff --git a/layerindex/views.py b/layerindex/views.py
index d9480c9..1f42039 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -630,29 +630,30 @@ def annotate_revision(sender, **kwargs):
for ver, inst in zip(versions, instances):
currentVersion = ver.field_dict
modelmeta = ver.content_type.model_class()._meta
- if ver.type == reversion.models.VERSION_DELETE:
- changelist.append("Deleted %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
+ #FIXME modern django-reversion dropped the type field (argh!)
+ #if ver.type == reversion.models.VERSION_DELETE:
+ # changelist.append("Deleted %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
+ #else:
+ pastver = reversion.get_for_object(inst)
+ if pastver:# and ver.type != reversion.models.VERSION_ADD:
+ pastVersion = pastver[0].field_dict
+ changes = set(currentVersion.items()) - set(pastVersion.items())
+ changedVars = [var[0] for var in changes]
+ fieldchanges = []
+ for field in changedVars:
+ if field not in ignorefields:
+ modelfield = modelmeta.get_field(field)
+ newvalue = currentVersion[field]
+ if modelfield.choices:
+ for v in modelfield.choices:
+ if v[0] == newvalue:
+ newvalue = v[1]
+ break
+ fieldchanges.append("%s to '%s'" % (modelfield.verbose_name.lower(), newvalue))
+ if fieldchanges:
+ changelist.append("Changed %s %s %s" % (modelmeta.verbose_name.lower(), ver.object_repr, ", ".join(fieldchanges)))
else:
- pastver = reversion.get_for_object(inst)
- if pastver and ver.type != reversion.models.VERSION_ADD:
- pastVersion = pastver[0].field_dict
- changes = set(currentVersion.items()) - set(pastVersion.items())
- changedVars = [var[0] for var in changes]
- fieldchanges = []
- for field in changedVars:
- if field not in ignorefields:
- modelfield = modelmeta.get_field(field)
- newvalue = currentVersion[field]
- if modelfield.choices:
- for v in modelfield.choices:
- if v[0] == newvalue:
- newvalue = v[1]
- break
- fieldchanges.append("%s to '%s'" % (modelfield.verbose_name.lower(), newvalue))
- if fieldchanges:
- changelist.append("Changed %s %s %s" % (modelmeta.verbose_name.lower(), ver.object_repr, ", ".join(fieldchanges)))
- else:
- changelist.append("Added %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
+ changelist.append("Added %s: %s" % (modelmeta.verbose_name.lower(), ver.object_repr))
comment = '\n'.join(changelist)
if not comment:
comment = 'No changes'
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 09/15] Support (and require) Python 3
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (7 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 08/15] Fix for changes in modern django-reversion Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 10/15] update_layer.py: rename confusing loop variables Paul Eggleton
` (5 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
We need to be able to support Python 3 so that we can parse master of
OE-Core with bitbake (which now requires it). This now means the
interface itself and the update script require Python 3.4+.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
README | 6 ++++--
layerindex/bulkchange.py | 10 +++++-----
layerindex/models.py | 24 ++++++++++++------------
layerindex/recipedesc.py | 4 ++--
layerindex/templatetags/addurlparameter.py | 2 +-
layerindex/tools/import_layer.py | 2 +-
layerindex/update.py | 2 +-
layerindex/urls.py | 2 +-
layerindex/utils.py | 7 ++++---
layerindex/views.py | 9 +++++----
10 files changed, 36 insertions(+), 32 deletions(-)
diff --git a/README b/README
index db788df..b3374c9 100644
--- a/README
+++ b/README
@@ -11,6 +11,7 @@ Setup
In order to make use of this application you will need:
+* Python 3.4+
* Django 1.6.x - tested with 1.6.10; newer versions may work, but
the application has not been tested with 1.7 or newer.
* For production usage, a web server set up to host Django applications
@@ -32,8 +33,9 @@ In order to make use of this application you will need:
have to have Django installed, have the same or similar configuration
in settings.py and have access to the database used by the web
application):
- * Python 2.7.3
- * GitPython (python-git) version 0.3.1 or later
+ * Python 2.7.6+ / Python 3.4+ to match with the version of BitBake
+ for the OpenEmbedded branch being parsed
+ * GitPython (python-git) version 2.0 or later
Setup instructions:
diff --git a/layerindex/bulkchange.py b/layerindex/bulkchange.py
index 08d6d78..8102571 100644
--- a/layerindex/bulkchange.py
+++ b/layerindex/bulkchange.py
@@ -50,7 +50,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
outfile = open(os.path.join(tmpoutdir, patchname), 'w')
last_layer = layer
recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
- varlist = list(set(fields.keys() + meta_vars))
+ varlist = list(set(list(fields.keys()) + meta_vars))
varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
filevars = localise_file_vars(recipefile, varfiles, fields.keys())
for f, fvars in filevars.items():
@@ -66,7 +66,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
ret = None
if len(patches) > 1:
(tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir)
- tmptarfile = os.fdopen(tmptarfd, "w")
+ tmptarfile = os.fdopen(tmptarfd, "wb")
tar = tarfile.open(None, "w:gz", tmptarfile)
for patch in patches:
patchfn = os.path.join(tmpoutdir, patch)
@@ -75,7 +75,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
ret = tmptarname
elif len(patches) == 1:
(tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir)
- tmppatchfile = os.fdopen(tmppatchfd, "w")
+ tmppatchfile = os.fdopen(tmppatchfd, "wb")
with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile:
shutil.copyfileobj(patchfile, tmppatchfile)
tmppatchfile.close()
@@ -93,7 +93,7 @@ def patch_recipe(fn, relpath, values):
remainingnames = {}
for k in values.keys():
remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
- remainingnames = SortedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))
+ remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
with tempfile.NamedTemporaryFile('w', delete=False) as tf:
def outputvalue(name):
@@ -234,7 +234,7 @@ def main():
utils.unlock_file(lockfile)
if outp:
- print outp
+ print(outp)
else:
sys.stderr.write("No changes to write\n")
sys.exit(1)
diff --git a/layerindex/models.py b/layerindex/models.py
index 8d82846..e753fbe 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -26,7 +26,7 @@ class Branch(models.Model):
class Meta:
verbose_name_plural = "Branches"
- def __unicode__(self):
+ def __str__(self):
return self.name
@@ -106,7 +106,7 @@ class LayerItem(models.Model):
def get_absolute_url(self):
return reverse('layer_item', args=('master',self.name));
- def __unicode__(self):
+ def __str__(self):
return self.name
@@ -188,7 +188,7 @@ class LayerBranch(models.Model):
url = resolveComponents(url)
return url
- def __unicode__(self):
+ def __str__(self):
return "%s: %s" % (self.layer.name, self.branch.name)
@@ -203,7 +203,7 @@ class LayerMaintainer(models.Model):
responsibility = models.CharField(max_length=200, blank=True, help_text='Specific area(s) this maintainer is responsible for, if not the entire layer')
status = models.CharField(max_length=1, choices=MAINTAINER_STATUS_CHOICES, default='A')
- def __unicode__(self):
+ def __str__(self):
respstr = ""
if self.responsibility:
respstr = " (%s)" % self.responsibility
@@ -217,7 +217,7 @@ class LayerDependency(models.Model):
class Meta:
verbose_name_plural = "Layer dependencies"
- def __unicode__(self):
+ def __str__(self):
return "%s depends on %s" % (self.layerbranch.layer.name, self.dependency.name)
@@ -225,7 +225,7 @@ class LayerNote(models.Model):
layer = models.ForeignKey(LayerItem)
text = models.TextField()
- def __unicode__(self):
+ def __str__(self):
return "%s: %s" % (self.layer.name, self.text)
@@ -266,7 +266,7 @@ class Recipe(models.Model):
else:
return self.filename.split('_')[0]
- def __unicode__(self):
+ def __str__(self):
return os.path.join(self.filepath, self.filename)
@@ -278,7 +278,7 @@ class RecipeFileDependency(models.Model):
class Meta:
verbose_name_plural = "Recipe file dependencies"
- def __unicode__(self):
+ def __str__(self):
return '%s' % self.path
@@ -341,7 +341,7 @@ class Machine(models.Model):
url = self.layerbranch.file_url(os.path.join('conf/machine/%s.conf' % self.name))
return url or ''
- def __unicode__(self):
+ def __str__(self):
return '%s (%s)' % (self.name, self.layerbranch.layer.name)
@@ -367,7 +367,7 @@ class BBAppend(models.Model):
return fnmatch.fnmatch(recipename, appendname.replace('%', '*'))
return False
- def __unicode__(self):
+ def __str__(self):
return os.path.join(self.filepath, self.filename)
@@ -383,7 +383,7 @@ class BBClass(models.Model):
url = self.layerbranch.file_url(os.path.join('classes', "%s.bbclass" % self.name))
return url or ''
- def __unicode__(self):
+ def __str__(self):
return '%s (%s)' % (self.name, self.layerbranch.layer.name)
@@ -391,7 +391,7 @@ class RecipeChangeset(models.Model):
user = models.ForeignKey(User)
name = models.CharField(max_length=255)
- def __unicode__(self):
+ def __str__(self):
return '%s' % (self.name)
diff --git a/layerindex/recipedesc.py b/layerindex/recipedesc.py
index 1925a27..43376cf 100644
--- a/layerindex/recipedesc.py
+++ b/layerindex/recipedesc.py
@@ -82,10 +82,10 @@ def main():
layerrecipes = Recipe.objects.filter(layer=layer)
for recipe in layerrecipes:
fullpath = str(os.path.join(repodir, layer.vcs_subdir, recipe.filepath, recipe.filename))
- print fullpath
+ print(fullpath)
try:
envdata = bb.cache.Cache.loadDataFull(fullpath, [], tinfoil.config_data)
- print "DESCRIPTION = \"%s\"" % envdata.getVar("DESCRIPTION", True)
+ print("DESCRIPTION = \"%s\"" % envdata.getVar("DESCRIPTION", True))
except Exception as e:
logger.info("Unable to read %s: %s", fullpath, str(e))
diff --git a/layerindex/templatetags/addurlparameter.py b/layerindex/templatetags/addurlparameter.py
index 58a454a..cfebd41 100644
--- a/layerindex/templatetags/addurlparameter.py
+++ b/layerindex/templatetags/addurlparameter.py
@@ -23,7 +23,7 @@ def addurlparameter(parser, token):
from re import split
bits = split(r'\s+', token.contents, 2)
if len(bits) < 2:
- raise TemplateSyntaxError, "'%s' tag requires two arguments" % bits[0]
+ raise TemplateSyntaxError("'%s' tag requires two arguments" % bits[0])
return AddParameter(bits[1],bits[2])
register.tag('addurlparameter', addurlparameter)
diff --git a/layerindex/tools/import_layer.py b/layerindex/tools/import_layer.py
index a0771b4..b472d2a 100755
--- a/layerindex/tools/import_layer.py
+++ b/layerindex/tools/import_layer.py
@@ -164,7 +164,7 @@ def get_github_layerinfo(layer_url, username = None, password = None):
data = resp.read()
json_data = json.loads(data)
#headers = dict((key, value) for key, value in resp.getheaders())
- #print headers
+ #print(headers)
owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
if resp.status in [200, 302]:
owner_data = owner_resp.read()
diff --git a/layerindex/update.py b/layerindex/update.py
index ee4138d..157d46f 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Fetch layer repositories and update layer index database
#
diff --git a/layerindex/urls.py b/layerindex/urls.py
index e7808e6..b170460 100644
--- a/layerindex/urls.py
+++ b/layerindex/urls.py
@@ -11,7 +11,7 @@ from django.core.urlresolvers import reverse_lazy
from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView
from layerindex.models import LayerItem, Recipe, RecipeChangeset
from rest_framework import routers
-import restviews
+from . import restviews
from django.conf.urls import include
router = routers.DefaultRouter()
diff --git a/layerindex/utils.py b/layerindex/utils.py
index ebb89d3..72dc342 100644
--- a/layerindex/utils.py
+++ b/layerindex/utils.py
@@ -6,7 +6,8 @@
# Licensed under the MIT license, see COPYING.MIT for details
import sys
-import os.path
+import os
+import tempfile
import subprocess
import logging
import time
@@ -32,10 +33,10 @@ def runcmd(cmd, destdir=None, printerr=True, logger=None):
return output if succeed
"""
#logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
- out = os.tmpfile()
+ out = tempfile.TemporaryFile()
try:
subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
- except subprocess.CalledProcessError,e:
+ except subprocess.CalledProcessError as e:
out.seek(0)
if printerr:
output = out.read()
diff --git a/layerindex/views.py b/layerindex/views.py
index 1f42039..dfe4453 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -4,6 +4,7 @@
#
# Licensed under the MIT license, see COPYING.MIT for details
+import sys
from django.shortcuts import get_object_or_404, render
from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404
from django.core.urlresolvers import reverse, reverse_lazy, resolve
@@ -26,7 +27,7 @@ from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from reversion.models import Revision
-import simplesearch
+from . import simplesearch
import settings
from django.dispatch import receiver
import reversion
@@ -212,14 +213,14 @@ def bulk_change_edit_view(request, template_name, pk):
def bulk_change_patch_view(request, pk):
import os
import os.path
- import utils
+ from layerindex.utils import runcmd
changeset = get_object_or_404(RecipeChangeset, pk=pk)
# FIXME this couples the web server and machine running the update script together,
# but given that it's a separate script the way is open to decouple them in future
try:
- ret = utils.runcmd('python bulkchange.py %d %s' % (int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
+ ret = runcmd('%s bulkchange.py %d %s' % (sys.executable, int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
if ret:
- fn = ret.splitlines()[-1]
+ fn = ret.splitlines()[-1].decode('utf-8')
if os.path.exists(fn):
if fn.endswith('.tar.gz'):
mimetype = 'application/x-gzip'
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 10/15] update_layer.py: rename confusing loop variables
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (8 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 09/15] Support (and require) Python 3 Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 11/15] update_layer.py: fix handling of renames with newer GitPython Paul Eggleton
` (4 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
Stop using "d" (sometimes multiple levels!) and use proper descriptive
variable names instead.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/update_layer.py | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index bf1dbb2..ac4ae1e 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -312,9 +312,9 @@ def main():
# Check if any paths should be ignored because there are layers within this layer
removedirs = []
for root, dirs, files in os.walk(layerdir):
- for d in dirs:
- if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
- removedirs.append(os.path.join(root, d) + os.sep)
+ for diritem in dirs:
+ if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
+ removedirs.append(os.path.join(root, diritem) + os.sep)
if diff:
# Apply git changes to existing recipe list
@@ -325,12 +325,12 @@ def main():
subdir_start = ""
updatedrecipes = set()
- for d in diff.iter_change_type('D'):
- path = d.a_blob.path
+ for diffitem in diff.iter_change_type('D'):
+ path = diffitem.a_blob.path
if path.startswith(subdir_start):
skip = False
- for d in removedirs:
- if path.startswith(d):
+ for removedir in removedirs:
+ if path.startswith(removedir):
skip = True
break
if skip:
@@ -351,12 +351,12 @@ def main():
elif typename == 'bbclass':
layerclasses.filter(name=filename).delete()
- for d in diff.iter_change_type('A'):
- path = d.b_blob.path
+ for diffitem in diff.iter_change_type('A'):
+ path = diffitem.b_blob.path
if path.startswith(subdir_start):
skip = False
- for d in removedirs:
- if path.startswith(d):
+ for removedir in removedirs:
+ if path.startswith(removedir):
skip = True
break
if skip:
@@ -385,12 +385,12 @@ def main():
bbclass.save()
dirtyrecipes = set()
- for d in diff.iter_change_type('M'):
- path = d.a_blob.path
+ for diffitem in diff.iter_change_type('M'):
+ path = diffitem.a_blob.path
if path.startswith(subdir_start):
skip = False
- for d in removedirs:
- if path.startswith(d):
+ for removedir in removedirs:
+ if path.startswith(removedir):
skip = True
break
if skip:
@@ -432,8 +432,8 @@ def main():
fullpath = os.path.join(root, v['filename'])
preserve = True
if os.path.exists(fullpath):
- for d in removedirs:
- if fullpath.startswith(d):
+ for removedir in removedirs:
+ if fullpath.startswith(removedir):
preserve = False
break
else:
@@ -455,10 +455,10 @@ def main():
for root, dirs, files in os.walk(layerdir):
if '.git' in dirs:
dirs.remove('.git')
- for d in dirs[:]:
- fullpath = os.path.join(root, d) + os.sep
+ for diritem in dirs[:]:
+ fullpath = os.path.join(root, diritem) + os.sep
if fullpath in removedirs:
- dirs.remove(d)
+ dirs.remove(diritem)
for f in files:
fullpath = os.path.join(root, f)
(typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 11/15] update_layer.py: fix handling of renames with newer GitPython
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (9 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 10/15] update_layer.py: rename confusing loop variables Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 12/15] Handle Python 2 and Python 3 branches in the same index Paul Eggleton
` (3 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
Newer versions of GitPython implement rename detection, which means that
such changes don't show up as adds and deletes anymore and you get a
bunch of ENOENT errors for renamed files. Add some code to explicitly
look for renames and process them appropriately.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/update_layer.py | 78 ++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 75 insertions(+), 3 deletions(-)
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index ac4ae1e..bda79cb 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -17,6 +17,7 @@ import re
import tempfile
import shutil
from distutils.version import LooseVersion
+import itertools
import utils
import recipeparse
@@ -325,7 +326,79 @@ def main():
subdir_start = ""
updatedrecipes = set()
- for diffitem in diff.iter_change_type('D'):
+ dirtyrecipes = set()
+ other_deletes = []
+ other_adds = []
+ for diffitem in diff.iter_change_type('R'):
+ oldpath = diffitem.a_blob.path
+ newpath = diffitem.b_blob.path
+ skip = False
+ for removedir in removedirs:
+ # FIXME what about files moved into removedirs?
+ if oldpath.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ if oldpath.startswith(subdir_start):
+ (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
+ (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
+ if oldtypename != newtypename:
+ # This is most likely to be a .inc file renamed to a .bb - and since
+ # there may be another recipe deleted at the same time we probably want
+ # to consider that, so just treat it as a delete and an add
+ logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
+ other_deletes.append(diffitem)
+ other_adds.append(diffitem)
+ elif oldtypename == 'recipe':
+ results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
+ if len(results):
+ recipe = results[0]
+ logger.debug("Rename recipe %s to %s" % (recipe, newpath))
+ recipe.filepath = newfilepath
+ recipe.filename = newfilename
+ recipe.save()
+ update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
+ updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
+ updatedrecipes.add(os.path.join(newfilepath, newfilename))
+ else:
+ logger.warn("Renamed recipe %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'bbappend':
+ results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
+ if len(results):
+ logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
+ results[0].filepath = newfilepath
+ results[0].filename = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed bbappend %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'machine':
+ results = layermachines.filter(name=oldfilename)
+ if len(results):
+ logger.debug("Rename machine %s to %s" % (results[0], newfilename))
+ results[0].name = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed machine %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'bbclass':
+ results = layerclasses.filter(name=oldfilename)
+ if len(results):
+ logger.debug("Rename class %s to %s" % (results[0], newfilename))
+ results[0].name = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed class %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+
+ for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
path = diffitem.a_blob.path
if path.startswith(subdir_start):
skip = False
@@ -351,7 +424,7 @@ def main():
elif typename == 'bbclass':
layerclasses.filter(name=filename).delete()
- for diffitem in diff.iter_change_type('A'):
+ for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
skip = False
@@ -384,7 +457,6 @@ def main():
bbclass.name = filename
bbclass.save()
- dirtyrecipes = set()
for diffitem in diff.iter_change_type('M'):
path = diffitem.a_blob.path
if path.startswith(subdir_start):
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 12/15] Handle Python 2 and Python 3 branches in the same index
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (10 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 11/15] update_layer.py: fix handling of renames with newer GitPython Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 13/15] Preemptive auto_now fix for Django 1.8 Paul Eggleton
` (2 subsequent siblings)
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
Add a model to support setting a python command and virtualenv per
branch, which allows you to parse master with python3 and krogoth with
python2 for example.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
README | 22 ++-
layerindex/admin.py | 1 +
...ronment__add_field_branch_update_environment.py | 218 +++++++++++++++++++++
layerindex/models.py | 19 +-
layerindex/update.py | 7 +-
5 files changed, 259 insertions(+), 8 deletions(-)
create mode 100644 layerindex/migrations/0013_auto__add_pythonenvironment__add_field_branch_update_environment.py
diff --git a/README b/README
index b3374c9..fa5c3b2 100644
--- a/README
+++ b/README
@@ -99,17 +99,27 @@ Setup instructions:
next to your login name at the top right, click on the
newly added layer entry, and then click on "Publish Layer".
-5. Set the site name (as displayed in the top bar and page titles) by
+5. If you need to support multiple branches of OpenEmbedded/BitBake
+ where some require Python 2.x and others require Python 3.x, then
+ you will need to set up "Python environment" records through the
+ admin interface to correspond to these so that the right Python
+ version gets used to parse the branch, and then set the
+ "Update environment" field on each branch record to point to the
+ appropriate environment. If you're using virtualenv you will need
+ separate virtual environments set up for Python 2 and 3 which you
+ should point to in the Python environment record.
+
+6. Set the site name (as displayed in the top bar and page titles) by
going into the admin interface (http://127.0.0.1:8000/admin/),
clicking on "Sites" at the bottom, and editing the first entry,
setting "Display name" to the desired name.
-6. You may wish to customise some of the page templates to suit your
+7. You may wish to customise some of the page templates to suit your
installation, in particular:
* templates/base.html
* templates/layerindex/about.html
-7. To use layerindex-web with Docker containers, refer to docker/README
+8. To use layerindex-web with Docker containers, refer to docker/README
keeping in mind you'll need to set up Docker properly as part of the
setup process.
@@ -156,7 +166,7 @@ Bundled nv.d3.js is redistributed under the Apache License 2.0.
Bundled d3.js is redistributed under the BSD License.
-All other content is copyright (C) 2013 Intel Corporation and licensed
-under the MIT license (unless otherwise noted) - see COPYING.MIT for
-details.
+All other content is copyright (C) 2013-2016 Intel Corporation and
+licensed under the MIT license (unless otherwise noted) - see
+COPYING.MIT for details.
diff --git a/layerindex/admin.py b/layerindex/admin.py
index 77f660b..accb954 100644
--- a/layerindex/admin.py
+++ b/layerindex/admin.py
@@ -115,3 +115,4 @@ admin.site.register(BBAppend, BBAppendAdmin)
admin.site.register(BBClass, BBClassAdmin)
admin.site.register(RecipeChangeset, RecipeChangesetAdmin)
admin.site.register(ClassicRecipe, ClassicRecipeAdmin)
+admin.site.register(PythonEnvironment)
diff --git a/layerindex/migrations/0013_auto__add_pythonenvironment__add_field_branch_update_environment.py b/layerindex/migrations/0013_auto__add_pythonenvironment__add_field_branch_update_environment.py
new file mode 100644
index 0000000..d4c80c6
--- /dev/null
+++ b/layerindex/migrations/0013_auto__add_pythonenvironment__add_field_branch_update_environment.py
@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding model 'PythonEnvironment'
+ db.create_table('layerindex_pythonenvironment', (
+ ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('name', self.gf('django.db.models.fields.CharField')(max_length=50)),
+ ('python_command', self.gf('django.db.models.fields.CharField')(default='python', max_length=255)),
+ ('virtualenv_path', self.gf('django.db.models.fields.CharField')(blank=True, max_length=255)),
+ ))
+ db.send_create_signal('layerindex', ['PythonEnvironment'])
+
+ # Adding field 'Branch.update_environment'
+ db.add_column('layerindex_branch', 'update_environment',
+ self.gf('django.db.models.fields.related.ForeignKey')(blank=True, to=orm['layerindex.PythonEnvironment'], on_delete=models.SET_NULL, null=True),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ # Deleting model 'PythonEnvironment'
+ db.delete_table('layerindex_pythonenvironment')
+
+ # Deleting field 'Branch.update_environment'
+ db.delete_column('layerindex_branch', 'update_environment_id')
+
+
+ models = {
+ 'auth.group': {
+ 'Meta': {'object_name': 'Group'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+ 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'to': "orm['auth.Permission']", 'symmetrical': 'False'})
+ },
+ 'auth.permission': {
+ 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'object_name': 'Permission'},
+ 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+ },
+ 'auth.user': {
+ 'Meta': {'object_name': 'User'},
+ 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'email': ('django.db.models.fields.EmailField', [], {'blank': 'True', 'max_length': '75'}),
+ 'first_name': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '30'}),
+ 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'to': "orm['auth.Group']", 'related_name': "'user_set'", 'symmetrical': 'False'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+ 'last_name': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '30'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+ 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'blank': 'True', 'to': "orm['auth.Permission']", 'related_name': "'user_set'", 'symmetrical': 'False'}),
+ 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+ },
+ 'contenttypes.contenttype': {
+ 'Meta': {'unique_together': "(('app_label', 'model'),)", 'ordering': "('name',)", 'db_table': "'django_content_type'", 'object_name': 'ContentType'},
+ 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.bbappend': {
+ 'Meta': {'object_name': 'BBAppend'},
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
+ },
+ 'layerindex.bbclass': {
+ 'Meta': {'object_name': 'BBClass'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+ },
+ 'layerindex.branch': {
+ 'Meta': {'object_name': 'Branch'},
+ 'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'short_description': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '50'}),
+ 'sort_priority': ('django.db.models.fields.IntegerField', [], {'blank': 'True', 'null': 'True'}),
+ 'update_environment': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'to': "orm['layerindex.PythonEnvironment']", 'on_delete': 'models.SET_NULL', 'null': 'True'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'blank': 'True', 'auto_now': 'True'}),
+ 'updates_enabled': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+ },
+ 'layerindex.classicrecipe': {
+ 'Meta': {'object_name': 'ClassicRecipe', '_ormbases': ['layerindex.Recipe']},
+ 'classic_category': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'cover_comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'cover_layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'to': "orm['layerindex.LayerBranch']", 'on_delete': 'models.SET_NULL', 'null': 'True'}),
+ 'cover_pn': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'cover_status': ('django.db.models.fields.CharField', [], {'default': "'U'", 'max_length': '1'}),
+ 'cover_verified': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'recipe_ptr': ('django.db.models.fields.related.OneToOneField', [], {'unique': 'True', 'primary_key': 'True', 'to': "orm['layerindex.Recipe']"})
+ },
+ 'layerindex.layerbranch': {
+ 'Meta': {'object_name': 'LayerBranch'},
+ 'actual_branch': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '80'}),
+ 'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'blank': 'True', 'auto_now': 'True'}),
+ 'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'blank': 'True', 'null': 'True'}),
+ 'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'blank': 'True', 'null': 'True'}),
+ 'vcs_last_rev': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '80'}),
+ 'vcs_subdir': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '40'})
+ },
+ 'layerindex.layerdependency': {
+ 'Meta': {'object_name': 'LayerDependency'},
+ 'dependency': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']", 'related_name': "'dependents_set'"}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']", 'related_name': "'dependencies_set'"})
+ },
+ 'layerindex.layeritem': {
+ 'Meta': {'object_name': 'LayerItem'},
+ 'classic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'description': ('django.db.models.fields.TextField', [], {}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+ 'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+ 'mailing_list_url': ('django.db.models.fields.URLField', [], {'blank': 'True', 'max_length': '200'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'blank': 'True', 'auto_now': 'True'}),
+ 'usage_url': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'vcs_web_url': ('django.db.models.fields.URLField', [], {'blank': 'True', 'max_length': '200'})
+ },
+ 'layerindex.layermaintainer': {
+ 'Meta': {'object_name': 'LayerMaintainer'},
+ 'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'responsibility': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '200'}),
+ 'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
+ },
+ 'layerindex.layernote': {
+ 'Meta': {'object_name': 'LayerNote'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+ 'text': ('django.db.models.fields.TextField', [], {})
+ },
+ 'layerindex.machine': {
+ 'Meta': {'object_name': 'Machine'},
+ 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'blank': 'True', 'auto_now': 'True'})
+ },
+ 'layerindex.pythonenvironment': {
+ 'Meta': {'object_name': 'PythonEnvironment'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+ 'python_command': ('django.db.models.fields.CharField', [], {'default': "'python'", 'max_length': '255'}),
+ 'virtualenv_path': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'})
+ },
+ 'layerindex.recipe': {
+ 'Meta': {'object_name': 'Recipe'},
+ 'bbclassextend': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'blacklisted': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'blank': 'True', 'max_length': '200'}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'filepath': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'blank': 'True', 'max_length': '200'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'inherits': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '255'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+ 'license': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '2048'}),
+ 'pn': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'provides': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '2048'}),
+ 'pv': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'section': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '200'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {'blank': 'True', 'auto_now': 'True'})
+ },
+ 'layerindex.recipechange': {
+ 'Meta': {'object_name': 'RecipeChange'},
+ 'bugtracker': ('django.db.models.fields.URLField', [], {'blank': 'True', 'max_length': '200'}),
+ 'changeset': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.RecipeChangeset']"}),
+ 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'homepage': ('django.db.models.fields.URLField', [], {'blank': 'True', 'max_length': '200'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'license': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']", 'related_name': "'+'"}),
+ 'section': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'}),
+ 'summary': ('django.db.models.fields.CharField', [], {'blank': 'True', 'max_length': '100'})
+ },
+ 'layerindex.recipechangeset': {
+ 'Meta': {'object_name': 'RecipeChangeset'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+ },
+ 'layerindex.recipefiledependency': {
+ 'Meta': {'object_name': 'RecipeFileDependency'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']", 'related_name': "'+'"}),
+ 'path': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255'}),
+ 'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
+ }
+ }
+
+ complete_apps = ['layerindex']
\ No newline at end of file
diff --git a/layerindex/models.py b/layerindex/models.py
index e753fbe..3a5796d 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -1,6 +1,6 @@
# layerindex-web - model definitions
#
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
#
# Licensed under the MIT license, see COPYING.MIT for details
@@ -14,12 +14,29 @@ import re
import posixpath
+class PythonEnvironment(models.Model):
+ name = models.CharField(max_length=50)
+ python_command = models.CharField(max_length=255, default='python')
+ virtualenv_path = models.CharField(max_length=255, blank=True)
+
+ def get_command(self):
+ if self.virtualenv_path:
+ cmd = '. %s/bin/activate; %s' % (self.virtualenv_path, self.python_command)
+ else:
+ cmd = self.python_command
+ return cmd
+
+ def __str__(self):
+ return self.name
+
+
class Branch(models.Model):
name = models.CharField(max_length=50)
bitbake_branch = models.CharField(max_length=50)
short_description = models.CharField(max_length=50, blank=True)
sort_priority = models.IntegerField(blank=True, null=True)
updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
+ update_environment = models.ForeignKey(PythonEnvironment, blank=True, null=True, on_delete=models.SET_NULL)
updated = models.DateTimeField(auto_now = True, default = datetime.now)
diff --git a/layerindex/update.py b/layerindex/update.py
index 157d46f..ac7de5e 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -177,7 +177,12 @@ def main():
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
- cmd = 'python update_layer.py -l %s -b %s' % (layer.name, branch)
+ branchobj = utils.get_branch(branch)
+ if branchobj.update_environment:
+ cmdprefix = branchobj.update_environment.get_command()
+ else:
+ cmdprefix = 'python'
+ cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch)
if options.reload:
cmd += ' --reload'
if options.fullreload:
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 13/15] Preemptive auto_now fix for Django 1.8
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (11 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 12/15] Handle Python 2 and Python 3 branches in the same index Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 14/15] update_layer.py: use new-style transaction API Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 15/15] Explicitly specify temporary redirection Paul Eggleton
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
Django 1.8 does not allow DateFields that have both auto_now and default
set - since they are mutually exclusive; if you have auto_now you don't
need a default.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/models.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/layerindex/models.py b/layerindex/models.py
index 3a5796d..0ed90a4 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -38,7 +38,7 @@ class Branch(models.Model):
updates_enabled = models.BooleanField('Enable updates', default=True, help_text='Enable automatically updating layer metadata for this branch via the update script')
update_environment = models.ForeignKey(PythonEnvironment, blank=True, null=True, on_delete=models.SET_NULL)
- updated = models.DateTimeField(auto_now = True, default = datetime.now)
+ updated = models.DateTimeField(auto_now=True)
class Meta:
verbose_name_plural = "Branches"
@@ -73,7 +73,7 @@ class LayerItem(models.Model):
index_preference = models.IntegerField('Preference', default=0, help_text='Number used to find preferred recipes in recipe search results (higher number is greater preference)')
classic = models.BooleanField('Classic', default=False, help_text='Is this OE-Classic?')
- updated = models.DateTimeField(auto_now = True)
+ updated = models.DateTimeField(auto_now=True)
class Meta:
verbose_name = "Layer"
@@ -136,7 +136,7 @@ class LayerBranch(models.Model):
vcs_last_commit = models.DateTimeField('Last commit date', blank=True, null=True)
actual_branch = models.CharField('Actual Branch', max_length=80, blank=True, help_text='Name of the actual branch in the repository matching the core branch')
- updated = models.DateTimeField(auto_now = True)
+ updated = models.DateTimeField(auto_now=True)
class Meta:
verbose_name_plural = "Layer branches"
@@ -261,7 +261,7 @@ class Recipe(models.Model):
provides = models.CharField(max_length=2048, blank=True)
bbclassextend = models.CharField(max_length=100, blank=True)
inherits = models.CharField(max_length=255, blank=True)
- updated = models.DateTimeField(auto_now = True)
+ updated = models.DateTimeField(auto_now=True)
blacklisted = models.CharField(max_length=255, blank=True)
def vcs_web_url(self):
@@ -352,7 +352,7 @@ class Machine(models.Model):
name = models.CharField(max_length=255)
description = models.CharField(max_length=255)
- updated = models.DateTimeField(auto_now = True)
+ updated = models.DateTimeField(auto_now=True)
def vcs_web_url(self):
url = self.layerbranch.file_url(os.path.join('conf/machine/%s.conf' % self.name))
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 14/15] update_layer.py: use new-style transaction API
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (12 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 13/15] Preemptive auto_now fix for Django 1.8 Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
2016-06-08 13:20 ` [layerindex-web][PATCH v2 15/15] Explicitly specify temporary redirection Paul Eggleton
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
The old transaction API has been removed in Django 1.8 and was
deprecated at 1.6. There's no explicit open transaction, commit or
rollback now - we just wrap the layer operations in
"with transaction.atomic()"; if we need to roll back we just raise a
"dummy" exception.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/update_layer.py | 781 ++++++++++++++++++++++-----------------------
1 file changed, 389 insertions(+), 392 deletions(-)
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index bda79cb..fd8d3d4 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -34,6 +34,10 @@ except ImportError:
sys.exit(1)
+class DryRunRollbackException(Exception):
+ pass
+
+
def check_machine_conf(path, subdir_start):
subpath = path[len(subdir_start):]
res = conf_re.match(subpath)
@@ -187,429 +191,422 @@ def main():
# why won't they just fix that?!)
tinfoil.config_data.setVar('LICENSE', '')
- transaction.enter_transaction_management()
- transaction.managed(True)
try:
- layer = utils.get_layer(options.layer)
- urldir = layer.get_fetch_dir()
- repodir = os.path.join(fetchdir, urldir)
-
- layerbranch = layer.get_layerbranch(options.branch)
-
- branchname = options.branch
- branchdesc = options.branch
- if layerbranch:
- if layerbranch.actual_branch:
- branchname = layerbranch.actual_branch
- branchdesc = "%s (%s)" % (options.branch, branchname)
-
- # Collect repo info
- repo = git.Repo(repodir)
- assert repo.bare == False
- try:
- if options.nocheckout:
- topcommit = repo.commit('HEAD')
- else:
- topcommit = repo.commit('origin/%s' % branchname)
- except:
- if layerbranch:
- logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
- else:
- logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
- transaction.rollback()
- sys.exit(1)
-
- newbranch = False
- if not layerbranch:
- # LayerBranch doesn't exist for this branch, create it
- newbranch = True
- layerbranch = LayerBranch()
- layerbranch.layer = layer
- layerbranch.branch = branch
- layerbranch_source = layer.get_layerbranch('master')
- if not layerbranch_source:
- layerbranch_source = layer.get_layerbranch(None)
- if layerbranch_source:
- layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
- layerbranch.save()
- if layerbranch_source:
- for maintainer in layerbranch_source.layermaintainer_set.all():
- maintainer.pk = None
- maintainer.id = None
- maintainer.layerbranch = layerbranch
- maintainer.save()
- for dep in layerbranch_source.dependencies_set.all():
- dep.pk = None
- dep.id = None
- dep.layerbranch = layerbranch
- dep.save()
-
- if layerbranch.vcs_subdir and not options.nocheckout:
- # Find latest commit in subdirectory
- # A bit odd to do it this way but apparently there's no other way in the GitPython API
- topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
- if not topcommit:
- # This will error out if the directory is completely invalid or had never existed at this point
- # If it previously existed but has since been deleted, you will get the revision where it was
- # deleted - so we need to handle that case separately later
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- elif layerbranch.vcs_subdir:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- else:
- logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
- transaction.rollback()
- sys.exit(1)
-
- layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
- layerdir_start = os.path.normpath(layerdir) + os.sep
- layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
- layermachines = Machine.objects.filter(layerbranch=layerbranch)
- layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
- layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
- if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
- # Check out appropriate branch
- if not options.nocheckout:
- out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
- out = utils.runcmd("git clean -f -x", repodir, logger=logger)
-
- if layerbranch.vcs_subdir and not os.path.exists(layerdir):
- if newbranch:
- logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
- else:
- logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
- transaction.rollback()
- sys.exit(1)
+ with transaction.atomic():
+ layer = utils.get_layer(options.layer)
+ urldir = layer.get_fetch_dir()
+ repodir = os.path.join(fetchdir, urldir)
- if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
- logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
- transaction.rollback()
- sys.exit(1)
+ layerbranch = layer.get_layerbranch(options.branch)
- logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+ branchname = options.branch
+ branchdesc = options.branch
+ if layerbranch:
+ if layerbranch.actual_branch:
+ branchname = layerbranch.actual_branch
+ branchdesc = "%s (%s)" % (options.branch, branchname)
+ # Collect repo info
+ repo = git.Repo(repodir)
+ assert repo.bare == False
try:
- config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
- except recipeparse.RecipeParseError as e:
- logger.error(str(e))
- transaction.rollback()
+ if options.nocheckout:
+ topcommit = repo.commit('HEAD')
+ else:
+ topcommit = repo.commit('origin/%s' % branchname)
+ except:
+ if layerbranch:
+ logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
+ else:
+ logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
sys.exit(1)
- if layerbranch.vcs_last_rev and not options.reload:
+ newbranch = False
+ if not layerbranch:
+ # LayerBranch doesn't exist for this branch, create it
+ newbranch = True
+ layerbranch = LayerBranch()
+ layerbranch.layer = layer
+ layerbranch.branch = branch
+ layerbranch_source = layer.get_layerbranch('master')
+ if not layerbranch_source:
+ layerbranch_source = layer.get_layerbranch(None)
+ if layerbranch_source:
+ layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
+ layerbranch.save()
+ if layerbranch_source:
+ for maintainer in layerbranch_source.layermaintainer_set.all():
+ maintainer.pk = None
+ maintainer.id = None
+ maintainer.layerbranch = layerbranch
+ maintainer.save()
+ for dep in layerbranch_source.dependencies_set.all():
+ dep.pk = None
+ dep.id = None
+ dep.layerbranch = layerbranch
+ dep.save()
+
+ if layerbranch.vcs_subdir and not options.nocheckout:
+ # Find latest commit in subdirectory
+ # A bit odd to do it this way but apparently there's no other way in the GitPython API
+ topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
+ if not topcommit:
+ # This will error out if the directory is completely invalid or had never existed at this point
+ # If it previously existed but has since been deleted, you will get the revision where it was
+ # deleted - so we need to handle that case separately later
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ elif layerbranch.vcs_subdir:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ else:
+ logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
+ sys.exit(1)
+
+ layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+ layerdir_start = os.path.normpath(layerdir) + os.sep
+ layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
+ layermachines = Machine.objects.filter(layerbranch=layerbranch)
+ layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
+ layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
+ if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
+ # Check out appropriate branch
+ if not options.nocheckout:
+ out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
+ out = utils.runcmd("git clean -f -x", repodir, logger=logger)
+
+ if layerbranch.vcs_subdir and not os.path.exists(layerdir):
+ if newbranch:
+ logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
+ else:
+ logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
+ sys.exit(1)
+
+ if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
+ logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
+ sys.exit(1)
+
+ logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
+
try:
- diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
- except Exception as e:
- logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
- diff = None
- else:
- diff = None
-
- # We handle recipes specially to try to preserve the same id
- # when recipe upgrades happen (so that if a user bookmarks a
- # recipe page it remains valid)
- layerrecipes_delete = []
- layerrecipes_add = []
-
- # Check if any paths should be ignored because there are layers within this layer
- removedirs = []
- for root, dirs, files in os.walk(layerdir):
- for diritem in dirs:
- if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
- removedirs.append(os.path.join(root, diritem) + os.sep)
-
- if diff:
- # Apply git changes to existing recipe list
-
- if layerbranch.vcs_subdir:
- subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+ config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
+ except recipeparse.RecipeParseError as e:
+ logger.error(str(e))
+ sys.exit(1)
+
+ if layerbranch.vcs_last_rev and not options.reload:
+ try:
+ diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
+ except Exception as e:
+ logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
+ diff = None
else:
- subdir_start = ""
-
- updatedrecipes = set()
- dirtyrecipes = set()
- other_deletes = []
- other_adds = []
- for diffitem in diff.iter_change_type('R'):
- oldpath = diffitem.a_blob.path
- newpath = diffitem.b_blob.path
- skip = False
- for removedir in removedirs:
- # FIXME what about files moved into removedirs?
- if oldpath.startswith(removedir):
- skip = True
- break
- if skip:
- continue
- if oldpath.startswith(subdir_start):
- (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
- (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
- if oldtypename != newtypename:
- # This is most likely to be a .inc file renamed to a .bb - and since
- # there may be another recipe deleted at the same time we probably want
- # to consider that, so just treat it as a delete and an add
- logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
- other_deletes.append(diffitem)
- other_adds.append(diffitem)
- elif oldtypename == 'recipe':
- results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
- if len(results):
- recipe = results[0]
- logger.debug("Rename recipe %s to %s" % (recipe, newpath))
- recipe.filepath = newfilepath
- recipe.filename = newfilename
- recipe.save()
- update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
- updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
- updatedrecipes.add(os.path.join(newfilepath, newfilename))
- else:
- logger.warn("Renamed recipe %s could not be found" % oldpath)
- other_adds.append(diffitem)
- elif oldtypename == 'bbappend':
- results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
- if len(results):
- logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
- results[0].filepath = newfilepath
- results[0].filename = newfilename
- results[0].save()
- else:
- logger.warn("Renamed bbappend %s could not be found" % oldpath)
- other_adds.append(diffitem)
- elif oldtypename == 'machine':
- results = layermachines.filter(name=oldfilename)
- if len(results):
- logger.debug("Rename machine %s to %s" % (results[0], newfilename))
- results[0].name = newfilename
- results[0].save()
- else:
- logger.warn("Renamed machine %s could not be found" % oldpath)
- other_adds.append(diffitem)
- elif oldtypename == 'bbclass':
- results = layerclasses.filter(name=oldfilename)
- if len(results):
- logger.debug("Rename class %s to %s" % (results[0], newfilename))
- results[0].name = newfilename
- results[0].save()
- else:
- logger.warn("Renamed class %s could not be found" % oldpath)
- other_adds.append(diffitem)
-
- deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
- for dep in deps:
- dirtyrecipes.add(dep.recipe)
+ diff = None
+ # We handle recipes specially to try to preserve the same id
+ # when recipe upgrades happen (so that if a user bookmarks a
+ # recipe page it remains valid)
+ layerrecipes_delete = []
+ layerrecipes_add = []
- for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
- path = diffitem.a_blob.path
- if path.startswith(subdir_start):
+ # Check if any paths should be ignored because there are layers within this layer
+ removedirs = []
+ for root, dirs, files in os.walk(layerdir):
+ for diritem in dirs:
+ if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
+ removedirs.append(os.path.join(root, diritem) + os.sep)
+
+ if diff:
+ # Apply git changes to existing recipe list
+
+ if layerbranch.vcs_subdir:
+ subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
+ else:
+ subdir_start = ""
+
+ updatedrecipes = set()
+ dirtyrecipes = set()
+ other_deletes = []
+ other_adds = []
+ for diffitem in diff.iter_change_type('R'):
+ oldpath = diffitem.a_blob.path
+ newpath = diffitem.b_blob.path
skip = False
for removedir in removedirs:
- if path.startswith(removedir):
+ # FIXME what about files moved into removedirs?
+ if oldpath.startswith(removedir):
skip = True
break
if skip:
continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
- if len(values):
- layerrecipes_delete.append(values[0])
- logger.debug("Mark %s for deletion" % values[0])
- updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+ if oldpath.startswith(subdir_start):
+ (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
+ (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
+ if oldtypename != newtypename:
+ # This is most likely to be a .inc file renamed to a .bb - and since
+ # there may be another recipe deleted at the same time we probably want
+ # to consider that, so just treat it as a delete and an add
+ logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
+ other_deletes.append(diffitem)
+ other_adds.append(diffitem)
+ elif oldtypename == 'recipe':
+ results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
+ if len(results):
+ recipe = results[0]
+ logger.debug("Rename recipe %s to %s" % (recipe, newpath))
+ recipe.filepath = newfilepath
+ recipe.filename = newfilename
+ recipe.save()
+ update_recipe_file(config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
+ updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
+ updatedrecipes.add(os.path.join(newfilepath, newfilename))
+ else:
+ logger.warn("Renamed recipe %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'bbappend':
+ results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
+ if len(results):
+ logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
+ results[0].filepath = newfilepath
+ results[0].filename = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed bbappend %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'machine':
+ results = layermachines.filter(name=oldfilename)
+ if len(results):
+ logger.debug("Rename machine %s to %s" % (results[0], newfilename))
+ results[0].name = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed machine %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+ elif oldtypename == 'bbclass':
+ results = layerclasses.filter(name=oldfilename)
+ if len(results):
+ logger.debug("Rename class %s to %s" % (results[0], newfilename))
+ results[0].name = newfilename
+ results[0].save()
+ else:
+ logger.warn("Renamed class %s could not be found" % oldpath)
+ other_adds.append(diffitem)
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+
+ for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
+ path = diffitem.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for removedir in removedirs:
+ if path.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
+ if len(values):
+ layerrecipes_delete.append(values[0])
+ logger.debug("Mark %s for deletion" % values[0])
+ updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
+ else:
+ logger.warn("Deleted recipe %s could not be found" % path)
+ elif typename == 'bbappend':
+ layerappends.filter(filepath=filepath).filter(filename=filename).delete()
+ elif typename == 'machine':
+ layermachines.filter(name=filename).delete()
+ elif typename == 'bbclass':
+ layerclasses.filter(name=filename).delete()
+
+ for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
+ path = diffitem.b_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for removedir in removedirs:
+ if path.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ layerrecipes_add.append(os.path.join(repodir, path))
+ logger.debug("Mark %s for addition" % path)
+ updatedrecipes.add(os.path.join(filepath, filename))
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = filename
+ append.filepath = filepath
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ for diffitem in diff.iter_change_type('M'):
+ path = diffitem.a_blob.path
+ if path.startswith(subdir_start):
+ skip = False
+ for removedir in removedirs:
+ if path.startswith(removedir):
+ skip = True
+ break
+ if skip:
+ continue
+ (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
+ if typename == 'recipe':
+ logger.debug("Mark %s for update" % path)
+ results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
+ if results:
+ recipe = results[0]
+ update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
+ recipe.save()
+ updatedrecipes.add(recipe.full_path())
+ elif typename == 'machine':
+ results = layermachines.filter(name=filename)
+ if results:
+ machine = results[0]
+ update_machine_conf_file(os.path.join(repodir, path), machine)
+ machine.save()
+
+ deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
+ for dep in deps:
+ dirtyrecipes.add(dep.recipe)
+
+ for recipe in dirtyrecipes:
+ if not recipe.full_path() in updatedrecipes:
+ update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+ else:
+ # Collect recipe data from scratch
+
+ layerrecipe_fns = []
+ if options.fullreload:
+ layerrecipes.delete()
+ else:
+ # First, check which recipes still exist
+ layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
+ for v in layerrecipe_values:
+ root = os.path.join(layerdir, v['filepath'])
+ fullpath = os.path.join(root, v['filename'])
+ preserve = True
+ if os.path.exists(fullpath):
+ for removedir in removedirs:
+ if fullpath.startswith(removedir):
+ preserve = False
+ break
else:
- logger.warn("Deleted recipe %s could not be found" % path)
- elif typename == 'bbappend':
- layerappends.filter(filepath=filepath).filter(filename=filename).delete()
- elif typename == 'machine':
- layermachines.filter(name=filename).delete()
- elif typename == 'bbclass':
- layerclasses.filter(name=filename).delete()
-
- for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
- path = diffitem.b_blob.path
- if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- layerrecipes_add.append(os.path.join(repodir, path))
- logger.debug("Mark %s for addition" % path)
- updatedrecipes.add(os.path.join(filepath, filename))
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = filename
- append.filepath = filepath
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(os.path.join(repodir, path), machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- for diffitem in diff.iter_change_type('M'):
- path = diffitem.a_blob.path
- if path.startswith(subdir_start):
- skip = False
- for removedir in removedirs:
- if path.startswith(removedir):
- skip = True
- break
- if skip:
- continue
- (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
- if typename == 'recipe':
- logger.debug("Mark %s for update" % path)
- results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
- if results:
+ preserve = False
+
+ if preserve:
+ # Recipe still exists, update it
+ results = layerrecipes.filter(id=v['id'])[:1]
recipe = results[0]
- update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
- recipe.save()
- updatedrecipes.add(recipe.full_path())
- elif typename == 'machine':
- results = layermachines.filter(name=filename)
- if results:
- machine = results[0]
- update_machine_conf_file(os.path.join(repodir, path), machine)
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ else:
+ # Recipe no longer exists, mark it for later on
+ layerrecipes_delete.append(v)
+ layerrecipe_fns.append(fullpath)
+
+ layermachines.delete()
+ layerappends.delete()
+ layerclasses.delete()
+ for root, dirs, files in os.walk(layerdir):
+ if '.git' in dirs:
+ dirs.remove('.git')
+ for diritem in dirs[:]:
+ fullpath = os.path.join(root, diritem) + os.sep
+ if fullpath in removedirs:
+ dirs.remove(diritem)
+ for f in files:
+ fullpath = os.path.join(root, f)
+ (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
+ if typename == 'recipe':
+ if fullpath not in layerrecipe_fns:
+ layerrecipes_add.append(fullpath)
+ elif typename == 'bbappend':
+ append = BBAppend()
+ append.layerbranch = layerbranch
+ append.filename = f
+ append.filepath = os.path.relpath(root, layerdir)
+ append.save()
+ elif typename == 'machine':
+ machine = Machine()
+ machine.layerbranch = layerbranch
+ machine.name = filename
+ update_machine_conf_file(fullpath, machine)
machine.save()
+ elif typename == 'bbclass':
+ bbclass = BBClass()
+ bbclass.layerbranch = layerbranch
+ bbclass.name = filename
+ bbclass.save()
+
+ for added in layerrecipes_add:
+ # This is good enough without actually parsing the file
+ (pn, pv) = split_recipe_fn(added)
+ oldid = -1
+ for deleted in layerrecipes_delete:
+ if deleted['pn'] == pn:
+ oldid = deleted['id']
+ layerrecipes_delete.remove(deleted)
+ break
+ if oldid > -1:
+ # Reclaim a record we would have deleted
+ results = Recipe.objects.filter(id=oldid)[:1]
+ recipe = results[0]
+ logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
+ else:
+ # Create new record
+ logger.debug("Add new recipe %s" % added)
+ recipe = Recipe()
+ recipe.layerbranch = layerbranch
+ recipe.filename = os.path.basename(added)
+ root = os.path.dirname(added)
+ recipe.filepath = os.path.relpath(root, layerdir)
+ update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
+ recipe.save()
- deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
- for dep in deps:
- dirtyrecipes.add(dep.recipe)
+ for deleted in layerrecipes_delete:
+ logger.debug("Delete %s" % deleted)
+ results = Recipe.objects.filter(id=deleted['id'])[:1]
+ recipe = results[0]
+ recipe.delete()
- for recipe in dirtyrecipes:
- if not recipe.full_path() in updatedrecipes:
- update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
+ # Save repo info
+ layerbranch.vcs_last_rev = topcommit.hexsha
+ layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
else:
- # Collect recipe data from scratch
+ logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
+
+ layerbranch.vcs_last_fetch = datetime.now()
+ layerbranch.save()
+
+ if options.dryrun:
+ raise DryRunRollbackException()
- layerrecipe_fns = []
- if options.fullreload:
- layerrecipes.delete()
- else:
- # First, check which recipes still exist
- layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
- for v in layerrecipe_values:
- root = os.path.join(layerdir, v['filepath'])
- fullpath = os.path.join(root, v['filename'])
- preserve = True
- if os.path.exists(fullpath):
- for removedir in removedirs:
- if fullpath.startswith(removedir):
- preserve = False
- break
- else:
- preserve = False
-
- if preserve:
- # Recipe still exists, update it
- results = layerrecipes.filter(id=v['id'])[:1]
- recipe = results[0]
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- else:
- # Recipe no longer exists, mark it for later on
- layerrecipes_delete.append(v)
- layerrecipe_fns.append(fullpath)
-
- layermachines.delete()
- layerappends.delete()
- layerclasses.delete()
- for root, dirs, files in os.walk(layerdir):
- if '.git' in dirs:
- dirs.remove('.git')
- for diritem in dirs[:]:
- fullpath = os.path.join(root, diritem) + os.sep
- if fullpath in removedirs:
- dirs.remove(diritem)
- for f in files:
- fullpath = os.path.join(root, f)
- (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
- if typename == 'recipe':
- if fullpath not in layerrecipe_fns:
- layerrecipes_add.append(fullpath)
- elif typename == 'bbappend':
- append = BBAppend()
- append.layerbranch = layerbranch
- append.filename = f
- append.filepath = os.path.relpath(root, layerdir)
- append.save()
- elif typename == 'machine':
- machine = Machine()
- machine.layerbranch = layerbranch
- machine.name = filename
- update_machine_conf_file(fullpath, machine)
- machine.save()
- elif typename == 'bbclass':
- bbclass = BBClass()
- bbclass.layerbranch = layerbranch
- bbclass.name = filename
- bbclass.save()
-
- for added in layerrecipes_add:
- # This is good enough without actually parsing the file
- (pn, pv) = split_recipe_fn(added)
- oldid = -1
- for deleted in layerrecipes_delete:
- if deleted['pn'] == pn:
- oldid = deleted['id']
- layerrecipes_delete.remove(deleted)
- break
- if oldid > -1:
- # Reclaim a record we would have deleted
- results = Recipe.objects.filter(id=oldid)[:1]
- recipe = results[0]
- logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
- else:
- # Create new record
- logger.debug("Add new recipe %s" % added)
- recipe = Recipe()
- recipe.layerbranch = layerbranch
- recipe.filename = os.path.basename(added)
- root = os.path.dirname(added)
- recipe.filepath = os.path.relpath(root, layerdir)
- update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
- recipe.save()
-
- for deleted in layerrecipes_delete:
- logger.debug("Delete %s" % deleted)
- results = Recipe.objects.filter(id=deleted['id'])[:1]
- recipe = results[0]
- recipe.delete()
-
- # Save repo info
- layerbranch.vcs_last_rev = topcommit.hexsha
- layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
- else:
- logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
-
- layerbranch.vcs_last_fetch = datetime.now()
- layerbranch.save()
-
- if options.dryrun:
- transaction.rollback()
- else:
- transaction.commit()
except KeyboardInterrupt:
- transaction.rollback()
logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
sys.exit(254)
+ except SystemExit:
+ raise
+ except DryRunRollbackException:
+ pass
except:
import traceback
traceback.print_exc()
- transaction.rollback()
- finally:
- transaction.leave_transaction_management()
shutil.rmtree(tempdir)
sys.exit(0)
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread* [layerindex-web][PATCH v2 15/15] Explicitly specify temporary redirection
2016-06-08 13:19 [layerindex-web][PATCH v2 00/15] Layer index improvements Paul Eggleton
` (13 preceding siblings ...)
2016-06-08 13:20 ` [layerindex-web][PATCH v2 14/15] update_layer.py: use new-style transaction API Paul Eggleton
@ 2016-06-08 13:20 ` Paul Eggleton
14 siblings, 0 replies; 16+ messages in thread
From: Paul Eggleton @ 2016-06-08 13:20 UTC (permalink / raw)
To: yocto
Django 1.8 warns that the default for the "permanent" parameter to
RedirectView is changing in 1.9 from True to False, but I believe we
should be specifying False for these redirections - these are not just
redirections from old URLs and may in fact change in the future if the
site structure changes.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/urls.py | 12 ++++++------
layerindex/urls_branch.py | 2 +-
urls.py | 2 +-
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/layerindex/urls.py b/layerindex/urls.py
index b170460..cbd29ad 100644
--- a/layerindex/urls.py
+++ b/layerindex/urls.py
@@ -24,19 +24,19 @@ router.register(r'machines', restviews.MachineViewSet)
urlpatterns = patterns('',
url(r'^$',
- RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',))),
+ RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)), permanent=False),
name='frontpage'),
url(r'^api/', include(router.urls)),
url(r'^layers/$',
- RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)))),
+ RedirectView.as_view(url=reverse_lazy('layer_list', args=('master',)), permanent=False)),
url(r'^layer/(?P<slug>[-\w]+)/$',
- RedirectParamsView.as_view(), {'redirect_name': 'layer_item', 'branch':'master'}),
+ RedirectParamsView.as_view(permanent=False), {'redirect_name': 'layer_item', 'branch':'master'}),
url(r'^recipes/$',
- RedirectView.as_view(url=reverse_lazy('recipe_search', args=('master',)))),
+ RedirectView.as_view(url=reverse_lazy('recipe_search', args=('master',)), permanent=False)),
url(r'^machines/$',
- RedirectView.as_view(url=reverse_lazy('machine_search', args=('master',)))),
+ RedirectView.as_view(url=reverse_lazy('machine_search', args=('master',)), permanent=False)),
url(r'^submit/$', edit_layer_view, {'template_name': 'layerindex/submitlayer.html'}, name="submit_layer"),
url(r'^submit/thanks$',
@@ -107,7 +107,7 @@ urlpatterns = patterns('',
template_name='layerindex/about.html'),
name="about"),
url(r'^oe-classic/$',
- RedirectView.as_view(url=reverse_lazy('classic_recipe_search')),
+ RedirectView.as_view(url=reverse_lazy('classic_recipe_search'), permanent=False),
name='classic'),
url(r'^oe-classic/recipes/$',
ClassicRecipeSearchView.as_view(
diff --git a/layerindex/urls_branch.py b/layerindex/urls_branch.py
index ab5e2d5..3313290 100644
--- a/layerindex/urls_branch.py
+++ b/layerindex/urls_branch.py
@@ -11,7 +11,7 @@ from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView,
urlpatterns = patterns('',
url(r'^$',
- RedirectParamsView.as_view(), {'redirect_name': 'layer_list'}),
+ RedirectParamsView.as_view(permanent=False), {'redirect_name': 'layer_list'}),
url(r'^layers/$',
LayerListView.as_view(
template_name='layerindex/layers.html'),
diff --git a/urls.py b/urls.py
index 98fc734..649635c 100644
--- a/urls.py
+++ b/urls.py
@@ -16,6 +16,6 @@ urlpatterns = patterns('',
url(r'^admin/', include(admin.site.urls)),
url(r'^accounts/', include('registration.backends.default.urls')),
url(r'^captcha/', include('captcha.urls')),
- url(r'.*', RedirectView.as_view(url='/layerindex/')),
+ url(r'.*', RedirectView.as_view(url='/layerindex/', permanent=False)),
)
--
2.5.5
^ permalink raw reply related [flat|nested] 16+ messages in thread