[layerindex-web][PATCH 0/6] Layer index improvements

[layerindex-web][PATCH 0/6] Layer index improvements

[Paul Eggleton]

Fixes for some minor issues, plus add a few small features.


The following changes since commit 09f629b997da1e52b1cb93d36566e04908e0517c:

  Add CSV export for layer recipes (2018-05-07 15:30:04 +1200)

are available in the Git repository at:

  git://git.yoctoproject.org/layerindex-web paule/fixes2
  http://git.yoctoproject.org/cgit.cgi//log/?h=paule/fixes2

Paul Eggleton (6):
  settings: allow disabling layer publishing emails
  update_layer: drop debug message for deleting recipe file dependencies
  update_layer: avoid errors on modified & renamed files
  Add site-wide notice support
  Add a script to import layers from an existing layer index
  update_classic_status: fix matching on wrong layer

 TODO                                      |   1 -
 layerindex/admin.py                       |   1 +
 layerindex/context_processors.py          |   7 +-
 layerindex/migrations/0014_sitenotice.py  |  24 +++
 layerindex/models.py                      |  25 ++-
 layerindex/tools/import_layers.py         | 251 ++++++++++++++++++++++++++++++
 layerindex/tools/update_classic_status.py |   2 +-
 layerindex/update_layer.py                |   3 +-
 layerindex/utils.py                       |  14 ++
 layerindex/views.py                       |  66 ++++----
 requirements.txt                          |   1 +
 settings.py                               |   3 +
 templates/base.html                       |   5 +
 13 files changed, 364 insertions(+), 39 deletions(-)
 create mode 100644 layerindex/migrations/0014_sitenotice.py
 create mode 100755 layerindex/tools/import_layers.py

-- 
2.14.3

[layerindex-web][PATCH 1/6] settings: allow disabling layer publishing emails

[Paul Eggleton]

If you're running a testing / internal instance then you really don't
want to be emailing maintainers on publish, so provide a setting you can
use to disable that.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/views.py | 66 +++++++++++++++++++++++++++--------------------------
 settings.py         |  3 +++
 2 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/layerindex/views.py b/layerindex/views.py
index a905c347..d02e4f91 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -259,38 +259,40 @@ def _check_url_branch(kwargs):
 def publish_view(request, name):
     if not (request.user.is_authenticated() and request.user.has_perm('layerindex.publish_layer')):
         raise PermissionDenied
-    layeritem = get_object_or_404(LayerItem, name=name)
-    layerbranch = get_object_or_404(LayerBranch, layer=layeritem)
-    layer_url = request.build_absolute_uri(reverse('layer_item', args=(layerbranch.branch, layeritem.name)))
-    maintainers = get_list_or_404(LayerMaintainer, layerbranch=layerbranch)
-    from_email = settings.SUBMIT_EMAIL_FROM
-    subjecttext = get_template('layerindex/publishemailsubject.txt')
-    bodytext = get_template('layerindex/publishemail.txt')
-    maintainer_names = [m.name for m in maintainers]
-    # find appropriate help contact
-    help_contact = None
-    for user in User.objects.all():
-        if user.username != 'root' and (user.is_staff or user.is_superuser) and user.is_active:
-            help_contact = user
-            break
-
-    # create subject from subject template
-    d = {
-        'layer_name': layeritem.name,
-        'site_name': request.META['HTTP_HOST'],
-    }
-    subject = subjecttext.render(d).rstrip()
-
-    #create body from body template
-    d = {
-        'maintainers': maintainer_names,
-        'layer_name': layeritem.name,
-        'layer_url': layer_url,
-        'help_contact': help_contact,
-    }
-    body = bodytext.render(d)
-
-    tasks.send_email.apply_async((subject, body, from_email, [m.email for m in maintainers]))
+
+    if getattr(settings, 'SEND_PUBLISH_EMAIL', True):
+        layeritem = get_object_or_404(LayerItem, name=name)
+        layerbranch = get_object_or_404(LayerBranch, layer=layeritem)
+        layer_url = request.build_absolute_uri(reverse('layer_item', args=(layerbranch.branch, layeritem.name)))
+        maintainers = get_list_or_404(LayerMaintainer, layerbranch=layerbranch)
+        from_email = settings.SUBMIT_EMAIL_FROM
+        subjecttext = get_template('layerindex/publishemailsubject.txt')
+        bodytext = get_template('layerindex/publishemail.txt')
+        maintainer_names = [m.name for m in maintainers]
+        # find appropriate help contact
+        help_contact = None
+        for user in User.objects.all():
+            if user.username != 'root' and (user.is_staff or user.is_superuser) and user.is_active:
+                help_contact = user
+                break
+
+        # create subject from subject template
+        d = {
+            'layer_name': layeritem.name,
+            'site_name': request.META['HTTP_HOST'],
+        }
+        subject = subjecttext.render(d).rstrip()
+
+        #create body from body template
+        d = {
+            'maintainers': maintainer_names,
+            'layer_name': layeritem.name,
+            'layer_url': layer_url,
+            'help_contact': help_contact,
+        }
+        body = bodytext.render(d)
+
+        tasks.send_email.apply_async((subject, body, from_email, [m.email for m in maintainers]))
 
     return _statuschange(request, name, 'P')
 
diff --git a/settings.py b/settings.py
index c3075a5e..3d70aaa5 100644
--- a/settings.py
+++ b/settings.py
@@ -227,6 +227,9 @@ FORCE_REVIEW_HTTPS = False
 SUBMIT_EMAIL_FROM = 'noreply@example.com'
 SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
 
+# Send email to maintainer(s) when their layer is published
+SEND_PUBLISH_EMAIL = True
+
 # RabbitMQ settings
 RABBIT_BROKER = 'amqp://'
 RABBIT_BACKEND = 'rpc://'
-- 
2.14.3

[layerindex-web][PATCH 2/6] update_layer: drop debug message for deleting recipe file dependencies

[Paul Eggleton]

This was too noisy and not particularly useful.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/update_layer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index bbfaba9a..018d7bcd 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -228,7 +228,6 @@ def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir, ski
             recipedep.save()
 
         for filedep in recipedeps_delete:
-            logger.debug('%s: removing %s' % (recipe.layerbranch, filedep))
             recipedeps.filter(path=filedep).delete()
 
     except KeyboardInterrupt:
-- 
2.14.3

[layerindex-web][PATCH 3/6] update_layer: avoid errors on modified & renamed files

[Paul Eggleton]

If a file is modified and renamed it will show up in both
iter_change_type('M') and iter_change_type('R'), however naturally the
file that will exist will be the b path and not the a one, so we should
be looking at the b path or we will get errors.

FYI you can reproduce this with OE-Core (in a scratch database) using
the following procedure:
1) (in the OE-Core layer directory):
   git checkout 59285b324f6d9ed270b0bef209ef5da22a620a83
2) update.py -l openembedded-core -b master -x --nofetch -r --fullreload
3) (in the OE-Core layer directory):
   git checkout 086308aa2a5e332de6f00ed397c4a55d132f158f
4) update.py -l openembedded-core -b master -x --nofetch

Without this change you'll see the following error:

ERROR: Unable to read /opt/layerindex/layers/git___git_openembedded_org_openembedded-core/meta/recipes-devtools/python-numpy/python-numpy_1.13.1.bb: Traceback (most recent call last):
  File "/opt/layerindex/layers/bitbake/lib/bb/command.py", line 84, in runCommand
    result = command_method(self, commandline)
  File "/opt/layerindex/layers/bitbake/lib/bb/command.py", line 568, in parseRecipeFile
    envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)['']
  File "/opt/layerindex/layers/bitbake/lib/bb/cache.py", line 315, in parse_recipe
    bb_data = bb.parse.handle(bbfile, bb_data)
  File "/opt/layerindex/layers/bitbake/lib/bb/parse/__init__.py", line 117, in handle
    return h['handle'](fn, data, include)
  File "/opt/layerindex/layers/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 132, in handle
    abs_fn = resolve_file(fn, d)
  File "/opt/layerindex/layers/bitbake/lib/bb/parse/__init__.py", line 141, in resolve_file
    raise IOError(errno.ENOENT, "file %s not found" % fn)
FileNotFoundError: [Errno 2] file /opt/layerindex/layers/git___git_openembedded_org_openembedded-core/meta/recipes-devtools/python-numpy/python-numpy_1.13.1.bb not found

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 TODO                       | 1 -
 layerindex/update_layer.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/TODO b/TODO
index 96d4feb8..7d522f84 100644
--- a/TODO
+++ b/TODO
@@ -10,7 +10,6 @@ TODO:
 
 Bugs
 * Duplication of first maintainer when editing to add a second?
-* GitPython sometimes reports renamed files as type M but we don't handle that properly e.g. OE-Core 59285b324f6d9ed270b0bef209ef5da22a620a83 086308aa2a5e332de6f00ed397c4a55d132f158f
 
 Other
 * Full-text search on layer contents
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index 018d7bcd..272fe51a 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -636,7 +636,7 @@ def main():
                                 bbclass.save()
 
                     for diffitem in diff.iter_change_type('M'):
-                        path = diffitem.a_blob.path
+                        path = diffitem.b_blob.path
                         if path.startswith(subdir_start):
                             skip = False
                             for removedir in removedirs:
-- 
2.14.3

[layerindex-web][PATCH 4/6] Add site-wide notice support

[Paul Eggleton]

Add the ability to show a notice at the top of every page; this provides
the ability for admins to display a message to visitors in the case of
infrastructure or index data issues. Notices can have an expiry date and
can be disabled and re-enabled if needed. A subset of HTML can be used
for formatting the text, URLs will be made into clickable links, and
four "levels" are supported (info, success, warning and error).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/admin.py                      |  1 +
 layerindex/context_processors.py         |  7 +++++--
 layerindex/migrations/0014_sitenotice.py | 24 ++++++++++++++++++++++++
 layerindex/models.py                     | 25 ++++++++++++++++++++++++-
 layerindex/utils.py                      | 14 ++++++++++++++
 requirements.txt                         |  1 +
 templates/base.html                      |  5 +++++
 7 files changed, 74 insertions(+), 3 deletions(-)
 create mode 100644 layerindex/migrations/0014_sitenotice.py

diff --git a/layerindex/admin.py b/layerindex/admin.py
index 3cb59691..312f7a30 100644
--- a/layerindex/admin.py
+++ b/layerindex/admin.py
@@ -198,3 +198,4 @@ admin.site.register(Patch)
 admin.site.register(RecipeChangeset, RecipeChangesetAdmin)
 admin.site.register(ClassicRecipe, ClassicRecipeAdmin)
 admin.site.register(PythonEnvironment)
+admin.site.register(SiteNotice)
diff --git a/layerindex/context_processors.py b/layerindex/context_processors.py
index db8e3fa3..7cf20ede 100644
--- a/layerindex/context_processors.py
+++ b/layerindex/context_processors.py
@@ -1,11 +1,13 @@
 # layerindex-web - custom context processor
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013, 2018 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
-from layerindex.models import Branch, LayerItem
+from layerindex.models import Branch, LayerItem, SiteNotice
 from django.contrib.sites.models import Site
+from django.db.models import Q
+from datetime import datetime
 
 def layerindex_context(request):
     import settings
@@ -20,4 +22,5 @@ def layerindex_context(request):
         'oe_classic': Branch.objects.filter(name='oe-classic'),
         'site_name': site_name,
         'rrs_enabled': 'rrs' in settings.INSTALLED_APPS,
+        'notices': SiteNotice.objects.filter(disabled=False).filter(Q(expires__isnull=True) | Q(expires__gte=datetime.now())),
     }
diff --git a/layerindex/migrations/0014_sitenotice.py b/layerindex/migrations/0014_sitenotice.py
new file mode 100644
index 00000000..630700cf
--- /dev/null
+++ b/layerindex/migrations/0014_sitenotice.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layerindex', '0013_patch'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SiteNotice',
+            fields=[
+                ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)),
+                ('text', models.TextField(help_text='Text to show in the notice. A limited subset of HTML is supported for formatting.')),
+                ('level', models.CharField(choices=[('I', 'Info'), ('S', 'Success'), ('W', 'Warning'), ('E', 'Error')], help_text='Level of notice to display', default='I', max_length=1)),
+                ('disabled', models.BooleanField(verbose_name='Disabled', help_text='Use to temporarily disable this notice', default=False)),
+                ('expires', models.DateTimeField(blank=True, help_text='Optional date/time when this notice will stop showing', null=True)),
+            ],
+        ),
+    ]
diff --git a/layerindex/models.py b/layerindex/models.py
index ff164baa..891f5dfb 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -1,6 +1,6 @@
 # layerindex-web - model definitions
 #
-# Copyright (C) 2013-2016 Intel Corporation
+# Copyright (C) 2013-2018 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
@@ -658,3 +658,26 @@ class RecipeChange(models.Model):
     def reset_fields(self):
         for fieldname in self.RECIPE_VARIABLE_MAP:
             setattr(self, fieldname, getattr(self.recipe, fieldname))
+
+class SiteNotice(models.Model):
+    NOTICE_LEVEL_CHOICES = [
+        ('I', 'Info'),
+        ('S', 'Success'),
+        ('W', 'Warning'),
+        ('E', 'Error'),
+    ]
+    text = models.TextField(help_text='Text to show in the notice. A limited subset of HTML is supported for formatting.')
+    level = models.CharField(max_length=1, choices=NOTICE_LEVEL_CHOICES, default='I', help_text='Level of notice to display')
+    disabled = models.BooleanField('Disabled', default=False, help_text='Use to temporarily disable this notice')
+    expires = models.DateTimeField(blank=True, null=True, help_text='Optional date/time when this notice will stop showing')
+
+    def __str__(self):
+        prefix = ''
+        if self.expires and datetime.now() >= self.expires:
+            prefix = '[expired] '
+        elif self.disabled:
+            prefix = '[disabled] '
+        return '%s%s' % (prefix, self.text)
+
+    def text_sanitised(self):
+        return utils.sanitise_html(self.text)
diff --git a/layerindex/utils.py b/layerindex/utils.py
index f8c5fd45..fe994439 100644
--- a/layerindex/utils.py
+++ b/layerindex/utils.py
@@ -14,6 +14,7 @@ import time
 import fcntl
 import signal
 import codecs
+from bs4 import BeautifulSoup
 
 def get_branch(branchname):
     from layerindex.models import Branch
@@ -376,6 +377,7 @@ def setup_core_layer_sys_path(settings, branchname):
     core_layerdir = os.path.join(core_repodir, core_layerbranch.vcs_subdir)
     sys.path.insert(0, os.path.join(core_layerdir, 'lib'))
 
+
 def run_command_interruptible(cmd):
     """
     Run a command with output displayed on the console, but ensure any Ctrl+C is
@@ -404,3 +406,15 @@ def run_command_interruptible(cmd):
     finally:
         signal.signal(signal.SIGINT, signal.SIG_DFL)
     return process.returncode, buf
+
+
+def sanitise_html(html):
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup.findAll(True):
+        if tag.name not in ['strong', 'em', 'b', 'i', 'p', 'ul', 'ol', 'li', 'br', 'p']:
+            tag.hidden = True
+        elif tag.attrs:
+            tag.attrs = []
+
+    return soup.renderContents()
+
diff --git a/requirements.txt b/requirements.txt
index 5ca7cf6e..7facd2b6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
 amqp==2.2.2
 anyjson==0.3.3
+beautifulsoup4==4.6.0
 billiard==3.5.0.3
 celery==4.1.0
 confusable-homoglyphs==3.0.0
diff --git a/templates/base.html b/templates/base.html
index 38373ab8..a5e97cd5 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -89,6 +89,11 @@
     {% endblock %}
 
 {% block contenttag %}<div id="content" class="container top-padded">{% endblock %}
+        {% if notices %}
+            {% for notice in notices %}
+            <div class="alert {% if notice.level == 'I' %}alert-info{% elif notice.level == 'S' %}alert-success{% elif notice.level == 'W' %}{% elif notice.level == 'E' %}alert-error{% endif %}">{{ notice.text_sanitised|safe|urlize }}</div>
+            {% endfor %}
+        {% endif %}
         {% if messages %}
             {% for message in messages %}
             <div{% if message.tags %} class="alert {{ message.tags }}"{% endif %}>{{ message }}</div>
-- 
2.14.3

[layerindex-web][PATCH 5/6] Add a script to import layers from an existing layer index

[Paul Eggleton]

Add a script that uses the REST API to gather information from another
running layer index instance (e.g. layers.openembedded.org) and import
that into the local index. Only information for branches that are
already set up is imported, and only manually entered information -
no recipes, machines, etc.

Partially implements [YOCTO #9760].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/tools/import_layers.py | 251 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 251 insertions(+)
 create mode 100755 layerindex/tools/import_layers.py

diff --git a/layerindex/tools/import_layers.py b/layerindex/tools/import_layers.py
new file mode 100755
index 00000000..b02759ea
--- /dev/null
+++ b/layerindex/tools/import_layers.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+
+# Import layers from another layer index instance
+#
+# Copyright (C) 2018 Intel Corporation
+# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+import optparse
+import re
+import glob
+import logging
+import subprocess
+import urllib.request
+import json
+
+sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
+
+import utils
+from layerconfparse import LayerConfParse
+
+class DryRunRollbackException(Exception):
+    pass
+
+logger = utils.logger_create('LayerIndexImport')
+
+
+
+def main():
+    valid_layer_name = re.compile('[-\w]+$')
+
+    parser = optparse.OptionParser(
+        usage = """
+    %prog [options] <url>""")
+
+    parser.add_option("-n", "--dry-run",
+            help = "Don't write any data back to the database",
+            action="store_true", dest="dryrun")
+    parser.add_option("-d", "--debug",
+            help = "Enable debug output",
+            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+    parser.add_option("-q", "--quiet",
+            help = "Hide all output except error messages",
+            action="store_const", const=logging.ERROR, dest="loglevel")
+
+    options, args = parser.parse_args(sys.argv)
+
+    if len(args) < 2:
+        print("Please specify URL of the layer index")
+        sys.exit(1)
+
+    layerindex_url = args[1]
+
+    utils.setup_django()
+    import settings
+    from layerindex.models import Branch, LayerItem, LayerBranch, LayerDependency, LayerMaintainer, LayerNote
+    from django.db import transaction
+
+    logger.setLevel(options.loglevel)
+
+    fetchdir = settings.LAYER_FETCH_DIR
+    if not fetchdir:
+        logger.error("Please set LAYER_FETCH_DIR in settings.py")
+        sys.exit(1)
+
+    if not os.path.exists(fetchdir):
+        os.makedirs(fetchdir)
+
+    if not layerindex_url.endswith('/'):
+        layerindex_url += '/'
+    if not '/layerindex/api/' in layerindex_url:
+        layerindex_url += '/layerindex/api/'
+
+    rq = urllib.request.Request(layerindex_url)
+    data = urllib.request.urlopen(rq).read()
+    jsdata = json.loads(data.decode('utf-8'))
+
+    branches_url = jsdata['branches']
+    layers_url = jsdata['layerItems']
+    layerdeps_url = jsdata['layerDependencies']
+    layerbranches_url = jsdata['layerBranches']
+    layermaintainers_url = jsdata.get('layerMaintainers', None)
+    layernotes_url = jsdata.get('layerNotes', None)
+
+    logger.debug('Getting branches')
+
+    # Get branches (we assume the ones we want are already there, so skip any that aren't)
+    rq = urllib.request.Request(branches_url)
+    data = urllib.request.urlopen(rq).read()
+    jsdata = json.loads(data.decode('utf-8'))
+    branch_idmap = {}
+    for branchjs in jsdata:
+        res = Branch.objects.filter(name=branchjs['name'])
+        if res:
+            branch = res.first()
+            branch_idmap[branchjs['id']] = branch
+
+    try:
+        with transaction.atomic():
+            # Get layers
+            logger.debug('Importing layers')
+            rq = urllib.request.Request(layers_url)
+            data = urllib.request.urlopen(rq).read()
+            jsdata = json.loads(data.decode('utf-8'))
+
+            layer_idmap = {}
+            exclude_fields = ['id', 'updated']
+            for layerjs in jsdata:
+                res = LayerItem.objects.filter(name=layerjs['name'])
+                if res:
+                    # Already have this layer
+                    logger.debug('Skipping layer %s, already in database' % layerjs['name'])
+                    layer_idmap[layerjs['id']] = res[0]
+                    continue
+                logger.debug('Adding layer %s' % layerjs['name'])
+                layeritem = LayerItem()
+                for key, value in layerjs.items():
+                    if key in exclude_fields:
+                        continue
+                    setattr(layeritem, key, value)
+                layeritem.save()
+                layer_idmap[layerjs['id']] = layeritem
+
+            # Get layer branches
+            logger.debug('Importing layer branches')
+            rq = urllib.request.Request(layerbranches_url)
+            data = urllib.request.urlopen(rq).read()
+            jsdata = json.loads(data.decode('utf-8'))
+
+            layerbranch_idmap = {}
+            exclude_fields = ['id', 'layer', 'branch', 'vcs_last_fetch', 'vcs_last_rev', 'vcs_last_commit', 'yp_compatible_version', 'updated']
+            for layerbranchjs in jsdata:
+                branch = branch_idmap.get(layerbranchjs['branch'], None)
+                if not branch:
+                    # We don't have this branch, skip it
+                    logger.debug('Skipping layerbranch %s, branch not imported' % layerbranchjs['id'])
+                    continue
+                layer = layer_idmap.get(layerbranchjs['layer'], None)
+                if not layer:
+                    # We didn't import this layer, skip it
+                    logger.debug('Skipping layerbranch %s, layer not imported' % layerbranchjs['id'])
+                    continue
+                res = LayerBranch.objects.filter(layer=layer).filter(branch=branch)
+                if res:
+                    # The layerbranch already exists (this will occur for layers
+                    # that already existed, since we need to have those in layer_idmap
+                    # to be able to import layer dependencies)
+                    logger.debug('Skipping layerbranch %s, already exists' % layerbranchjs['id'])
+                    continue
+
+                layerbranch = LayerBranch()
+                for key, value in layerbranchjs.items():
+                    if key in exclude_fields:
+                        continue
+                    setattr(layerbranch, key, value)
+                layerbranch.branch = branch
+                layerbranch.layer = layer
+                layerbranch.save()
+                layerbranch_idmap[layerbranchjs['id']] = layerbranch
+
+            # Get layer dependencies
+            logger.debug('Importing layer dependencies')
+            rq = urllib.request.Request(layerdeps_url)
+            data = urllib.request.urlopen(rq).read()
+            jsdata = json.loads(data.decode('utf-8'))
+
+            exclude_fields = ['id', 'layerbranch', 'dependency', 'updated']
+            for layerdepjs in jsdata:
+                layerbranch = layerbranch_idmap.get(layerdepjs['layerbranch'], None)
+                if not layerbranch:
+                    # We didn't import this layerbranch, skip it
+                    continue
+                dependency = layer_idmap.get(layerdepjs['dependency'], None)
+                if not dependency:
+                    # We didn't import the dependency, skip it
+                    continue
+
+                layerdep = LayerDependency()
+                for key, value in layerdepjs.items():
+                    if key in exclude_fields:
+                        continue
+                    setattr(layerdep, key, value)
+                layerdep.layerbranch = layerbranch
+                layerdep.dependency = dependency
+                layerdep.save()
+
+            if layermaintainers_url:
+                # Get layer maintainers (only available in latest code)
+                logger.debug('Importing layer maintainers')
+                rq = urllib.request.Request(layermaintainers_url)
+                data = urllib.request.urlopen(rq).read()
+                jsdata = json.loads(data.decode('utf-8'))
+
+                exclude_fields = ['id', 'layerbranch']
+                for layermaintainerjs in jsdata:
+                    layerbranch = layerbranch_idmap.get(layermaintainerjs['layerbranch'], None)
+                    if not layerbranch:
+                        # We didn't import this layerbranch, skip it
+                        continue
+
+                    layermaintainer = LayerMaintainer()
+                    for key, value in layermaintainerjs.items():
+                        if key in exclude_fields:
+                            continue
+                        setattr(layermaintainer, key, value)
+                    layermaintainer.layerbranch = layerbranch
+                    layermaintainer.save()
+
+            if layernotes_url:
+                # Get layer notes (only available in latest code)
+                logger.debug('Importing layer notes')
+                rq = urllib.request.Request(layernotes_url)
+                data = urllib.request.urlopen(rq).read()
+                jsdata = json.loads(data.decode('utf-8'))
+
+                exclude_fields = ['id', 'layer']
+                for layernotejs in jsdata:
+                    layer = layer_idmap.get(layernotejs['layer'], None)
+                    if not layer:
+                        # We didn't import this layer, skip it
+                        continue
+                    res = LayerNote.objects.filter(layer=layer).filter(text=layernotejs['text'])
+                    if res:
+                        # The note already exists (this will occur for layers
+                        # that already existed, since we need to have those in layer_idmap
+                        # to be able to import layer dependencies)
+                        logger.debug('Skipping note %s, already exists' % layernotejs['id'])
+                        continue
+
+                    layernote = LayerNote()
+                    for key, value in layernotejs.items():
+                        if key in exclude_fields:
+                            continue
+                        setattr(layernote, key, value)
+                    layernote.layer = layer
+                    layernote.save()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
+    except DryRunRollbackException:
+        pass
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()
-- 
2.14.3

[layerindex-web][PATCH 6/6] update_classic_status: fix matching on wrong layer

[Paul Eggleton]

We were sorting the layers in the wrong order when trying to choose
which layer to pick a matching recipe from - it needs to be descending
priority order, not ascending.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/tools/update_classic_status.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/layerindex/tools/update_classic_status.py b/layerindex/tools/update_classic_status.py
index f980e64a..be7807d2 100755
--- a/layerindex/tools/update_classic_status.py
+++ b/layerindex/tools/update_classic_status.py
@@ -68,7 +68,7 @@ def main():
     try:
         with transaction.atomic():
             def recipe_pn_query(pn):
-                return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference')
+                return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('-layerbranch__layer__index_preference')
 
             recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N'])
             for recipe in recipequery:
-- 
2.14.3