* [layerindex-web][PATCH 1/3] Track SRCREV for each recipe
2019-10-20 22:49 [layerindex-web][PATCH 0/3] Branch comparison functionality Paul Eggleton
@ 2019-10-20 22:49 ` Paul Eggleton
2019-10-20 22:49 ` [layerindex-web][PATCH 2/3] Add branch comparison function Paul Eggleton
2019-10-20 22:49 ` [layerindex-web][PATCH 3/3] Fix cgit commit URL setting Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2019-10-20 22:49 UTC (permalink / raw)
To: yocto
For the purposes of the branch comparison function I'm about to add it
would be useful to track the value of SRCREV, so we can see if it has
changed even if PV hasn't.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/migrations/0043_recipe_srcrev.py | 20 ++++++++++++++++++++
layerindex/models.py | 1 +
layerindex/update_layer.py | 4 ++++
templates/layerindex/recipedetail.html | 2 +-
4 files changed, 26 insertions(+), 1 deletion(-)
create mode 100644 layerindex/migrations/0043_recipe_srcrev.py
diff --git a/layerindex/migrations/0043_recipe_srcrev.py b/layerindex/migrations/0043_recipe_srcrev.py
new file mode 100644
index 00000000..64293767
--- /dev/null
+++ b/layerindex/migrations/0043_recipe_srcrev.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-10-20 22:15
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('layerindex', '0042_recipe_pe_pr'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='recipe',
+ name='srcrev',
+ field=models.CharField(blank=True, max_length=64),
+ ),
+ ]
diff --git a/layerindex/models.py b/layerindex/models.py
index 332ba39e..253c725c 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -475,6 +475,7 @@ class Recipe(models.Model):
updated = models.DateTimeField(auto_now=True)
blacklisted = models.CharField(max_length=255, blank=True)
configopts = models.CharField(max_length=4096, blank=True)
+ srcrev = models.CharField(max_length=64, blank=True)
def vcs_web_url(self):
url = self.layerbranch.file_url(os.path.join(self.filepath, self.filename))
diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index fcae54f8..17098379 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -110,6 +110,10 @@ def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir, sto
recipe.pv = envdata.getVar("PV", True)
recipe.pr = envdata.getVar("PR", True) or ""
recipe.pe = envdata.getVar("PE", True) or ""
+ recipe.srcrev = envdata.getVar('SRCREV', True) or ''
+ if recipe.srcrev == 'INVALID':
+ # INVALID is the default from bitbake.conf, but we don't want to see it
+ recipe.srcrev = ''
recipe.summary = envdata.getVar("SUMMARY", True)
recipe.description = envdata.getVar("DESCRIPTION", True)
recipe.section = envdata.getVar("SECTION", True)
diff --git a/templates/layerindex/recipedetail.html b/templates/layerindex/recipedetail.html
index b3ba65eb..64463863 100644
--- a/templates/layerindex/recipedetail.html
+++ b/templates/layerindex/recipedetail.html
@@ -54,7 +54,7 @@
</tr>
<tr>
<th>Version</th>
- <td>{{ recipe.pv }}</td>
+ <td>{{ recipe.pv }}{% if recipe.srcrev %} ({{ recipe.srcrev }}){% endif %}</td>
</tr>
<tr>
<th>Summary</th>
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [layerindex-web][PATCH 2/3] Add branch comparison function
2019-10-20 22:49 [layerindex-web][PATCH 0/3] Branch comparison functionality Paul Eggleton
2019-10-20 22:49 ` [layerindex-web][PATCH 1/3] Track SRCREV for each recipe Paul Eggleton
@ 2019-10-20 22:49 ` Paul Eggleton
2019-10-20 22:49 ` [layerindex-web][PATCH 3/3] Fix cgit commit URL setting Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2019-10-20 22:49 UTC (permalink / raw)
To: yocto
Add the ability to compare available recipes and their versions between
two branches for a selection of layers (default is just OE-Core). This
was mainly intended to help us with the Yocto Project release notes
preparation (hence the "Plain text" button at the bottom of the page)
but is also useful in its own right.
Note: for readability, SRCREVs are only shown when PV has not changed.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/forms.py | 19 ++
layerindex/urls.py | 12 +-
layerindex/views.py | 113 +++++++++-
templates/base.html | 1 +
templates/layerindex/branchcompare.html | 214 +++++++++++++++++++
templates/layerindex/branchcompare_plain.txt | 17 ++
6 files changed, 374 insertions(+), 2 deletions(-)
create mode 100644 templates/layerindex/branchcompare.html
create mode 100644 templates/layerindex/branchcompare_plain.txt
diff --git a/layerindex/forms.py b/layerindex/forms.py
index 13ba3aad..51583c60 100644
--- a/layerindex/forms.py
+++ b/layerindex/forms.py
@@ -354,3 +354,22 @@ class PatchDispositionForm(StyledModelForm):
}
PatchDispositionFormSet = modelformset_factory(PatchDisposition, form=PatchDispositionForm, extra=0)
+
+
+class BranchComparisonForm(StyledForm):
+ from_branch = forms.ModelChoiceField(label='From', queryset=Branch.objects.none())
+ to_branch = forms.ModelChoiceField(label='To', queryset=Branch.objects.none())
+ layers = forms.CharField(widget=forms.HiddenInput())
+
+ def __init__(self, *args, request=None, **kwargs):
+ super(BranchComparisonForm, self).__init__(*args, **kwargs)
+ qs = Branch.objects.filter(comparison=False, hidden=False).order_by('sort_priority', 'name')
+ self.fields['from_branch'].queryset = qs
+ self.fields['to_branch'].queryset = qs
+ self.request = request
+
+ def clean(self):
+ cleaned_data = super(BranchComparisonForm, self).clean()
+ if cleaned_data['from_branch'] == cleaned_data['to_branch']:
+ raise forms.ValidationError({'to_branch': 'From and to branches cannot be the same'})
+ return cleaned_data
diff --git a/layerindex/urls.py b/layerindex/urls.py
index 89e70a22..abeb0928 100644
--- a/layerindex/urls.py
+++ b/layerindex/urls.py
@@ -14,7 +14,8 @@ from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDeta
bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, \
ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, \
UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView, TaskStatusView, \
- ComparisonRecipeSelectView, ComparisonRecipeSelectDetailView, task_log_view, task_stop_view, email_test_view
+ ComparisonRecipeSelectView, ComparisonRecipeSelectDetailView, task_log_view, task_stop_view, email_test_view, \
+ BranchCompareView
from layerindex.models import LayerItem, Recipe, RecipeChangeset
from rest_framework import routers
from . import restviews
@@ -185,6 +186,15 @@ urlpatterns = [
url(r'^stoptask/(?P<task_id>[-\w]+)/$',
task_stop_view,
name='task_stop'),
+ url(r'^branch_comparison/$',
+ BranchCompareView.as_view(
+ template_name='layerindex/branchcompare.html'),
+ name='branch_comparison'),
+ url(r'^branch_comparison_plain/$',
+ BranchCompareView.as_view(
+ content_type='text/plain',
+ template_name='layerindex/branchcompare_plain.txt'),
+ name='branch_comparison_plain'),
url(r'^ajax/layerchecklist/(?P<branch>[-.\w]+)/$',
LayerCheckListView.as_view(
template_name='layerindex/layerchecklist.html'),
diff --git a/layerindex/views.py b/layerindex/views.py
index 2dacf516..12054fe7 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -47,7 +47,8 @@ from layerindex.forms import (AdvancedRecipeSearchForm, BulkChangeEditFormSet,
ComparisonRecipeSelectForm, EditLayerForm,
EditNoteForm, EditProfileForm,
LayerMaintainerFormSet, RecipeChangesetForm,
- PatchDispositionForm, PatchDispositionFormSet)
+ PatchDispositionForm, PatchDispositionFormSet,
+ BranchComparisonForm)
from layerindex.models import (BBAppend, BBClass, Branch, ClassicRecipe,
Distro, DynamicBuildDep, IncFile, LayerBranch,
LayerDependency, LayerItem, LayerMaintainer,
@@ -1705,3 +1706,113 @@ class ComparisonRecipeSelectDetailView(DetailView):
messages.error(request, 'Failed to save changes: %s' % form.errors)
return self.get(request, *args, **kwargs)
+
+
+class BranchCompareView(FormView):
+ form_class = BranchComparisonForm
+
+ def get_recipes(self, from_branch, to_branch, layer_ids):
+ from distutils.version import LooseVersion
+ class BranchComparisonResult:
+ def __init__(self, pn, short_desc):
+ self.pn = pn
+ self.short_desc = short_desc
+ self.from_versions = []
+ self.to_versions = []
+ self.id = None
+ def pv_changed(self):
+ from_pvs = sorted([x.pv for x in self.from_versions])
+ to_pvs = sorted([x.pv for x in self.to_versions])
+ return (from_pvs != to_pvs)
+ class BranchComparisonVersionResult:
+ def __init__(self, id, pv, srcrev):
+ self.id = id
+ self.pv = pv
+ self.srcrev = srcrev
+ def version_expr(self):
+ return (self.pv, self.srcrev)
+
+ def map_name(recipe):
+ pn = recipe.pn
+ if pn.startswith('gcc-source-'):
+ pn = pn.replace('-%s' % recipe.pv, '')
+ elif pn.endswith(('-i586', '-i686')):
+ pn = pn[:-5]
+ elif pn.endswith('-x86_64-oesdk-linux'):
+ pn = pn[:-19]
+ return pn
+
+ from_recipes = Recipe.objects.filter(layerbranch__branch=from_branch)
+ to_recipes = Recipe.objects.filter(layerbranch__branch=to_branch)
+ if layer_ids:
+ from_recipes = from_recipes.filter(layerbranch__layer__in=layer_ids)
+ to_recipes = to_recipes.filter(layerbranch__layer__in=layer_ids)
+ recipes = {}
+ for recipe in from_recipes:
+ pn = map_name(recipe)
+ res = recipes.get(pn, None)
+ if not res:
+ res = BranchComparisonResult(pn, recipe.short_desc)
+ recipes[pn] = res
+ res.from_versions.append(BranchComparisonVersionResult(id=recipe.id, pv=recipe.pv, srcrev=recipe.srcrev))
+ for recipe in to_recipes:
+ pn = map_name(recipe)
+ res = recipes.get(pn, None)
+ if not res:
+ res = BranchComparisonResult(pn, recipe.short_desc)
+ recipes[pn] = res
+ res.to_versions.append(BranchComparisonVersionResult(id=recipe.id, pv=recipe.pv, srcrev=recipe.srcrev))
+
+ added = []
+ changed = []
+ removed = []
+ for _, recipe in sorted(recipes.items(), key=lambda item: item[0]):
+ recipe.from_versions = sorted(recipe.from_versions, key=lambda item: LooseVersion(item.pv))
+ from_version_exprs = [x.version_expr() for x in recipe.from_versions]
+ recipe.to_versions = sorted(recipe.to_versions, key=lambda item: LooseVersion(item.pv))
+ to_version_exprs = [x.version_expr() for x in recipe.to_versions]
+ if not from_version_exprs:
+ added.append(recipe)
+ elif not to_version_exprs:
+ recipe.id = recipe.from_versions[-1].id
+ removed.append(recipe)
+ elif from_version_exprs != to_version_exprs:
+ changed.append(recipe)
+ return added, changed, removed
+
+ def form_valid(self, form):
+ return HttpResponseRedirect(reverse_lazy('branch_comparison', args=(form.cleaned_data['from_branch'].name, form.cleaned_data['to_branch'].name)))
+
+ def get_initial(self):
+ initial = super(BranchCompareView, self).get_initial()
+ from_branch_id = self.request.GET.get('from_branch', None)
+ if from_branch_id is not None:
+ initial['from_branch'] = get_object_or_404(Branch, id=from_branch_id)
+ to_branch_id = self.request.GET.get('to_branch', None)
+ if to_branch_id is not None:
+ initial['to_branch'] = get_object_or_404(Branch, id=to_branch_id)
+ initial['layers'] = self.request.GET.get('layers', str(LayerItem.objects.get(name=settings.CORE_LAYER_NAME).id))
+ return initial
+
+ def get_context_data(self, **kwargs):
+ context = super(BranchCompareView, self).get_context_data(**kwargs)
+ from_branch_id = self.request.GET.get('from_branch', None)
+ to_branch_id = self.request.GET.get('to_branch', None)
+
+ layer_ids = self.request.GET.get('layers', self.request.GET.get('layers', str(LayerItem.objects.get(name=settings.CORE_LAYER_NAME).id)))
+ from_branch = None
+ if from_branch_id is not None:
+ from_branch = get_object_or_404(Branch, id=from_branch_id)
+ context['from_branch'] = from_branch
+ to_branch = None
+ if from_branch_id is not None:
+ to_branch = get_object_or_404(Branch, id=to_branch_id)
+ context['to_branch'] = to_branch
+ if from_branch and to_branch:
+ context['added'], context['changed'], context['removed'] = self.get_recipes(from_branch, to_branch, layer_ids)
+ context['this_url_name'] = resolve(self.request.path_info).url_name
+ context['layers'] = LayerItem.objects.filter(status__in=['P', 'X']).order_by('name')
+ context['showlayers'] = layer_ids
+
+ return context
+
diff --git a/templates/base.html b/templates/base.html
index ae1ad01c..126784d1 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -87,6 +87,7 @@
<li><a href="{% url 'duplicates' 'master' %}">Duplicates</a></li>
<li><a href="{% url 'update_list' %}">Updates</a></li>
<li><a href="{% url 'stats' %}">Statistics</a></li>
+ <li><a href="{% url 'branch_comparison' %}">Branch Comparison</a></li>
{% if rrs_enabled %}
<li><a href="{% url 'rrs_frontpage' %}">Recipe Maintenance</a></li>
{% endif %}
diff --git a/templates/layerindex/branchcompare.html b/templates/layerindex/branchcompare.html
new file mode 100644
index 00000000..56b23109
--- /dev/null
+++ b/templates/layerindex/branchcompare.html
@@ -0,0 +1,214 @@
+{% extends "base.html" %}
+{% load i18n %}
+{% load static %}
+
+{% comment %}
+
+ layerindex-web - branch comparison page template
+
+ Copyright (C) 2019 Intel Corporation
+ Licensed under the MIT license, see COPYING.MIT for details
+
+{% endcomment %}
+
+
+<!--
+{% block title_append %} - branch comparison{% endblock %}
+-->
+
+{% block content %}
+{% autoescape on %}
+
+ <div class="row">
+ <div class="col-md-12">
+
+ <div class="pull-right">
+ <form class="form-inline" method="GET">
+ {{ form }}
+
+ <div id="layerDialog" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="layerDialogLabel">
+ <div class="modal-dialog" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
+ <h3 id="layerDialogLabel">Select layers to include</h3>
+ </div>
+ <div class="modal-body">
+ <div class="form-group has-feedback has-clear">
+ <input type="text" class="form-control" id="layersearchtext" placeholder="search layers">
+ <a class="glyphicon glyphicon-remove-sign form-control-feedback form-control-clear" id="layersearchclear" style="pointer-events: auto; text-decoration: none;cursor: pointer;"></a>
+ </div>
+ <div class="scrolling">
+ <table class="layerstable"><tbody>
+ {% for layer in layers %}
+ <tr>
+ <td class="checkboxtd"><input
+ type="checkbox"
+ class="filterlayercheckbox"
+ name="l"
+ value="{{ layer.id }}" id="id_layercheckbox_{{layer.id}}"
+ {% if showlayers and layer.id in showlayers %}
+ checked
+ {% endif %}
+ />
+ </td>
+ <td><label for="id_layercheckbox_{{layer.id}}">{{ layer.name }}</label></td>
+ </tr>
+ {% endfor %}
+ </tbody></table>
+ </div>
+ <div class="buttonblock">
+ <button type="button" class="btn btn-default buttonblock-btn" id="id_select_none">Clear selections</button>
+ </div>
+ </div>
+ <div class="modal-footer">
+ <button type="button" class="btn btn-primary" id="id_layerdialog_ok" data-dismiss="modal">Filter</button>
+ <button type="button" class="btn btn-default" id="id_cancel" data-dismiss="modal">Cancel</button>
+ </div>
+ </div><!-- /.modal-content -->
+ </div><!-- /.modal-dialog -->
+ </div>
+
+ <a href="#layerDialog" role="button" id="id_select_layers" class="btn btn-default nav-spacer" data-toggle="modal">Filter layers <span class="badge badge-success" id="id_layers_count">{{ showlayers|length }}</span></a>
+
+ <button type="submit" class="btn btn-primary">Show</button>
+ </form>
+ </div>
+
+ <h2>Branch recipe comparison</h2>
+{% if added or changed or removed %}
+ <h3>Added</h3>
+ <table class="table table-striped table-bordered recipestable">
+ <thead>
+ <tr>
+ <th>Recipe</th>
+ <th>Description</th>
+ <th>Version - {{ to_branch }}</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {% for recipe in added %}
+ <tr>
+ <td class="success">{{ recipe.pn }}</td>
+ <td class="success">{{ recipe.short_desc }}</td>
+ <td class="success">{% for rv in recipe.to_versions %}<a href="{% url 'recipe' rv.id %}">{{ rv.pv }}{% if not forloop.last %}, {% endif %}</a>{% endfor %}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+
+ <h3>Changed</h3>
+ <table class="table table-striped table-bordered recipestable">
+ <thead>
+ <tr>
+ <th>Recipe</th>
+ <th>Description</th>
+ <th>Version - {{ from_branch }}</th>
+ <th>Version - {{ to_branch }}</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {% for recipe in changed %}
+ {% with pv_changed=recipe.pv_changed %}
+ <tr>
+ <td>{{ recipe.pn }}</td>
+ <td>{{ recipe.short_desc }}</td>
+ <td>{% for rv in recipe.from_versions %}<a href="{% url 'recipe' rv.id %}">{{ rv.pv }}{% if rv.srcrev and not pv_changed %} ({{ rv.srcrev|truncatechars:13 }}){% endif %}{% if not forloop.last %}, {% endif %}</a>{% endfor %}</td>
+ <td>{% for rv in recipe.to_versions %}<a href="{% url 'recipe' rv.id %}">{{ rv.pv }}{% if rv.srcrev and not pv_changed %} ({{ rv.srcrev|truncatechars:13 }}){% endif %}{% if not forloop.last %}, {% endif %}</a>{% endfor %}</td>
+ </tr>
+ {% endwith %}
+ {% endfor %}
+ </tbody>
+ </table>
+
+ <h3>Removed</h3>
+ <table class="table table-striped table-bordered recipestable">
+ <thead>
+ <tr>
+ <th>Recipe</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {% for recipe in removed %}
+ <tr>
+ <td class="error"><a href="{% url 'recipe' recipe.id %}">{{ recipe.pn }}</a></td>
+ <td class="error">{{ recipe.short_desc }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+{% elif from_branch and to_branch %}
+ <p>No matching recipes in database.</p>
+{% else %}
+ <p>Select some parameters above to begin comparison.</p>
+{% endif %}
+ </div>
+ </div>
+
+ <span class="pull-right">
+ <a class="btn btn-default" href="{% url 'branch_comparison_plain' %}?{{ request.GET.urlencode }}"><i class="glyphicon glyphicon-file"></i> Plain text</a>
+ </span>
+
+
+{% endautoescape %}
+
+{% endblock %}
+
+
+{% block scripts %}
+<script>
+ $(document).ready(function() {
+ firstfield = $("#filter-form input:text").first()
+ if( ! firstfield.val() )
+ firstfield.focus()
+ });
+ $('#id_select_none').click(function (e) {
+ $('.layerstable').find('tr:visible').find('.filterlayercheckbox').prop('checked', false);
+ });
+
+ function clearLayerSearch() {
+ $("#layersearchtext").val('');
+ $(".layerstable > tbody > tr").show();
+ }
+
+ update_selected_layer_display = function() {
+ //layernames = [];
+ layerids = [];
+ $('.filterlayercheckbox:checked').each(function() {
+ //layernames.push($("label[for="+$(this).attr('id')+"]").html());
+ layerids.push($(this).attr('value'))
+ });
+ $('#id_layers').val(layerids)
+ $('#id_layers_count').html(layerids.length)
+ }
+ select_layer_checkboxes = function() {
+ $('.filterlayercheckbox').prop('checked', false);
+ selectedlayers = $('#id_layers').val().split(',');
+ for(i in selectedlayers) {
+ $('#id_layercheckbox_' + selectedlayers[i]).prop('checked', true);
+ }
+ }
+
+ $('#id_layerdialog_ok').click(function (e) {
+ update_selected_layer_display()
+ });
+ $("#layersearchtext").on("input", function() {
+ var value = $(this).val().toLowerCase();
+ $(".layerstable > tbody > tr").filter(function() {
+ $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)
+ });
+ });
+ $("#layersearchclear").click(function(){
+ clearLayerSearch();
+ $("#layersearchtext").focus();
+ });
+ $('#id_select_layers').click(function (e) {
+ clearLayerSearch();
+ select_layer_checkboxes();
+ })
+</script>
+{% endblock %}
diff --git a/templates/layerindex/branchcompare_plain.txt b/templates/layerindex/branchcompare_plain.txt
new file mode 100644
index 00000000..91bfe192
--- /dev/null
+++ b/templates/layerindex/branchcompare_plain.txt
@@ -0,0 +1,17 @@
+From {{ from_branch }} to {{ to_branch }}
+
+
+Added
+-----
+{% for recipe in added %}{{ recipe.pn }} {% for rv in recipe.to_versions %}{{ rv.pv }}{% if not forloop.last %}, {% endif %}{% endfor %}
+{% endfor %}
+
+Changed
+-------
+{% for recipe in changed %}{% with pv_changed=recipe.pv_changed %}{{ recipe.pn }} {% for rv in recipe.from_versions %}{{ rv.pv }}{% if rv.srcrev and not pv_changed %} ({{ rv.srcrev|truncatechars:13 }}){% endif %}{% if not forloop.last %}, {% endif %}{% endfor %} -> {% for rv in recipe.to_versions %}{{ rv.pv }}{% if rv.srcrev and not pv_changed %} ({{ rv.srcrev|truncatechars:13 }}){% endif %}{% if not forloop.last %}, {% endif %}{% endfor %}
+{% endwith %}{% endfor %}
+
+Removed
+-------
+{% for recipe in removed %}{{ recipe.pn }}
+{% endfor %}
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread