* [PATCH v2 1/9] toaster: loadconf remove Loading LayerSources
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 2/9] toaster: bldcollector admin Remove LayerSourceAdmin Michael Wood
` (7 subsequent siblings)
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
We don't need to configure layer sources in the initial configuration as
this information is provided by the models.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
.../bldcontrol/management/commands/loadconf.py | 69 +---------------------
1 file changed, 2 insertions(+), 67 deletions(-)
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
index 173e3ce..6f08f1d 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -1,6 +1,6 @@
from django.core.management.base import BaseCommand, CommandError
from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
-from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority
+from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer
from django.db import IntegrityError
import os
@@ -42,7 +42,7 @@ class Command(BaseCommand):
data = json.loads(open(filepath, "r").read())
# verify config file validity before updating settings
- for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
+ for i in ['bitbake', 'releases', 'defaultrelease', 'config']:
assert i in data
def _read_git_url_from_local_repository(address):
@@ -81,65 +81,6 @@ class Command(BaseCommand):
bvo.dirpath = bvi['dirpath']
bvo.save()
- # set the layer sources
- for lsi in data['layersources']:
- assert 'sourcetype' in lsi
- assert 'apiurl' in lsi
- assert 'name' in lsi
- assert 'branches' in lsi
-
-
- if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
- apiurl = lsi['apiurl']
- else:
- apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl']))
-
- assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl)
-
- try:
- ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl)
- ls.name = lsi['name']
- ls.save()
- except IntegrityError as e:
- logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls))
-
-
- layerbranches = []
- for branchname in lsi['branches']:
- bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname)
- layerbranches.append(bo)
-
- if 'layers' in lsi:
- for layerinfo in lsi['layers']:
- lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name'])
- if layerinfo['local_path'].startswith("/"):
- lo.local_path = layerinfo['local_path']
- else:
- lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path']))
-
- if not os.path.exists(lo.local_path):
- logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
-
- if layerinfo['vcs_url'].startswith("remote:"):
- lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
- if lo.vcs_url is None:
- logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
-
- if lo.vcs_url is None:
- lo.vcs_url = layerinfo['vcs_url']
-
- if 'layer_index_url' in layerinfo:
- lo.layer_index_url = layerinfo['layer_index_url']
- lo.save()
-
- for branch in layerbranches:
- lvo, created = Layer_Version.objects.get_or_create(layer_source = ls,
- up_branch = branch,
- commit = branch.name,
- layer = lo)
- lvo.dirpath = layerinfo['dirpath']
- lvo.save()
- # set releases
for ri in data['releases']:
bvo = BitbakeVersion.objects.get(name = ri['bitbake'])
assert bvo is not None
@@ -149,12 +90,6 @@ class Command(BaseCommand):
ro.helptext = ri['helptext']
ro.save()
- # save layer source priority for release
- for ls_name in ri['layersourcepriority'].keys():
- rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name))
- rlspo.priority = ri['layersourcepriority'][ls_name]
- rlspo.save()
-
for dli in ri['defaultlayers']:
# find layers with the same name
ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli)
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 2/9] toaster: bldcollector admin Remove LayerSourceAdmin
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
2016-07-07 17:21 ` [PATCH v2 1/9] toaster: loadconf remove Loading LayerSources Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 3/9] toaster: models Remove LayerSource models and replace with enum Michael Wood
` (6 subsequent siblings)
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
Remove the LayerSource admin from django admin interface. LayerSources
are not going to be manageable from the admin interface.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
bitbake/lib/toaster/bldcollector/admin.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/bitbake/lib/toaster/bldcollector/admin.py b/bitbake/lib/toaster/bldcollector/admin.py
index c1f85d7..7b69ecd 100644
--- a/bitbake/lib/toaster/bldcollector/admin.py
+++ b/bitbake/lib/toaster/bldcollector/admin.py
@@ -1,15 +1,12 @@
from django.contrib import admin
from django.contrib.admin.filters import RelatedFieldListFilter
-from orm.models import BitbakeVersion, Release, LayerSource, ToasterSetting
+from orm.models import BitbakeVersion, Release, ToasterSetting
from django.forms.widgets import Textarea
from django import forms
import django.db.models as models
from django.contrib.admin import widgets, helpers
-class LayerSourceAdmin(admin.ModelAdmin):
- pass
-
class BitbakeVersionAdmin(admin.ModelAdmin):
# we override the formfield for db URLField because of broken URL validation
@@ -27,7 +24,6 @@ class ReleaseAdmin(admin.ModelAdmin):
class ToasterSettingAdmin(admin.ModelAdmin):
pass
-admin.site.register(LayerSource, LayerSourceAdmin)
admin.site.register(BitbakeVersion, BitbakeVersionAdmin)
admin.site.register(Release, ReleaseAdmin)
admin.site.register(ToasterSetting, ToasterSettingAdmin)
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 3/9] toaster: models Remove LayerSource models and replace with enum
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
2016-07-07 17:21 ` [PATCH v2 1/9] toaster: loadconf remove Loading LayerSources Michael Wood
2016-07-07 17:21 ` [PATCH v2 2/9] toaster: bldcollector admin Remove LayerSourceAdmin Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 4/9] toaster: lsupdates Add layerindex fetcher Michael Wood
` (5 subsequent siblings)
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
We had a lot of complexity around different layer sources to allow for
multiple sources and different priorities for these source. This was
implemented using rigged abstract classes which represented the
different layer sources when in fact just an enum/flag on the
layer_version object is sufficient for our needs.
Remove the LayerSourcePriority object as this is not needed. We no longer
have a problem of multiple layers coming from multiple sources so this
is not needed. Two migrations are added to first remove the child models
which represented layersources. Then a second migration is needed to
remove the LayerSource model it's self as Django can't understand the
non-standard base class dependency. Triggering this issue:
https://docs.djangoproject.com/en/1.8/topics/migrations/#dependencies
Clean up a number of flake8 warnings in classes which were modified.
[YOCTO #9853]
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
.../0008_delete_layer_source_references.py | 118 +++++
.../orm/migrations/0009_delete_layersource.py | 17 +
bitbake/lib/toaster/orm/models.py | 481 +++++----------------
3 files changed, 238 insertions(+), 378 deletions(-)
create mode 100644 bitbake/lib/toaster/orm/migrations/0008_delete_layer_source_references.py
create mode 100644 bitbake/lib/toaster/orm/migrations/0009_delete_layersource.py
diff --git a/bitbake/lib/toaster/orm/migrations/0008_delete_layer_source_references.py b/bitbake/lib/toaster/orm/migrations/0008_delete_layer_source_references.py
new file mode 100644
index 0000000..f2c0904
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0008_delete_layer_source_references.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0007_auto_20160523_1446'),
+ ]
+
+ operations = [
+ migrations.AlterUniqueTogether(
+ name='releaselayersourcepriority',
+ unique_together=set([]),
+ ),
+ migrations.RemoveField(
+ model_name='releaselayersourcepriority',
+ name='layer_source',
+ ),
+ migrations.RemoveField(
+ model_name='releaselayersourcepriority',
+ name='release',
+ ),
+ migrations.DeleteModel(
+ name='ImportedLayerSource',
+ ),
+ migrations.DeleteModel(
+ name='LayerIndexLayerSource',
+ ),
+ migrations.DeleteModel(
+ name='LocalLayerSource',
+ ),
+ migrations.RemoveField(
+ model_name='recipe',
+ name='layer_source',
+ ),
+ migrations.RemoveField(
+ model_name='recipe',
+ name='up_id',
+ ),
+ migrations.AlterField(
+ model_name='layer',
+ name='up_date',
+ field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+ ),
+ migrations.AlterField(
+ model_name='layer_version',
+ name='layer_source',
+ field=models.IntegerField(default=0, choices=[(0, 'local'), (1, 'layerindex'), (2, 'imported'), (3, 'build')]),
+ ),
+ migrations.AlterField(
+ model_name='layer_version',
+ name='up_date',
+ field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+ ),
+ migrations.AlterUniqueTogether(
+ name='branch',
+ unique_together=set([]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='layer',
+ unique_together=set([]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='layer_version',
+ unique_together=set([]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='layerversiondependency',
+ unique_together=set([]),
+ ),
+ migrations.AlterUniqueTogether(
+ name='machine',
+ unique_together=set([]),
+ ),
+ migrations.DeleteModel(
+ name='ReleaseLayerSourcePriority',
+ ),
+ migrations.RemoveField(
+ model_name='branch',
+ name='layer_source',
+ ),
+ migrations.RemoveField(
+ model_name='branch',
+ name='up_id',
+ ),
+ migrations.RemoveField(
+ model_name='layer',
+ name='layer_source',
+ ),
+ migrations.RemoveField(
+ model_name='layer',
+ name='up_id',
+ ),
+ migrations.RemoveField(
+ model_name='layer_version',
+ name='up_id',
+ ),
+ migrations.RemoveField(
+ model_name='layerversiondependency',
+ name='layer_source',
+ ),
+ migrations.RemoveField(
+ model_name='layerversiondependency',
+ name='up_id',
+ ),
+ migrations.RemoveField(
+ model_name='machine',
+ name='layer_source',
+ ),
+ migrations.RemoveField(
+ model_name='machine',
+ name='up_id',
+ ),
+ ]
diff --git a/bitbake/lib/toaster/orm/migrations/0009_delete_layersource.py b/bitbake/lib/toaster/orm/migrations/0009_delete_layersource.py
new file mode 100644
index 0000000..41645cb
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0009_delete_layersource.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0008_delete_layer_source_references'),
+ ]
+
+ operations = [
+ migrations.DeleteModel(
+ name='LayerSource',
+ ),
+ ]
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index 61737c7..8fe4c20 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -22,7 +22,7 @@
from __future__ import unicode_literals
from django.db import models, IntegrityError
-from django.db.models import F, Q, Avg, Max, Sum
+from django.db.models import F, Q, Sum
from django.utils import timezone
from django.utils.encoding import force_bytes
@@ -117,39 +117,48 @@ class ToasterSetting(models.Model):
def __unicode__(self):
return "Setting %s = %s" % (self.name, self.value)
+
class ProjectManager(models.Manager):
def create_project(self, name, release):
if release is not None:
- prj = self.model(name = name, bitbake_version = release.bitbake_version, release = release)
+ prj = self.model(name=name,
+ bitbake_version=release.bitbake_version,
+ release=release)
else:
- prj = self.model(name = name, bitbake_version = None, release = None)
+ prj = self.model(name=name,
+ bitbake_version=None,
+ release=None)
prj.save()
- for defaultconf in ToasterSetting.objects.filter(name__startswith="DEFCONF_"):
+ for defaultconf in ToasterSetting.objects.filter(
+ name__startswith="DEFCONF_"):
name = defaultconf.name[8:]
- ProjectVariable.objects.create( project = prj,
- name = name,
- value = defaultconf.value)
+ ProjectVariable.objects.create(project=prj,
+ name=name,
+ value=defaultconf.value)
if release is None:
return prj
for rdl in release.releasedefaultlayer_set.all():
- try:
- lv = Layer_Version.objects.filter(layer__name = rdl.layer_name, up_branch__name = release.branch_name)[0].get_equivalents_wpriority(prj)[0]
- ProjectLayer.objects.create( project = prj,
- layercommit = lv,
- optional = False )
- except IndexError:
- # we may have no valid layer version objects, and that's ok
- pass
+ lv = Layer_Version.objects.filter(
+ layer__name=rdl.layer_name,
+ up_branch__name=release.branch_name).first()
+
+ if lv:
+ ProjectLayer.objects.create(project=prj,
+ layercommit=lv,
+ optional=False)
+ else:
+ logger.warning("Default project layer %s not found" %
+ rdl.layer_name)
return prj
# return single object with is_default = True
def get_or_create_default_project(self):
- projects = super(ProjectManager, self).filter(is_default = True)
+ projects = super(ProjectManager, self).filter(is_default=True)
if len(projects) > 1:
raise Exception('Inconsistent project data: multiple ' +
@@ -157,7 +166,8 @@ class ProjectManager(models.Manager):
elif len(projects) < 1:
options = {
'name': 'Command line builds',
- 'short_description': 'Project for builds started outside Toaster',
+ 'short_description':
+ 'Project for builds started outside Toaster',
'is_default': True
}
project = Project.objects.create(**options)
@@ -971,21 +981,27 @@ class Target_Installed_Package(models.Model):
target = models.ForeignKey(Target)
package = models.ForeignKey(Package, related_name='buildtargetlist_package')
+
class Package_File(models.Model):
package = models.ForeignKey(Package, related_name='buildfilelist_package')
path = models.FilePathField(max_length=255, blank=True)
size = models.IntegerField()
+
class Recipe(models.Model):
- search_allowed_fields = ['name', 'version', 'file_path', 'section', 'summary', 'description', 'license', 'layer_version__layer__name', 'layer_version__branch', 'layer_version__commit', 'layer_version__local_path', 'layer_version__layer_source__name']
+ search_allowed_fields = ['name', 'version', 'file_path', 'section',
+ 'summary', 'description', 'license',
+ 'layer_version__layer__name',
+ 'layer_version__branch', 'layer_version__commit',
+ 'layer_version__local_path',
+ 'layer_version__layer_source']
- layer_source = models.ForeignKey('LayerSource', default = None, null = True) # from where did we get this recipe
- up_id = models.IntegerField(null = True, default = None) # id of entry in the source
- up_date = models.DateTimeField(null = True, default = None)
+ up_date = models.DateTimeField(null=True, default=None)
- name = models.CharField(max_length=100, blank=True) # pn
- version = models.CharField(max_length=100, blank=True) # pv
- layer_version = models.ForeignKey('Layer_Version', related_name='recipe_layer_version')
+ name = models.CharField(max_length=100, blank=True)
+ version = models.CharField(max_length=100, blank=True)
+ layer_version = models.ForeignKey('Layer_Version',
+ related_name='recipe_layer_version')
summary = models.TextField(blank=True)
description = models.TextField(blank=True)
section = models.CharField(max_length=100, blank=True)
@@ -996,13 +1012,6 @@ class Recipe(models.Model):
pathflags = models.CharField(max_length=200, blank=True)
is_image = models.BooleanField(default=False)
- def get_layersource_view_url(self):
- if self.layer_source is None:
- return ""
-
- url = self.layer_source.get_object_view(self.layer_version.up_branch, "recipes", self.name)
- return url
-
def __unicode__(self):
return "Recipe " + self.name + ":" + self.version
@@ -1048,8 +1057,6 @@ class Recipe_Dependency(models.Model):
class Machine(models.Model):
search_allowed_fields = ["name", "description", "layer_version__layer__name"]
- layer_source = models.ForeignKey('LayerSource', default = None, null = True) # from where did we get this machine
- up_id = models.IntegerField(null = True, default = None) # id of entry in the source
up_date = models.DateTimeField(null = True, default = None)
layer_version = models.ForeignKey('Layer_Version')
@@ -1064,293 +1071,9 @@ class Machine(models.Model):
def __unicode__(self):
return "Machine " + self.name + "(" + self.description + ")"
- class Meta:
- unique_together = ("layer_source", "up_id")
-
-
-from django.db.models.base import ModelBase
-class InheritanceMetaclass(ModelBase):
- def __call__(cls, *args, **kwargs):
- obj = super(InheritanceMetaclass, cls).__call__(*args, **kwargs)
- return obj.get_object()
-class LayerSource(models.Model):
- __metaclass__ = InheritanceMetaclass
-
- class Meta:
- unique_together = (('sourcetype', 'apiurl'), )
-
- TYPE_LOCAL = 0
- TYPE_LAYERINDEX = 1
- TYPE_IMPORTED = 2
- SOURCE_TYPE = (
- (TYPE_LOCAL, "local"),
- (TYPE_LAYERINDEX, "layerindex"),
- (TYPE_IMPORTED, "imported"),
- )
-
- name = models.CharField(max_length=63, unique = True)
- sourcetype = models.IntegerField(choices=SOURCE_TYPE)
- apiurl = models.CharField(max_length=255, null=True, default=None)
-
- def __init__(self, *args, **kwargs):
- super(LayerSource, self).__init__(*args, **kwargs)
- if self.sourcetype == LayerSource.TYPE_LOCAL:
- self.__class__ = LocalLayerSource
- elif self.sourcetype == LayerSource.TYPE_LAYERINDEX:
- self.__class__ = LayerIndexLayerSource
- elif self.sourcetype == LayerSource.TYPE_IMPORTED:
- self.__class__ = ImportedLayerSource
- elif self.sourcetype == None:
- raise Exception("Unknown LayerSource-derived class. If you added a new layer source type, fill out all code stubs.")
-
-
- def update(self):
- """
- Updates the local database information from the upstream layer source
- """
- raise Exception("Abstract, update() must be implemented by all LayerSource-derived classes (object is %s)" % str(vars(self)))
-
- def save(self, *args, **kwargs):
- return super(LayerSource, self).save(*args, **kwargs)
-
- def get_object(self):
- # preset an un-initilized object
- if None == self.name:
- self.name=""
- if None == self.apiurl:
- self.apiurl=""
- if None == self.sourcetype:
- self.sourcetype=LayerSource.TYPE_LOCAL
-
- if self.sourcetype == LayerSource.TYPE_LOCAL:
- self.__class__ = LocalLayerSource
- elif self.sourcetype == LayerSource.TYPE_LAYERINDEX:
- self.__class__ = LayerIndexLayerSource
- elif self.sourcetype == LayerSource.TYPE_IMPORTED:
- self.__class__ = ImportedLayerSource
- else:
- raise Exception("Unknown LayerSource type. If you added a new layer source type, fill out all code stubs.")
- return self
-
- def __unicode__(self):
- return "%s (%s)" % (self.name, self.sourcetype)
-
-
-class LocalLayerSource(LayerSource):
- class Meta(LayerSource._meta.__class__):
- proxy = True
-
- def __init__(self, *args, **kwargs):
- super(LocalLayerSource, self).__init__(args, kwargs)
- self.sourcetype = LayerSource.TYPE_LOCAL
-
- def update(self):
- """
- Fetches layer, recipe and machine information from local repository
- """
- pass
-
-class ImportedLayerSource(LayerSource):
- class Meta(LayerSource._meta.__class__):
- proxy = True
-
- def __init__(self, *args, **kwargs):
- super(ImportedLayerSource, self).__init__(args, kwargs)
- self.sourcetype = LayerSource.TYPE_IMPORTED
-
- def update(self):
- """
- Fetches layer, recipe and machine information from local repository
- """
- pass
-
-
-class LayerIndexLayerSource(LayerSource):
- class Meta(LayerSource._meta.__class__):
- proxy = True
-
- def __init__(self, *args, **kwargs):
- super(LayerIndexLayerSource, self).__init__(args, kwargs)
- self.sourcetype = LayerSource.TYPE_LAYERINDEX
-
- def get_object_view(self, branch, objectype, upid):
- return self.apiurl + "../branch/" + branch.name + "/" + objectype + "/?q=" + str(upid)
-
- def update(self):
- """
- Fetches layer, recipe and machine information from remote repository
- """
- assert self.apiurl is not None
- from django.db import transaction, connection
-
- import json
- import os
-
- try:
- from urllib.request import urlopen, URLError
- from urllib.parse import urlparse
- except ImportError:
- from urllib2 import urlopen, URLError
- from urlparse import urlparse
-
- proxy_settings = os.environ.get("http_proxy", None)
- oe_core_layer = 'openembedded-core'
-
- def _get_json_response(apiurl = self.apiurl):
- _parsedurl = urlparse(apiurl)
- path = _parsedurl.path
-
- try:
- res = urlopen(apiurl)
- except URLError as e:
- raise Exception("Failed to read %s: %s" % (path, e.reason))
-
- return json.loads(res.read().decode('utf-8'))
-
- # verify we can get the basic api
- try:
- apilinks = _get_json_response()
- except Exception as e:
- import traceback
- if proxy_settings is not None:
- logger.info("EE: Using proxy %s" % proxy_settings)
- logger.warning("EE: could not connect to %s, skipping update: %s\n%s" % (self.apiurl, e, traceback.format_exc()))
- return
-
- # update branches; only those that we already have names listed in the
- # Releases table
- whitelist_branch_names = [rel.branch_name for rel in Release.objects.all()]
- if len(whitelist_branch_names) == 0:
- raise Exception("Failed to make list of branches to fetch")
-
- logger.debug("Fetching branches")
- branches_info = _get_json_response(apilinks['branches']
- + "?filter=name:%s" % "OR".join(whitelist_branch_names))
- for bi in branches_info:
- b, created = Branch.objects.get_or_create(layer_source = self, name = bi['name'])
- b.up_id = bi['id']
- b.up_date = bi['updated']
- b.name = bi['name']
- b.short_description = bi['short_description']
- b.save()
-
- # update layers
- layers_info = _get_json_response(apilinks['layerItems'])
-
- for li in layers_info:
- # Special case for the openembedded-core layer
- if li['name'] == oe_core_layer:
- try:
- # If we have an existing openembedded-core for example
- # from the toasterconf.json augment the info using the
- # layerindex rather than duplicate it
- oe_core_l = Layer.objects.get(name=oe_core_layer)
- # Take ownership of the layer as now coming from the
- # layerindex
- oe_core_l.layer_source = self
- oe_core_l.up_id = li['id']
- oe_core_l.summary = li['summary']
- oe_core_l.description = li['description']
- oe_core_l.save()
- continue
-
- except Layer.DoesNotExist:
- pass
-
- l, created = Layer.objects.get_or_create(layer_source = self, name = li['name'])
- l.up_id = li['id']
- l.up_date = li['updated']
- l.vcs_url = li['vcs_url']
- l.vcs_web_url = li['vcs_web_url']
- l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
- l.vcs_web_file_base_url = li['vcs_web_file_base_url']
- l.summary = li['summary']
- l.description = li['description']
- l.save()
-
- # update layerbranches/layer_versions
- logger.debug("Fetching layer information")
- layerbranches_info = _get_json_response(apilinks['layerBranches']
- + "?filter=branch:%s" % "OR".join(map(lambda x: str(x.up_id), [i for i in Branch.objects.filter(layer_source = self) if i.up_id is not None] ))
- )
-
- for lbi in layerbranches_info:
- lv, created = Layer_Version.objects.get_or_create(layer_source = self,
- up_id = lbi['id'],
- layer=Layer.objects.get(layer_source = self, up_id = lbi['layer'])
- )
-
- lv.up_date = lbi['updated']
- lv.up_branch = Branch.objects.get(layer_source = self, up_id = lbi['branch'])
- lv.branch = lbi['actual_branch']
- lv.commit = lbi['actual_branch']
- lv.dirpath = lbi['vcs_subdir']
- lv.save()
-
- # update layer dependencies
- layerdependencies_info = _get_json_response(apilinks['layerDependencies'])
- dependlist = {}
- for ldi in layerdependencies_info:
- try:
- lv = Layer_Version.objects.get(layer_source = self, up_id = ldi['layerbranch'])
- except Layer_Version.DoesNotExist as e:
- continue
-
- if lv not in dependlist:
- dependlist[lv] = []
- try:
- dependlist[lv].append(Layer_Version.objects.get(layer_source = self, layer__up_id = ldi['dependency'], up_branch = lv.up_branch))
- except Layer_Version.DoesNotExist:
- logger.warning("Cannot find layer version (ls:%s), up_id:%s lv:%s" % (self, ldi['dependency'], lv))
-
- for lv in dependlist:
- LayerVersionDependency.objects.filter(layer_version = lv).delete()
- for lvd in dependlist[lv]:
- LayerVersionDependency.objects.get_or_create(layer_version = lv, depends_on = lvd)
-
-
- # update machines
- logger.debug("Fetching machine information")
- machines_info = _get_json_response(apilinks['machines']
- + "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
- )
-
- for mi in machines_info:
- mo, created = Machine.objects.get_or_create(layer_source = self, up_id = mi['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = mi['layerbranch']))
- mo.up_date = mi['updated']
- mo.name = mi['name']
- mo.description = mi['description']
- mo.save()
-
- # update recipes; paginate by layer version / layer branch
- logger.debug("Fetching target information")
- recipes_info = _get_json_response(apilinks['recipes']
- + "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
- )
- for ri in recipes_info:
- try:
- ro, created = Recipe.objects.get_or_create(layer_source = self, up_id = ri['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = ri['layerbranch']))
- ro.up_date = ri['updated']
- ro.name = ri['pn']
- ro.version = ri['pv']
- ro.summary = ri['summary']
- ro.description = ri['description']
- ro.section = ri['section']
- ro.license = ri['license']
- ro.homepage = ri['homepage']
- ro.bugtracker = ri['bugtracker']
- ro.file_path = ri['filepath'] + "/" + ri['filename']
- if 'inherits' in ri:
- ro.is_image = 'image' in ri['inherits'].split()
- else: # workaround for old style layer index
- ro.is_image = "-image-" in ri['pn']
- ro.save()
- except IntegrityError as e:
- logger.debug("Failed saving recipe, ignoring: %s (%s:%s)" % (e, ro.layer_version, ri['filepath']+"/"+ri['filename']))
- ro.delete()
class BitbakeVersion(models.Model):
@@ -1374,87 +1097,98 @@ class Release(models.Model):
def __unicode__(self):
return "%s (%s)" % (self.name, self.branch_name)
-class ReleaseLayerSourcePriority(models.Model):
- """ Each release selects layers from the set up layer sources, ordered by priority """
- release = models.ForeignKey("Release")
- layer_source = models.ForeignKey("LayerSource")
- priority = models.IntegerField(default = 0)
-
- def __unicode__(self):
- return "%s-%s:%d" % (self.release.name, self.layer_source.name, self.priority)
- class Meta:
- unique_together = (('release', 'layer_source'),)
-
-
class ReleaseDefaultLayer(models.Model):
release = models.ForeignKey(Release)
layer_name = models.CharField(max_length=100, default="")
-# Branch class is synced with layerindex.Branch, branches can only come from remote layer indexes
+# Branch class is synced with layerindex.Branch, branches can only come
+# from remote layer indexes
class Branch(models.Model):
- layer_source = models.ForeignKey('LayerSource', null = True, default = True)
- up_id = models.IntegerField(null = True, default = None) # id of branch in the source
- up_date = models.DateTimeField(null = True, default = None)
+ # id of branch in the layerindex
+ up_date = models.DateTimeField(null=True, default=None)
name = models.CharField(max_length=50)
short_description = models.CharField(max_length=50, blank=True)
class Meta:
verbose_name_plural = "Branches"
- unique_together = (('layer_source', 'name'),('layer_source', 'up_id'))
def __unicode__(self):
return self.name
-# Layer class synced with layerindex.LayerItem
+class LayerSource(object):
+ """ Where the layer metadata came from """
+ TYPE_LOCAL = 0
+ TYPE_LAYERINDEX = 1
+ TYPE_IMPORTED = 2
+ TYPE_BUILD = 3
+
+ SOURCE_TYPE = (
+ (TYPE_LOCAL, "local"),
+ (TYPE_LAYERINDEX, "layerindex"),
+ (TYPE_IMPORTED, "imported"),
+ (TYPE_BUILD, "build"),
+ )
+
+
class Layer(models.Model):
- layer_source = models.ForeignKey(LayerSource, null = True, default = None) # from where did we got this layer
- up_id = models.IntegerField(null = True, default = None) # id of layer in the remote source
- up_date = models.DateTimeField(null = True, default = None)
+
+ up_date = models.DateTimeField(null=True, default=timezone.now)
name = models.CharField(max_length=100)
layer_index_url = models.URLField()
- vcs_url = GitURLField(default = None, null = True)
- vcs_web_url = models.URLField(null = True, default = None)
- vcs_web_tree_base_url = models.URLField(null = True, default = None)
- vcs_web_file_base_url = models.URLField(null = True, default = None)
+ vcs_url = GitURLField(default=None, null=True)
+ vcs_web_url = models.URLField(null=True, default=None)
+ vcs_web_tree_base_url = models.URLField(null=True, default=None)
+ vcs_web_file_base_url = models.URLField(null=True, default=None)
- summary = models.TextField(help_text='One-line description of the layer', null = True, default = None)
- description = models.TextField(null = True, default = None)
+ summary = models.TextField(help_text='One-line description of the layer',
+ null=True, default=None)
+ description = models.TextField(null=True, default=None)
def __unicode__(self):
- return "%s / %s " % (self.name, self.layer_source)
-
- class Meta:
- unique_together = (("layer_source", "up_id"), ("layer_source", "name"))
+ return "%s / %s " % (self.name, self.summary)
-# LayerCommit class is synced with layerindex.LayerBranch
class Layer_Version(models.Model):
"""
A Layer_Version either belongs to a single project or no project
"""
- search_allowed_fields = ["layer__name", "layer__summary", "layer__description", "layer__vcs_url", "dirpath", "up_branch__name", "commit", "branch"]
- build = models.ForeignKey(Build, related_name='layer_version_build', default = None, null = True)
+ search_allowed_fields = ["layer__name", "layer__summary",
+ "layer__description", "layer__vcs_url",
+ "dirpath", "up_branch__name", "commit", "branch"]
+
+ build = models.ForeignKey(Build, related_name='layer_version_build',
+ default=None, null=True)
+
layer = models.ForeignKey(Layer, related_name='layer_version_layer')
- layer_source = models.ForeignKey(LayerSource, null = True, default = None) # from where did we get this Layer Version
- up_id = models.IntegerField(null = True, default = None) # id of layerbranch in the remote source
- up_date = models.DateTimeField(null = True, default = None)
- up_branch = models.ForeignKey(Branch, null = True, default = None)
+ layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
+ default=0)
+
+ up_date = models.DateTimeField(null=True, default=timezone.now)
- branch = models.CharField(max_length=80) # LayerBranch.actual_branch
- commit = models.CharField(max_length=100) # LayerBranch.vcs_last_rev
- dirpath = models.CharField(max_length=255, null = True, default = None) # LayerBranch.vcs_subdir
- priority = models.IntegerField(default = 0) # if -1, this is a default layer
+ # layerindex specific field
+ up_branch = models.ForeignKey(Branch, null=True, default=None)
- local_path = models.FilePathField(max_length=1024, default = "/") # where this layer was checked-out
+ branch = models.CharField(max_length=80)
+ commit = models.CharField(max_length=100)
+ # If the layer is in a subdir
+ dirpath = models.CharField(max_length=255, null=True, default=None)
- project = models.ForeignKey('Project', null = True, default = None) # Set if this layer is project-specific; always set for imported layers, and project-set branches
+ # if -1, this is a default layer
+ priority = models.IntegerField(default=0)
- # code lifted, with adaptations, from the layerindex-web application https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
+ # where this layer exists on the filesystem
+ local_path = models.FilePathField(max_length=1024, default="/")
+
+ # Set if this layer is restricted to a particular project
+ project = models.ForeignKey('Project', null=True, default=None)
+
+ # code lifted, with adaptations, from the layerindex-web application
+ # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
def _handle_url_path(self, base_url, path):
import re, posixpath
if base_url:
@@ -1496,18 +1230,14 @@ class Layer_Version(models.Model):
def get_vcs_file_link_url(self, file_path=""):
if self.layer.vcs_web_file_base_url is None:
return None
- return self._handle_url_path(self.layer.vcs_web_file_base_url, file_path)
+ return self._handle_url_path(self.layer.vcs_web_file_base_url,
+ file_path)
def get_vcs_dirpath_link_url(self):
if self.layer.vcs_web_tree_base_url is None:
return None
return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
- def get_equivalents_wpriority(self, project):
- layer_versions = project.get_all_compatible_layer_versions()
- filtered = layer_versions.filter(layer__name = self.layer.name)
- return filtered.order_by("-layer_source__releaselayersourcepriority__priority")
-
def get_vcs_reference(self):
if self.branch is not None and len(self.branch) > 0:
return self.branch
@@ -1540,20 +1270,15 @@ class Layer_Version(models.Model):
return sorted(result, key=lambda x: x.layer.name)
def __unicode__(self):
- return "%d %s (VCS %s, Project %s)" % (self.pk, str(self.layer), self.get_vcs_reference(), self.build.project if self.build is not None else "No project")
+ return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
- class Meta:
- unique_together = ("layer_source", "up_id")
class LayerVersionDependency(models.Model):
- layer_source = models.ForeignKey(LayerSource, null = True, default = None) # from where did we got this layer
- up_id = models.IntegerField(null = True, default = None) # id of layerbranch in the remote source
-
- layer_version = models.ForeignKey(Layer_Version, related_name="dependencies")
- depends_on = models.ForeignKey(Layer_Version, related_name="dependees")
- class Meta:
- unique_together = ("layer_source", "up_id")
+ layer_version = models.ForeignKey(Layer_Version,
+ related_name="dependencies")
+ depends_on = models.ForeignKey(Layer_Version,
+ related_name="dependees")
class ProjectLayer(models.Model):
project = models.ForeignKey(Project)
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 4/9] toaster: lsupdates Add layerindex fetcher
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
` (2 preceding siblings ...)
2016-07-07 17:21 ` [PATCH v2 3/9] toaster: models Remove LayerSource models and replace with enum Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-08 10:45 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 5/9] toaster: Replace references to LayerSource models Michael Wood
` (4 subsequent siblings)
8 siblings, 1 reply; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
Move and refactor the layerindex layer source update mechanism so that
we don't have to track the layerindex objects in the toaster database.
Move this out of the orm and into the management command.
Paves the way for future improvement to allow you to specify a layer
index server as an argument to the command.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
.../toaster/orm/management/commands/lsupdates.py | 250 ++++++++++++++++++++-
1 file changed, 244 insertions(+), 6 deletions(-)
diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 75e9513..f4eba81 100644
--- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -1,12 +1,250 @@
-from django.core.management.base import NoArgsCommand, CommandError
-from orm.models import LayerSource
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.management.base import NoArgsCommand
+
+from orm.models import LayerSource, Layer, Release, Layer_Version
+from orm.models import LayerVersionDependency, Machine, Recipe
+
import os
+import json
+import logging
+logger = logging.getLogger("toaster")
+
+DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
+
class Command(NoArgsCommand):
- args = ""
- help = "Updates locally cached information from all LayerSources"
+ args = ""
+ help = "Updates locally cached information from a layerindex server"
+
+ def update(self):
+ """
+ Fetches layer, recipe and machine information from a layerindex
+ server
+ """
+
+ self.apiurl = DEFAULT_LAYERINDEX_SERVER
+
+ assert self.apiurl is not None
+ try:
+ from urllib.request import urlopen, URLError
+ from urllib.parse import urlparse
+ except ImportError:
+ from urllib2 import urlopen, URLError
+ from urlparse import urlparse
+
+ proxy_settings = os.environ.get("http_proxy", None)
+ oe_core_layer = 'openembedded-core'
+
+ def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
+ _parsedurl = urlparse(apiurl)
+ path = _parsedurl.path
+
+ # logger.debug("Fetching %s", apiurl)
+ try:
+ res = urlopen(apiurl)
+ except URLError as e:
+ raise Exception("Failed to read %s: %s" % (path, e.reason))
+
+ return json.loads(res.read().decode('utf-8'))
+
+ # verify we can get the basic api
+ try:
+ apilinks = _get_json_response()
+ except Exception as e:
+ import traceback
+ if proxy_settings is not None:
+ logger.info("EE: Using proxy %s" % proxy_settings)
+ logger.warning("EE: could not connect to %s, skipping update:"
+ "%s\n%s" % (self.apiurl, e, traceback.format_exc()))
+ return
+
+ # update branches; only those that we already have names listed in the
+ # Releases table
+ whitelist_branch_names = [rel.branch_name
+ for rel in Release.objects.all()]
+ if len(whitelist_branch_names) == 0:
+ raise Exception("Failed to make list of branches to fetch")
+
+ logger.debug("Fetching branches")
+
+ # keep a track of the id mappings so that layer_versions can be created
+ # for these layers later on
+ li_layer_id_to_toaster_layer_id = {}
+
+ # We may need this? TODO
+ #branches_info = _get_json_response(apilinks['branches'] +
+ # "?filter=name:%s"
+ # % "OR".join(whitelist_branch_names))
+
+ # update layers
+ layers_info = _get_json_response(apilinks['layerItems'])
+
+ for li in layers_info:
+ # Special case for the openembedded-core layer
+ if li['name'] == oe_core_layer:
+ try:
+ # If we have an existing openembedded-core for example
+ # from the toasterconf.json augment the info using the
+ # layerindex rather than duplicate it
+ oe_core_l = Layer.objects.get(name=oe_core_layer)
+ # Take ownership of the layer as now coming from the
+ # layerindex
+ oe_core_l.summary = li['summary']
+ oe_core_l.description = li['description']
+ oe_core_l.save()
+ li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
+ continue
+
+ except Layer.DoesNotExist:
+ pass
+
+ l, created = Layer.objects.get_or_create(name=li['name'])
+ l.up_date = li['updated']
+ l.vcs_url = li['vcs_url']
+ l.vcs_web_url = li['vcs_web_url']
+ l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+ l.vcs_web_file_base_url = li['vcs_web_file_base_url']
+ l.summary = li['summary']
+ l.description = li['description']
+ l.save()
+
+ li_layer_id_to_toaster_layer_id[li['id']] = l.pk
+
+ # update layerbranches/layer_versions
+ logger.debug("Fetching layer information")
+ layerbranches_info = _get_json_response(
+ apilinks['layerBranches'] + "?filter=branch__name:%s" %
+ "OR".join(whitelist_branch_names))
+
+ # Map Layer index layer_branch object id to
+ # layer_version toaster object id
+ li_layer_branch_id_to_toaster_lv_id = {}
+
+ for lbi in layerbranches_info:
+
+ try:
+ lv, created = Layer_Version.objects.get_or_create(
+ layer_source=LayerSource.TYPE_LAYERINDEX,
+ layer=Layer.objects.get(
+ pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
+ )
+ except KeyError:
+ print("No such layerindex layer referenced by layerbranch %d" %
+ lbi['layer'])
+ continue
+
+ lv.up_date = lbi['updated']
+ lv.commit = lbi['actual_branch']
+ lv.dirpath = lbi['vcs_subdir']
+ lv.save()
+
+ li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
+ lv.pk
+
+ # update layer dependencies
+ layerdependencies_info = _get_json_response(
+ apilinks['layerDependencies'] +
+ "?filter=layerbranch__branch__name:%s" %
+ "OR".join(whitelist_branch_names))
+
+ dependlist = {}
+ for ldi in layerdependencies_info:
+ try:
+ lv = Layer_Version.objects.get(
+ pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']])
+ except Layer_Version.DoesNotExist as e:
+ continue
+
+ if lv not in dependlist:
+ dependlist[lv] = []
+ try:
+ layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
+
+ dependlist[lv].append(
+ Layer_Version.objects.get(
+ layer_source=LayerSource.TYPE_LAYERINDEX,
+ layer__pk=layer_id))
+
+ except Layer_Version.DoesNotExist:
+ logger.warning("Cannot find layer version (ls:%s),"
+ "up_id:%s lv:%s" %
+ (self, ldi['dependency'], lv))
+
+ for lv in dependlist:
+ LayerVersionDependency.objects.filter(layer_version=lv).delete()
+ for lvd in dependlist[lv]:
+ LayerVersionDependency.objects.get_or_create(layer_version=lv,
+ depends_on=lvd)
+
+ # update machines
+ logger.debug("Fetching machine information")
+ machines_info = _get_json_response(
+ apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
+ "OR".join(whitelist_branch_names))
+
+ for mi in machines_info:
+ mo, created = Machine.objects.get_or_create(
+ layer_version=Layer_Version.objects.get(
+ pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
+ mo.up_date = mi['updated']
+ mo.name = mi['name']
+ mo.description = mi['description']
+ mo.save()
+
+ # update recipes; paginate by layer version / layer branch
+ logger.debug("Fetching target information")
+ recipes_info = _get_json_response(
+ apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
+ "OR".join(whitelist_branch_names))
+
+ for ri in recipes_info:
+ try:
+ lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
+ lv = Layer_Version.objects.get(pk=lv_id)
+
+ ro, created = Recipe.objects.get_or_create(
+ layer_version=lv,
+ name=ri['pn']
+ )
+ ro.layer_version = lv
+ ro.up_date = ri['updated']
+ ro.name = ri['pn']
+ ro.version = ri['pv']
+ ro.summary = ri['summary']
+ ro.description = ri['description']
+ ro.section = ri['section']
+ ro.license = ri['license']
+ ro.homepage = ri['homepage']
+ ro.bugtracker = ri['bugtracker']
+ ro.file_path = ri['filepath'] + "/" + ri['filename']
+ if 'inherits' in ri:
+ ro.is_image = 'image' in ri['inherits'].split()
+ else: # workaround for old style layer index
+ ro.is_image = "-image-" in ri['pn']
+ ro.save()
+ except Exception as e:
+ logger.debug("Failed saving recipe %s", e)
def handle_noargs(self, **options):
- for ls in LayerSource.objects.all():
- ls.update()
+ self.update()
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH v2 4/9] toaster: lsupdates Add layerindex fetcher
2016-07-07 17:21 ` [PATCH v2 4/9] toaster: lsupdates Add layerindex fetcher Michael Wood
@ 2016-07-08 10:45 ` Michael Wood
0 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-08 10:45 UTC (permalink / raw)
To: toaster@yoctoproject.org
On 07/07/16 18:21, Michael Wood wrote:
> Move and refactor the layerindex layer source update mechanism so that
> we don't have to track the layerindex objects in the toaster database.
> Move this out of the orm and into the management command.
>
> Paves the way for future improvement to allow you to specify a layer
> index server as an argument to the command.
>
> Signed-off-by: Michael Wood <michael.g.wood@intel.com>
> ---
> .../toaster/orm/management/commands/lsupdates.py | 250 ++++++++++++++++++++-
> 1 file changed, 244 insertions(+), 6 deletions(-)
>
> diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> index 75e9513..f4eba81 100644
> --- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> +++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
> @@ -1,12 +1,250 @@
> -from django.core.management.base import NoArgsCommand, CommandError
> -from orm.models import LayerSource
> +#
> +# ex:ts=4:sw=4:sts=4:et
> +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
> +#
> +# BitBake Toaster Implementation
> +#
> +# Copyright (C) 2016 Intel Corporation
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +from django.core.management.base import NoArgsCommand
> +
> +from orm.models import LayerSource, Layer, Release, Layer_Version
> +from orm.models import LayerVersionDependency, Machine, Recipe
> +
> import os
> +import json
> +import logging
> +logger = logging.getLogger("toaster")
> +
> +DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
> +
>
> class Command(NoArgsCommand):
> - args = ""
> - help = "Updates locally cached information from all LayerSources"
> + args = ""
> + help = "Updates locally cached information from a layerindex server"
> +
> + def update(self):
> + """
> + Fetches layer, recipe and machine information from a layerindex
> + server
> + """
> +
> + self.apiurl = DEFAULT_LAYERINDEX_SERVER
> +
> + assert self.apiurl is not None
> + try:
> + from urllib.request import urlopen, URLError
> + from urllib.parse import urlparse
> + except ImportError:
> + from urllib2 import urlopen, URLError
> + from urlparse import urlparse
> +
> + proxy_settings = os.environ.get("http_proxy", None)
> + oe_core_layer = 'openembedded-core'
> +
> + def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
> + _parsedurl = urlparse(apiurl)
> + path = _parsedurl.path
> +
> + # logger.debug("Fetching %s", apiurl)
> + try:
> + res = urlopen(apiurl)
> + except URLError as e:
> + raise Exception("Failed to read %s: %s" % (path, e.reason))
> +
> + return json.loads(res.read().decode('utf-8'))
> +
> + # verify we can get the basic api
> + try:
> + apilinks = _get_json_response()
> + except Exception as e:
> + import traceback
> + if proxy_settings is not None:
> + logger.info("EE: Using proxy %s" % proxy_settings)
> + logger.warning("EE: could not connect to %s, skipping update:"
> + "%s\n%s" % (self.apiurl, e, traceback.format_exc()))
> + return
> +
> + # update branches; only those that we already have names listed in the
> + # Releases table
> + whitelist_branch_names = [rel.branch_name
> + for rel in Release.objects.all()]
> + if len(whitelist_branch_names) == 0:
> + raise Exception("Failed to make list of branches to fetch")
> +
> + logger.debug("Fetching branches")
> +
> + # keep a track of the id mappings so that layer_versions can be created
> + # for these layers later on
> + li_layer_id_to_toaster_layer_id = {}
> +
> + # We may need this? TODO
> + #branches_info = _get_json_response(apilinks['branches'] +
> + # "?filter=name:%s"
> + # % "OR".join(whitelist_branch_names))
> +
> + # update layers
> + layers_info = _get_json_response(apilinks['layerItems'])
> +
> + for li in layers_info:
> + # Special case for the openembedded-core layer
> + if li['name'] == oe_core_layer:
> + try:
> + # If we have an existing openembedded-core for example
> + # from the toasterconf.json augment the info using the
> + # layerindex rather than duplicate it
> + oe_core_l = Layer.objects.get(name=oe_core_layer)
> + # Take ownership of the layer as now coming from the
> + # layerindex
> + oe_core_l.summary = li['summary']
> + oe_core_l.description = li['description']
> + oe_core_l.save()
> + li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
> + continue
> +
> + except Layer.DoesNotExist:
> + pass
> +
> + l, created = Layer.objects.get_or_create(name=li['name'])
> + l.up_date = li['updated']
> + l.vcs_url = li['vcs_url']
> + l.vcs_web_url = li['vcs_web_url']
> + l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
> + l.vcs_web_file_base_url = li['vcs_web_file_base_url']
> + l.summary = li['summary']
> + l.description = li['description']
> + l.save()
> +
> + li_layer_id_to_toaster_layer_id[li['id']] = l.pk
> +
> + # update layerbranches/layer_versions
> + logger.debug("Fetching layer information")
> + layerbranches_info = _get_json_response(
> + apilinks['layerBranches'] + "?filter=branch__name:%s" %
> + "OR".join(whitelist_branch_names))
> +
> + # Map Layer index layer_branch object id to
> + # layer_version toaster object id
> + li_layer_branch_id_to_toaster_lv_id = {}
> +
> + for lbi in layerbranches_info:
> +
> + try:
> + lv, created = Layer_Version.objects.get_or_create(
> + layer_source=LayerSource.TYPE_LAYERINDEX,
> + layer=Layer.objects.get(
> + pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
> + )
> + except KeyError:
> + print("No such layerindex layer referenced by layerbranch %d" %
> + lbi['layer'])
> + continue
> +
> + lv.up_date = lbi['updated']
> + lv.commit = lbi['actual_branch']
> + lv.dirpath = lbi['vcs_subdir']
> + lv.save()
> +
> + li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
> + lv.pk
> +
> + # update layer dependencies
> + layerdependencies_info = _get_json_response(
> + apilinks['layerDependencies'] +
> + "?filter=layerbranch__branch__name:%s" %
> + "OR".join(whitelist_branch_names))
> +
> + dependlist = {}
> + for ldi in layerdependencies_info:
> + try:
> + lv = Layer_Version.objects.get(
> + pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']])
> + except Layer_Version.DoesNotExist as e:
> + continue
> +
> + if lv not in dependlist:
> + dependlist[lv] = []
> + try:
> + layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
> +
> + dependlist[lv].append(
> + Layer_Version.objects.get(
> + layer_source=LayerSource.TYPE_LAYERINDEX,
> + layer__pk=layer_id))
> +
> + except Layer_Version.DoesNotExist:
> + logger.warning("Cannot find layer version (ls:%s),"
> + "up_id:%s lv:%s" %
> + (self, ldi['dependency'], lv))
> +
> + for lv in dependlist:
> + LayerVersionDependency.objects.filter(layer_version=lv).delete()
> + for lvd in dependlist[lv]:
> + LayerVersionDependency.objects.get_or_create(layer_version=lv,
> + depends_on=lvd)
> +
> + # update machines
> + logger.debug("Fetching machine information")
> + machines_info = _get_json_response(
> + apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
> + "OR".join(whitelist_branch_names))
> +
> + for mi in machines_info:
> + mo, created = Machine.objects.get_or_create(
> + layer_version=Layer_Version.objects.get(
> + pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
This was missing the name attribute and should have been:
+ mo, created = Machine.objects.get_or_create(
+ name=mi['name'],
+ layer_version=Layer_Version.objects.get(
+ pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
It has been corrected on the branch
> + mo.up_date = mi['updated']
> + mo.name = mi['name']
> + mo.description = mi['description']
> + mo.save()
> +
> + # update recipes; paginate by layer version / layer branch
> + logger.debug("Fetching target information")
> + recipes_info = _get_json_response(
> + apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
> + "OR".join(whitelist_branch_names))
> +
> + for ri in recipes_info:
> + try:
> + lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
> + lv = Layer_Version.objects.get(pk=lv_id)
> +
> + ro, created = Recipe.objects.get_or_create(
> + layer_version=lv,
> + name=ri['pn']
> + )
>
> + ro.layer_version = lv
> + ro.up_date = ri['updated']
> + ro.name = ri['pn']
> + ro.version = ri['pv']
> + ro.summary = ri['summary']
> + ro.description = ri['description']
> + ro.section = ri['section']
> + ro.license = ri['license']
> + ro.homepage = ri['homepage']
> + ro.bugtracker = ri['bugtracker']
> + ro.file_path = ri['filepath'] + "/" + ri['filename']
> + if 'inherits' in ri:
> + ro.is_image = 'image' in ri['inherits'].split()
> + else: # workaround for old style layer index
> + ro.is_image = "-image-" in ri['pn']
> + ro.save()
> + except Exception as e:
> + logger.debug("Failed saving recipe %s", e)
>
> def handle_noargs(self, **options):
> - for ls in LayerSource.objects.all():
> - ls.update()
> + self.update()
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v2 5/9] toaster: Replace references to LayerSource models
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
` (3 preceding siblings ...)
2016-07-07 17:21 ` [PATCH v2 4/9] toaster: lsupdates Add layerindex fetcher Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 6/9] toaster: tests: Remove references to LayerSource model Michael Wood
` (3 subsequent siblings)
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
Replace references to the now deprecated layersource models across
Toaster with the new enums for layer source types.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
bitbake/lib/toaster/bldcontrol/tests.py | 19 --------
bitbake/lib/toaster/orm/models.py | 8 ++++
.../tests/browser/test_layerdetails_page.py | 3 --
bitbake/lib/toaster/toastergui/api.py | 6 +--
.../toaster/toastergui/static/js/layerdetails.js | 2 +-
.../toaster/toastergui/templates/layerdetails.html | 16 +++----
bitbake/lib/toaster/toastergui/views.py | 56 +++++++++++++++-------
7 files changed, 57 insertions(+), 53 deletions(-)
diff --git a/bitbake/lib/toaster/bldcontrol/tests.py b/bitbake/lib/toaster/bldcontrol/tests.py
index 32985c3..475ac0a 100644
--- a/bitbake/lib/toaster/bldcontrol/tests.py
+++ b/bitbake/lib/toaster/bldcontrol/tests.py
@@ -139,22 +139,3 @@ class RunBuildsCommandTests(TestCase):
self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
# no more selections possible here
self.assertRaises(IndexError, command._selectBuildRequest)
-
-
-class UtilityTests(TestCase):
- def test_reduce_path(self):
- from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype
-
- self.assertTrue( _reduce_canon_path("/") == "/")
- self.assertTrue( _reduce_canon_path("/home/..") == "/")
- self.assertTrue( _reduce_canon_path("/home/../ana") == "/ana")
- self.assertTrue( _reduce_canon_path("/home/../ana/..") == "/")
- self.assertTrue( _reduce_canon_path("/home/ana/mihai/../maria") == "/home/ana/maria")
-
- def test_get_id_for_sorucetype(self):
- from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype
- self.assertTrue( _get_id_for_sourcetype("layerindex") == 1)
- self.assertTrue( _get_id_for_sourcetype("local") == 0)
- self.assertTrue( _get_id_for_sourcetype("imported") == 2)
- with self.assertRaises(Exception):
- _get_id_for_sourcetype("unknown")
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index 8fe4c20..cdcd7a7 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -1132,6 +1132,14 @@ class LayerSource(object):
(TYPE_BUILD, "build"),
)
+ def types_dict():
+ """ Turn the TYPES enums into a simple dictionary """
+ dictionary = {}
+ for key in LayerSource.__dict__:
+ if "TYPE" in key:
+ dictionary[key] = getattr(LayerSource, key)
+ return dictionary
+
class Layer(models.Model):
diff --git a/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py b/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
index fb1007f..71c5c12 100644
--- a/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
@@ -49,9 +49,6 @@ class TestLayerDetailsPage(SeleniumTestCase):
# project to add new custom images to
self.project = Project.objects.create(name='foo', release=release)
- layer_source = LayerSource.objects.create(
- sourcetype=LayerSource.TYPE_IMPORTED)
-
name = "meta-imported"
vcs_url = "git://example.com/meta-imported"
subdir = "/layer"
diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py
index 112ce58..414afce 100644
--- a/bitbake/lib/toaster/toastergui/api.py
+++ b/bitbake/lib/toaster/toastergui/api.py
@@ -147,7 +147,7 @@ class XhrLayer(View):
layer_version = Layer_Version.objects.get(
id=kwargs['layerversion_id'],
project=kwargs['pid'],
- layer_source__sourcetype=LayerSource.TYPE_IMPORTED)
+ layer_source=LayerSource.TYPE_IMPORTED)
except Layer_Version.DoesNotExist:
return error_response("Cannot find imported layer to update")
@@ -159,8 +159,6 @@ class XhrLayer(View):
if "commit" in request.POST:
layer_version.commit = request.POST["commit"]
layer_version.branch = request.POST["commit"]
- if "up_branch" in request.POST:
- layer_version.up_branch_id = int(request.POST["up_branch"])
if "summary" in request.POST:
layer_version.layer.summary = request.POST["summary"]
if "description" in request.POST:
@@ -193,7 +191,7 @@ class XhrLayer(View):
layer_version = Layer_Version.objects.get(
id=kwargs['layerversion_id'],
project=kwargs['pid'],
- layer_source__sourcetype=LayerSource.TYPE_IMPORTED)
+ layer_source=LayerSource.TYPE_IMPORTED)
except Layer_Version.DoesNotExist:
return error_response("Cannot find imported layer to delete")
diff --git a/bitbake/lib/toaster/toastergui/static/js/layerdetails.js b/bitbake/lib/toaster/toastergui/static/js/layerdetails.js
index 683486e..0d4240b 100644
--- a/bitbake/lib/toaster/toastergui/static/js/layerdetails.js
+++ b/bitbake/lib/toaster/toastergui/static/js/layerdetails.js
@@ -366,7 +366,7 @@ function layerDetailsPageInit (ctx) {
if ($(this).is("dt")) {
var dd = $(this).next("dd");
if (!dd.children("form:visible")|| !dd.find(".current-value").html()){
- if (ctx.layerVersion.sourceId == 3){
+ if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED){
/* There's no current value and the layer is editable
* so show the "Not set" and hide the delete icon
*/
diff --git a/bitbake/lib/toaster/toastergui/templates/layerdetails.html b/bitbake/lib/toaster/toastergui/templates/layerdetails.html
index 4b51d1a..029c93b 100644
--- a/bitbake/lib/toaster/toastergui/templates/layerdetails.html
+++ b/bitbake/lib/toaster/toastergui/templates/layerdetails.html
@@ -36,7 +36,7 @@
</ul>
{# If this is not an imported layer then hide the edit ui #}
- {% if not layerversion.layer_source_id or layerversion.layer_source.sourcetype != layerversion.layer_source.TYPE_IMPORTED %}
+ {% if layerversion.layer_source != layer_source.TYPE_IMPORTED %}
<style scoped>
.glyphicon-edit {
display:none;
@@ -69,8 +69,9 @@
inCurrentPrj : false,
{% endif %}
layerdetailurl : "{% url 'layerdetails' project.id layerversion.id %}",
- sourceId: {{layerversion.layer_source_id|json}},
- }
+ layer_source: {{layerversion.layer_source|json}},
+ },
+ layerSourceTypes: {{layer_source|json}},
};
try {
@@ -89,7 +90,6 @@
{% endif %}>({{layerversion.get_vcs_reference|truncatechars:13}})</small>
</h1>
</div>
-
<div class="row">
<!-- container for tabs -->
<div class="col-md-8 tabbable">
@@ -277,15 +277,15 @@
<span class="glyphicon glyphicon-edit"></span>
<span class="glyphicon glyphicon-trash delete-current-value" data-toggle="tooltip" title="Delete"></span>
</dd>
- {% if layerversion.layer.up_id %}
+ {% if layerversion.layer_source == layer_source.TYPE_LAYERINDEX %}
<dt>Layer index</dt>
<dd>
- <a href="http://layers.openembedded.org/layerindex/branch/{{layerversion.up_branch.name}}/layer/{{layerversion.layer.name}}">layer index link</a>
-
+ <a href="http://layers.openembedded.org/layerindex/branch/{{layerversion.release.name}}/layer/{{layerversion.layer.name}}">Layer index {{layerversion.layer.name}}</a>
</dd>
{% endif %}
</dl>
- {% if layerversion.layer_source_id and layerversion.layer_source.sourcetype == layerversion.layer_source.TYPE_IMPORTED %}
+ {# Only show delete link for imported layers #}
+ {% if layerversion.layer_source == layer_source.TYPE_IMPORTED %}
<i class="icon-trash text-danger"></i>
<a href="#delete-layer-modal" role="button" class="text-danger" data-toggle="modal" data-target="#delete-layer-modal">Delete {{layerversion.layer.name}}</a>
{% endif %}
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index ad85faf..c76e63a 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -1439,9 +1439,8 @@ if True:
layers = [{"id": x.layercommit.pk, "orderid": x.pk, "name" : x.layercommit.layer.name,
"vcs_url": x.layercommit.layer.vcs_url, "vcs_reference" : x.layercommit.get_vcs_reference(),
"url": x.layercommit.layer.layer_index_url, "layerdetailurl": x.layercommit.get_detailspage_url(prj.pk),
- # This branch name is actually the release
"branch" : {"name" : x.layercommit.get_vcs_reference(),
- "layersource" : x.layercommit.up_branch.layer_source.name if x.layercommit.up_branch != None else None}
+ "layersource" : x.layercommit.layer_source }
} for x in prj.projectlayer_set.all().order_by("id")]
context = {
@@ -1657,10 +1656,7 @@ if True:
# We need to know what release the current project is so that we
# can set the imported layer's up_branch_id
- prj_branch_name = Release.objects.get(pk=prj.release_id).branch_name
- up_branch, branch_created = Branch.objects.get_or_create(name=prj_branch_name, layer_source_id=LayerSource.TYPE_IMPORTED)
- layer_source = LayerSource.objects.get(sourcetype=LayerSource.TYPE_IMPORTED)
try:
layer, layer_created = Layer.objects.get_or_create(name=post_data['name'])
except MultipleObjectsReturned:
@@ -1668,7 +1664,6 @@ if True:
if layer:
if layer_created:
- layer.layer_source = layer_source
layer.vcs_url = post_data['vcs_url']
layer.up_date = timezone.now()
layer.save()
@@ -1679,12 +1674,24 @@ if True:
if layer.vcs_url != post_data['vcs_url']:
return HttpResponse(jsonfilter({"error": "hint-layer-exists-with-different-url" , "current_url" : layer.vcs_url, "current_id": layer.id }), content_type = "application/json")
-
- layer_version, version_created = Layer_Version.objects.get_or_create(layer_source=layer_source, layer=layer, project=prj, up_branch_id=up_branch.id,branch=post_data['git_ref'], commit=post_data['git_ref'], dirpath=post_data['dir_path'])
+ layer_version, version_created = \
+ Layer_Version.objects.get_or_create(
+ layer_source=LayerSource.TYPE_IMPORTED,
+ layer=layer, project=prj,
+ release=prj.release,
+ branch=post_data['git_ref'],
+ commit=post_data['git_ref'],
+ dirpath=post_data['dir_path'])
if layer_version:
if not version_created:
- return HttpResponse(jsonfilter({"error": "hint-layer-version-exists", "existing_layer_version": layer_version.id }), content_type = "application/json")
+ return HttpResponse(jsonfilter({"error":
+ "hint-layer-version-exists",
+ "existing_layer_version":
+ layer_version.id }),
+ content_type = "application/json")
+
+ layer_version.layer_source = LayerSource.TYPE_IMPORTED
layer_version.up_date = timezone.now()
layer_version.save()
@@ -2166,20 +2173,33 @@ if True:
}
return render(request, template, context)
+ # TODO merge with api pseudo api here is used for deps modal
@_template_renderer('layerdetails.html')
def layerdetails(request, pid, layerid):
project = Project.objects.get(pk=pid)
layer_version = Layer_Version.objects.get(pk=layerid)
- context = {'project' : project,
- 'layerversion' : layer_version,
- 'layerdeps' : {"list": [{"id": dep.id,
- "name": dep.layer.name,
- "layerdetailurl": reverse('layerdetails', args=(pid, dep.pk)),
- "vcs_url": dep.layer.vcs_url,
- "vcs_reference": dep.get_vcs_reference()} \
- for dep in layer_version.get_alldeps(project.id)]},
- 'projectlayers': [player.layercommit.id for player in ProjectLayer.objects.filter(project=project)]
+ project_layers = ProjectLayer.objects.filter(
+ project=project).values_list("layercommit_id",
+ flat=True)
+
+ context = {
+ 'project': project,
+ 'layer_source': LayerSource.types_dict(),
+ 'layerversion': layer_version,
+ 'layerdeps': {
+ "list": [
+ {
+ "id": dep.id,
+ "name": dep.layer.name,
+ "layerdetailurl": reverse('layerdetails',
+ args=(pid, dep.pk)),
+ "vcs_url": dep.layer.vcs_url,
+ "vcs_reference": dep.get_vcs_reference()
+ }
+ for dep in layer_version.get_alldeps(project.id)]
+ },
+ 'projectlayers': list(project_layers)
}
return context
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 6/9] toaster: tests: Remove references to LayerSource model
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
` (4 preceding siblings ...)
2016-07-07 17:21 ` [PATCH v2 5/9] toaster: Replace references to LayerSource models Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 7/9] toaster: lsupdates Add progress information and clean up logging Michael Wood
` (2 subsequent siblings)
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
Remove and replace layersource model references in the tests and test
data. Remove the orm/test as this only tested LayerSource interactions
which have now been removed.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
.../bldcontrol/management/commands/loadconf.py | 18 ---
bitbake/lib/toaster/orm/tests.py | 180 ---------------------
.../tests/browser/test_layerdetails_page.py | 2 +-
.../fixtures/toastergui-unittest-data.xml | 39 -----
bitbake/lib/toaster/toastergui/tests.py | 3 +-
5 files changed, 2 insertions(+), 240 deletions(-)
delete mode 100644 bitbake/lib/toaster/orm/tests.py
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
index 6f08f1d..1f57fc1 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -9,24 +9,6 @@ from .checksettings import DN
import logging
logger = logging.getLogger("toaster")
-def _reduce_canon_path(path):
- components = []
- for c in path.split("/"):
- if c == "..":
- del components[-1]
- elif c == ".":
- pass
- else:
- components.append(c)
- if len(components) < 2:
- components.append('')
- return "/".join(components)
-
-def _get_id_for_sourcetype(s):
- for i in LayerSource.SOURCE_TYPE:
- if s == i[1]:
- return i[0]
- raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE ))))
class Command(BaseCommand):
help = "Loads a toasterconf.json file in the database"
diff --git a/bitbake/lib/toaster/orm/tests.py b/bitbake/lib/toaster/orm/tests.py
deleted file mode 100644
index 719266e..0000000
--- a/bitbake/lib/toaster/orm/tests.py
+++ /dev/null
@@ -1,180 +0,0 @@
-#! /usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013-2015 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""Test cases for Toaster ORM."""
-
-from django.test import TestCase, TransactionTestCase
-from orm.models import LocalLayerSource, LayerIndexLayerSource, ImportedLayerSource, LayerSource
-from orm.models import Branch, LayerVersionDependency
-
-from orm.models import Project, Layer, Layer_Version, Branch, ProjectLayer
-from orm.models import Release, ReleaseLayerSourcePriority, BitbakeVersion
-
-from django.db import IntegrityError
-
-import os
-
-# set TTS_LAYER_INDEX to the base url to use a different instance of the layer index
-
-class LayerSourceVerifyInheritanceSaveLoad(TestCase):
- """
- Tests to verify inheritance for the LayerSource proxy-inheritance classes.
- """
- def test_object_creation(self):
- """Test LayerSource object creation."""
- for name, sourcetype in [("a1", LayerSource.TYPE_LOCAL),
- ("a2", LayerSource.TYPE_LAYERINDEX),
- ("a3", LayerSource.TYPE_IMPORTED)]:
- LayerSource.objects.create(name=name, sourcetype=sourcetype)
-
- objects = LayerSource.objects.all()
- self.assertTrue(isinstance(objects[0], LocalLayerSource))
- self.assertTrue(isinstance(objects[1], LayerIndexLayerSource))
- self.assertTrue(isinstance(objects[2], ImportedLayerSource))
-
- def test_duplicate_error(self):
- """Test creation of duplicate LayerSource objects."""
- stype = LayerSource.TYPE_LOCAL
- LayerSource.objects.create(name="a1", sourcetype=stype)
- with self.assertRaises(IntegrityError):
- LayerSource.objects.create(name="a1", sourcetype=stype)
-
-
-class LILSUpdateTestCase(TransactionTestCase):
- """Test Layer Source update."""
-
- def setUp(self):
- """Create release."""
- bbv = BitbakeVersion.objects.create(\
- name="master", giturl="git://git.openembedded.org/bitbake")
- Release.objects.create(name="default-release", bitbake_version=bbv,
- branch_name="master")
-
- def test_update(self):
- """Check if LayerSource.update can fetch branches."""
- url = os.getenv("TTS_LAYER_INDEX",
- default="http://layers.openembedded.org/")
-
- lsobj = LayerSource.objects.create(\
- name="b1", sourcetype=LayerSource.TYPE_LAYERINDEX,
- apiurl=url + "layerindex/api/")
- lsobj.update()
- self.assertTrue(lsobj.branch_set.all().count() > 0,
- "no branches fetched")
-
-class LayerVersionEquivalenceTestCase(TestCase):
- """Verify Layer_Version priority selection."""
-
- def setUp(self):
- """Create required objects."""
- # create layer source
- self.lsrc = LayerSource.objects.create(name="dummy-layersource",
- sourcetype=LayerSource.TYPE_LOCAL)
- # create release
- bbv = BitbakeVersion.objects.create(\
- name="master", giturl="git://git.openembedded.org/bitbake")
- self.release = Release.objects.create(name="default-release",
- bitbake_version=bbv,
- branch_name="master")
- # attach layer source to release
- ReleaseLayerSourcePriority.objects.create(\
- release=self.release, layer_source=self.lsrc, priority=1)
-
- # create a layer version for the layer on the specified branch
- self.layer = Layer.objects.create(name="meta-testlayer",
- layer_source=self.lsrc)
- self.branch = Branch.objects.create(name="master", layer_source=self.lsrc)
- self.lver = Layer_Version.objects.create(\
- layer=self.layer, layer_source=self.lsrc, up_branch=self.branch)
-
- # create project and project layer
- self.project = Project.objects.create_project(name="test-project",
- release=self.release)
- ProjectLayer.objects.create(project=self.project,
- layercommit=self.lver)
-
- # create spoof layer that should not appear in the search results
- layer = Layer.objects.create(name="meta-notvalid",
- layer_source=self.lsrc)
- self.lver2 = Layer_Version.objects.create(layer=layer,
- layer_source=self.lsrc,
- up_branch=self.branch)
-
- def test_single_layersource(self):
- """
- When we have a single layer version,
- get_equivalents_wpriority() should return a list with
- just this layer_version.
- """
- equivqs = self.lver.get_equivalents_wpriority(self.project)
- self.assertEqual(list(equivqs), [self.lver])
-
- def test_dual_layersource(self):
- """
- If we have two layers with the same name, from different layer sources,
- we expect both layers in, in increasing priority of the layer source.
- """
- lsrc2 = LayerSource.objects.create(\
- name="dummy-layersource2",
- sourcetype=LayerSource.TYPE_LOCAL,
- apiurl="test")
-
- # assign a lower priority for the second layer source
- self.release.releaselayersourcepriority_set.create(layer_source=lsrc2,
- priority=2)
-
- # create a new layer_version for a layer with the same name
- # coming from the second layer source
- layer2 = Layer.objects.create(name="meta-testlayer",
- layer_source=lsrc2)
- lver2 = Layer_Version.objects.create(layer=layer2, layer_source=lsrc2,
- up_branch=self.branch)
-
- # expect two layer versions, in the priority order
- equivqs = self.lver.get_equivalents_wpriority(self.project)
- self.assertEqual(list(equivqs), [lver2, self.lver])
-
- def test_compatible_layer_versions(self):
- """
- When we have a 2 layer versions, get_all_compatible_layerversions()
- should return a queryset with both.
- """
- compat_lv = self.project.get_all_compatible_layer_versions()
- self.assertEqual(list(compat_lv), [self.lver, self.lver2])
-
- def test_layerversion_get_alldeps(self):
- """Test Layer_Version.get_alldeps API."""
- lvers = {}
- for i in range(10):
- name = "layer%d" % i
- lvers[name] = Layer_Version.objects.create(layer=Layer.objects.create(name=name),
- project=self.project)
- if i:
- LayerVersionDependency.objects.create(layer_version=lvers["layer%d" % (i - 1)],
- depends_on=lvers[name])
- # Check dinamically added deps
- self.assertEqual(lvers['layer0'].get_alldeps(self.project.id),
- [lvers['layer%d' % n] for n in range(1, i+1)])
-
- # Check chain of deps created in previous loop
- for i in range(10):
- self.assertEqual(lvers['layer%d' % i].get_alldeps(self.project.id),
- [lvers['layer%d' % n] for n in range(i+1, 10)])
diff --git a/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py b/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
index 71c5c12..0005192 100644
--- a/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
@@ -63,7 +63,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
self.imported_layer_version = Layer_Version.objects.create(
layer=imported_layer,
- layer_source=layer_source,
+ layer_source=LayerSource.TYPE_IMPORTED,
branch=gitrev,
commit=gitrev,
dirpath=subdir,
diff --git a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
index 213bf0f..80c1e19 100644
--- a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
+++ b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
@@ -195,8 +195,6 @@
<field to="orm.customimagerecipe" name="recipe_appends" rel="ManyToManyRel"><object pk="3"></object></field>
</object>
<object pk="1" model="orm.recipe">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">z recipe</field>
<field type="CharField" name="version">5.2</field>
@@ -212,8 +210,6 @@
<field type="BooleanField" name="is_image">False</field>
</object>
<object pk="2" model="orm.recipe">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">a recipe</field>
<field type="CharField" name="version">1.2</field>
@@ -229,8 +225,6 @@
<field type="BooleanField" name="is_image">False</field>
</object>
<object pk="3" model="orm.recipe">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">a custom recipe</field>
<field type="CharField" name="version"></field>
@@ -246,8 +240,6 @@
<field type="BooleanField" name="is_image">False</field>
</object>
<object pk="4" model="orm.recipe">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">a image recipe</field>
<field type="CharField" name="version">1.2</field>
@@ -263,8 +255,6 @@
<field type="BooleanField" name="is_image">True</field>
</object>
<object pk="5" model="orm.recipe">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">z image recipe</field>
<field type="CharField" name="version">1.3</field>
@@ -280,8 +270,6 @@
<field type="BooleanField" name="is_image">True</field>
</object>
<object pk="6" model="orm.recipe">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">z custom recipe</field>
<field type="CharField" name="version"></field>
@@ -307,34 +295,23 @@
</object>
<object pk="1" model="orm.machine">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
<field type="CharField" name="name">a machine</field>
<field type="CharField" name="description">a machine</field>
</object>
<object pk="2" model="orm.machine">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">2</field>
<field type="CharField" name="name">z machine</field>
<field type="CharField" name="description">z machine</field>
</object>
<object pk="3" model="orm.machine">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
<field type="CharField" name="name">g machine</field>
<field type="CharField" name="description">g machine</field>
</object>
- <object pk="1" model="orm.layersource">
- <field type="CharField" name="name">local</field>
- <field type="IntegerField" name="sourcetype">1</field>
- <field type="CharField" name="apiurl"></field>
- </object>
<object pk="1" model="orm.bitbakeversion">
<field type="CharField" name="name">test bbv</field>
<field type="CharField" name="giturl">/tmp/</field>
@@ -355,26 +332,16 @@
<field type="CharField" name="branch_name">master</field>
<field type="TextField" name="helptext"><None></None></field>
</object>
- <object pk="1" model="orm.releaselayersourcepriority">
- <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="priority">0</field>
- </object>
<object pk="1" model="orm.branch">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
<field type="CharField" name="name">master</field>
<field type="CharField" name="short_description"></field>
</object>
<object pk="1" model="orm.layer">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">a layer</field>
<field type="CharField" name="vcs_url">/tmp/</field>
</object>
<object pk="2" model="orm.layer">
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">z layer</field>
<field type="CharField" name="layer_index_url"></field>
@@ -383,8 +350,6 @@
<object pk="1" model="orm.layer_version">
<field to="orm.build" name="build" rel="ManyToOneRel">1</field>
<field to="orm.layer" name="layer" rel="ManyToOneRel">1</field>
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
<field type="CharField" name="branch">master</field>
@@ -397,8 +362,6 @@
<object pk="2" model="orm.layer_version">
<field to="orm.build" name="build" rel="ManyToOneRel"><None></None></field>
<field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
<field type="CharField" name="branch">testing-branch</field>
@@ -411,8 +374,6 @@
<object pk="3" model="orm.layer_version">
<field to="orm.build" name="build" rel="ManyToOneRel">1</field>
<field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
- <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
- <field type="IntegerField" name="up_id"><None></None></field>
<field type="DateTimeField" name="up_date"><None></None></field>
<field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
<field type="CharField" name="branch">testing-branch</field>
diff --git a/bitbake/lib/toaster/toastergui/tests.py b/bitbake/lib/toaster/toastergui/tests.py
index da530a1..03b73a9 100644
--- a/bitbake/lib/toaster/toastergui/tests.py
+++ b/bitbake/lib/toaster/toastergui/tests.py
@@ -28,7 +28,7 @@ from django.utils import timezone
from django.db.models import Q
from orm.models import Project, Release, BitbakeVersion, Package, LogMessage
-from orm.models import ReleaseLayerSourcePriority, LayerSource, Layer, Build
+from orm.models import LayerSource, Layer, Build
from orm.models import Layer_Version, Recipe, Machine, ProjectLayer, Target
from orm.models import CustomImageRecipe, ProjectVariable
from orm.models import Branch, CustomImagePackage
@@ -149,7 +149,6 @@ class ViewTests(TestCase):
def test_xhr_import_layer(self):
"""Test xhr_importlayer API"""
- LayerSource.objects.create(sourcetype=LayerSource.TYPE_IMPORTED)
#Test for importing an already existing layer
args = {'vcs_url' : "git://git.example.com/test",
'name' : "base-layer",
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 7/9] toaster: lsupdates Add progress information and clean up logging
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
` (5 preceding siblings ...)
2016-07-07 17:21 ` [PATCH v2 6/9] toaster: tests: Remove references to LayerSource model Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 8/9] toaster: orm Remove the layerindex specific up_branch fields Michael Wood
2016-07-07 17:21 ` [PATCH v2 9/9] toaster: admin Add Layer_Version to the admin-able models Michael Wood
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
Adds basic progress % information and provides better description of
what is happening.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
.../toaster/orm/management/commands/lsupdates.py | 80 +++++++++++++++-------
1 file changed, 57 insertions(+), 23 deletions(-)
diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index f4eba81..390f056 100644
--- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -25,6 +25,8 @@ from orm.models import LayerSource, Layer, Release, Layer_Version
from orm.models import LayerVersionDependency, Machine, Recipe
import os
+import sys
+
import json
import logging
logger = logging.getLogger("toaster")
@@ -36,6 +38,18 @@ class Command(NoArgsCommand):
args = ""
help = "Updates locally cached information from a layerindex server"
+ def mini_progress(self, what, i, total):
+ i = i + 1
+ pec = (float(i)/float(total))*100
+
+ sys.stdout.write("\rUpdating %s %d%%" %
+ (what,
+ pec))
+ sys.stdout.flush()
+ if int(pec) is 100:
+ sys.stdout.write("\n")
+ sys.stdout.flush()
+
def update(self):
"""
Fetches layer, recipe and machine information from a layerindex
@@ -85,7 +99,8 @@ class Command(NoArgsCommand):
if len(whitelist_branch_names) == 0:
raise Exception("Failed to make list of branches to fetch")
- logger.debug("Fetching branches")
+ logger.info("Fetching metadata releases for %s",
+ " ".join(whitelist_branch_names))
# keep a track of the id mappings so that layer_versions can be created
# for these layers later on
@@ -96,10 +111,10 @@ class Command(NoArgsCommand):
# "?filter=name:%s"
# % "OR".join(whitelist_branch_names))
- # update layers
layers_info = _get_json_response(apilinks['layerItems'])
- for li in layers_info:
+ total = len(layers_info)
+ for i, li in enumerate(layers_info):
# Special case for the openembedded-core layer
if li['name'] == oe_core_layer:
try:
@@ -113,25 +128,33 @@ class Command(NoArgsCommand):
oe_core_l.description = li['description']
oe_core_l.save()
li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
+ self.mini_progress("layers", i, total)
continue
except Layer.DoesNotExist:
pass
- l, created = Layer.objects.get_or_create(name=li['name'])
- l.up_date = li['updated']
- l.vcs_url = li['vcs_url']
- l.vcs_web_url = li['vcs_web_url']
- l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
- l.vcs_web_file_base_url = li['vcs_web_file_base_url']
- l.summary = li['summary']
- l.description = li['description']
- l.save()
+ try:
+ l, created = Layer.objects.get_or_create(name=li['name'])
+ l.up_date = li['updated']
+ l.vcs_url = li['vcs_url']
+ l.vcs_web_url = li['vcs_web_url']
+ l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+ l.vcs_web_file_base_url = li['vcs_web_file_base_url']
+ l.summary = li['summary']
+ l.description = li['description']
+ l.save()
+ except Layer.MultipleObjectsReturned:
+ logger.info("Skipped %s as we found multiple layers and "
+ "don't know which to update" %
+ li['name'])
li_layer_id_to_toaster_layer_id[li['id']] = l.pk
- # update layerbranches/layer_versions
- logger.debug("Fetching layer information")
+ self.mini_progress("layers", i, total)
+
+ # update layer_versions
+ logger.info("Fetching layer versions")
layerbranches_info = _get_json_response(
apilinks['layerBranches'] + "?filter=branch__name:%s" %
"OR".join(whitelist_branch_names))
@@ -140,7 +163,8 @@ class Command(NoArgsCommand):
# layer_version toaster object id
li_layer_branch_id_to_toaster_lv_id = {}
- for lbi in layerbranches_info:
+ total = len(layerbranches_info)
+ for i, lbi in enumerate(layerbranches_info):
try:
lv, created = Layer_Version.objects.get_or_create(
@@ -149,8 +173,9 @@ class Command(NoArgsCommand):
pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
)
except KeyError:
- print("No such layerindex layer referenced by layerbranch %d" %
- lbi['layer'])
+ logger.warning(
+ "No such layerindex layer referenced by layerbranch %d" %
+ lbi['layer'])
continue
lv.up_date = lbi['updated']
@@ -160,7 +185,9 @@ class Command(NoArgsCommand):
li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
lv.pk
+ self.mini_progress("layer versions", i, total)
+ logger.info("Fetching layer version dependencies")
# update layer dependencies
layerdependencies_info = _get_json_response(
apilinks['layerDependencies'] +
@@ -190,19 +217,22 @@ class Command(NoArgsCommand):
"up_id:%s lv:%s" %
(self, ldi['dependency'], lv))
- for lv in dependlist:
+ total = len(dependlist)
+ for i, lv in enumerate(dependlist):
LayerVersionDependency.objects.filter(layer_version=lv).delete()
for lvd in dependlist[lv]:
LayerVersionDependency.objects.get_or_create(layer_version=lv,
depends_on=lvd)
+ self.mini_progress("Layer version dependencies", i, total)
# update machines
- logger.debug("Fetching machine information")
+ logger.info("Fetching machine information")
machines_info = _get_json_response(
apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
"OR".join(whitelist_branch_names))
- for mi in machines_info:
+ total = len(machines_info)
+ for i, mi in enumerate(machines_info):
mo, created = Machine.objects.get_or_create(
layer_version=Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
@@ -210,14 +240,16 @@ class Command(NoArgsCommand):
mo.name = mi['name']
mo.description = mi['description']
mo.save()
+ self.mini_progress("machines", i, total)
# update recipes; paginate by layer version / layer branch
- logger.debug("Fetching target information")
+ logger.info("Fetching recipe information")
recipes_info = _get_json_response(
apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
"OR".join(whitelist_branch_names))
- for ri in recipes_info:
+ total = len(recipes_info)
+ for i, ri in enumerate(recipes_info):
try:
lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
lv = Layer_Version.objects.get(pk=lv_id)
@@ -244,7 +276,9 @@ class Command(NoArgsCommand):
ro.is_image = "-image-" in ri['pn']
ro.save()
except Exception as e:
- logger.debug("Failed saving recipe %s", e)
+ logger.warning("Failed saving recipe %s", e)
+
+ self.mini_progress("recipes", i, total)
def handle_noargs(self, **options):
self.update()
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 8/9] toaster: orm Remove the layerindex specific up_branch fields
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
` (6 preceding siblings ...)
2016-07-07 17:21 ` [PATCH v2 7/9] toaster: lsupdates Add progress information and clean up logging Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
2016-07-07 17:21 ` [PATCH v2 9/9] toaster: admin Add Layer_Version to the admin-able models Michael Wood
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
We don't need to keep track of layerindex data in our database. And
using branch==release is very confusing in the schema. Instead use the
existing Release definition to keep track of which release a
layer_version is for.
Remove the Branch model and all references to it.
Create a migration path to convert from up_branches to their
corresponding releases.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
bitbake/lib/bb/ui/buildinfohelper.py | 6 +--
.../bldcontrol/management/commands/loadconf.py | 2 +-
.../toaster/orm/management/commands/lsupdates.py | 23 ++++++---
.../0010_use_release_instead_of_up_branch.py | 60 ++++++++++++++++++++++
bitbake/lib/toaster/orm/models.py | 32 +++---------
.../fixtures/toastergui-unittest-data.xml | 21 +++++---
bitbake/lib/toaster/toastergui/tests.py | 2 +-
bitbake/lib/toaster/toastergui/views.py | 8 +--
8 files changed, 106 insertions(+), 48 deletions(-)
create mode 100644 bitbake/lib/toaster/orm/migrations/0010_use_release_instead_of_up_branch.py
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 447670c..c56e24d 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -372,7 +372,7 @@ class ORMWrapper(object):
layer_copy, c = Layer_Version.objects.get_or_create(
build=build_obj,
layer=layer_obj.layer,
- up_branch=layer_obj.up_branch,
+ release=layer_obj.release,
branch=layer_version_information['branch'],
commit=layer_version_information['commit'],
local_path=layer_version_information['local_path'],
@@ -604,8 +604,8 @@ class ORMWrapper(object):
Recipe,
name=built_recipe.name,
layer_version__build=None,
- layer_version__up_branch=
- built_recipe.layer_version.up_branch,
+ layer_version__release=
+ built_recipe.layer_version.release,
file_path=built_recipe.file_path,
version=built_recipe.version
)
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
index 1f57fc1..59324ac 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -1,5 +1,5 @@
from django.core.management.base import BaseCommand, CommandError
-from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
+from orm.models import LayerSource, ToasterSetting, Layer, Layer_Version
from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer
from django.db import IntegrityError
import os
diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 390f056..026ad2c 100644
--- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -102,14 +102,24 @@ class Command(NoArgsCommand):
logger.info("Fetching metadata releases for %s",
" ".join(whitelist_branch_names))
- # keep a track of the id mappings so that layer_versions can be created
- # for these layers later on
+ branches_info = _get_json_response(apilinks['branches'] +
+ "?filter=name:%s"
+ % "OR".join(whitelist_branch_names))
+
+ # Map the layer index branches to toaster releases
+ li_branch_id_to_toaster_release = {}
+
+ total = len(branches_info)
+ for i, branch in enumerate(branches_info):
+ li_branch_id_to_toaster_release[branch['id']] = \
+ Release.objects.get(name=branch['name'])
+ self.mini_progress("Releases", i, total)
+
+ # keep a track of the layerindex (li) id mappings so that
+ # layer_versions can be created for these layers later on
li_layer_id_to_toaster_layer_id = {}
- # We may need this? TODO
- #branches_info = _get_json_response(apilinks['branches'] +
- # "?filter=name:%s"
- # % "OR".join(whitelist_branch_names))
+ logger.info("Fetching layers")
layers_info = _get_json_response(apilinks['layerItems'])
@@ -178,6 +188,7 @@ class Command(NoArgsCommand):
lbi['layer'])
continue
+ lv.release = li_branch_id_to_toaster_release[lbi['branch']]
lv.up_date = lbi['updated']
lv.commit = lbi['actual_branch']
lv.dirpath = lbi['vcs_subdir']
diff --git a/bitbake/lib/toaster/orm/migrations/0010_use_release_instead_of_up_branch.py b/bitbake/lib/toaster/orm/migrations/0010_use_release_instead_of_up_branch.py
new file mode 100644
index 0000000..0e95cad
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0010_use_release_instead_of_up_branch.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.db.models import Q
+
+
+def branch_to_release(apps, schema_editor):
+ Layer_Version = apps.get_model('orm', 'Layer_Version')
+ Release = apps.get_model('orm', 'Release')
+
+ print("Converting all layer version up_branches to releases")
+ # Find all the layer versions which have an upbranch and convert them to
+ # the release that they're for.
+ for layer_version in Layer_Version.objects.filter(
+ Q(release=None) & ~Q(up_branch=None)):
+ try:
+ # HEAD and local are equivalent
+ if "HEAD" in layer_version.up_branch.name:
+ release = Release.objects.get(name="local")
+ else:
+ release = Release.objects.get(
+ name=layer_version.up_branch.name)
+
+ layer_version.release = release
+ layer_version.save()
+ except Exception as e:
+ print("Couldn't work out an appropriate release for %s "
+ "the up_branch was %s "
+ "user the django admin interface to correct it" %
+ (layer_version.layer.name, layer_version.up_branch.name))
+ print(e)
+
+ continue
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0009_delete_layersource'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='layer_version',
+ name='release',
+ field=models.ForeignKey(to='orm.Release', default=None, null=True),
+ ),
+ migrations.RunPython(branch_to_release,
+ reverse_code=migrations.RunPython.noop),
+
+ migrations.RemoveField(
+ model_name='layer_version',
+ name='up_branch',
+ ),
+
+ migrations.DeleteModel(
+ name='Branch',
+ ),
+ ]
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index cdcd7a7..b42071b 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -144,7 +144,7 @@ class ProjectManager(models.Manager):
for rdl in release.releasedefaultlayer_set.all():
lv = Layer_Version.objects.filter(
layer__name=rdl.layer_name,
- up_branch__name=release.branch_name).first()
+ release=release).first()
if lv:
ProjectLayer.objects.create(project=prj,
@@ -280,7 +280,7 @@ class Project(models.Model):
# guard on release, as it can be null
if self.release:
queryset = Layer_Version.objects.filter(
- (Q(up_branch__name=self.release.branch_name) &
+ (Q(release=self.release) &
Q(build=None) &
Q(project=None)) |
Q(project=self))
@@ -1102,22 +1102,6 @@ class ReleaseDefaultLayer(models.Model):
layer_name = models.CharField(max_length=100, default="")
-# Branch class is synced with layerindex.Branch, branches can only come
-# from remote layer indexes
-class Branch(models.Model):
- # id of branch in the layerindex
- up_date = models.DateTimeField(null=True, default=None)
-
- name = models.CharField(max_length=50)
- short_description = models.CharField(max_length=50, blank=True)
-
- class Meta:
- verbose_name_plural = "Branches"
-
- def __unicode__(self):
- return self.name
-
-
class LayerSource(object):
""" Where the layer metadata came from """
TYPE_LOCAL = 0
@@ -1166,7 +1150,7 @@ class Layer_Version(models.Model):
"""
search_allowed_fields = ["layer__name", "layer__summary",
"layer__description", "layer__vcs_url",
- "dirpath", "up_branch__name", "commit", "branch"]
+ "dirpath", "release__name", "commit", "branch"]
build = models.ForeignKey(Build, related_name='layer_version_build',
default=None, null=True)
@@ -1178,8 +1162,8 @@ class Layer_Version(models.Model):
up_date = models.DateTimeField(null=True, default=timezone.now)
- # layerindex specific field
- up_branch = models.ForeignKey(Branch, null=True, default=None)
+ # To which metadata release does this layer version belong to
+ release = models.ForeignKey(Release, null=True, default=None)
branch = models.CharField(max_length=80)
commit = models.CharField(max_length=100)
@@ -1213,7 +1197,7 @@ class Layer_Version(models.Model):
extra_path = self.dirpath
else:
extra_path = path
- branchname = self.up_branch.name
+ branchname = self.release.name
url = base_url.replace('%branch%', branchname)
# If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
@@ -1249,8 +1233,8 @@ class Layer_Version(models.Model):
def get_vcs_reference(self):
if self.branch is not None and len(self.branch) > 0:
return self.branch
- if self.up_branch is not None:
- return self.up_branch.name
+ if self.release is not None:
+ return self.release.name
if self.commit is not None and len(self.commit) > 0:
return self.commit
return 'N/A'
diff --git a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
index 80c1e19..4517ed1 100644
--- a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
+++ b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
@@ -1,5 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<django-objects version="1.0">
+ <object pk="1" model="orm.bitbakeversion">
+ <field type="CharField" name="name">v2.3</field>
+ <field type="GitURLField" name="giturl">git://git.openembedded.org/bitbake</field>
+ <field type="CharField" name="dirpath">b</field>
+ <field type="CharField" name="branch">a</field>
+ </object>
+ <object pk="1" model="orm.release">
+ <field type="CharField" name="name">master</field>
+ <field type="CharField" name="description">master project</field>
+ <field to="orm.bitbake_version" name="bitbake_version">1</field>
+ </object>
<object pk="1" model="orm.project">
<field type="CharField" name="name">a test project</field>
<field type="CharField" name="short_description"></field>
@@ -332,10 +343,6 @@
<field type="CharField" name="branch_name">master</field>
<field type="TextField" name="helptext"><None></None></field>
</object>
- <object pk="1" model="orm.branch">
- <field type="CharField" name="name">master</field>
- <field type="CharField" name="short_description"></field>
- </object>
<object pk="1" model="orm.layer">
<field type="DateTimeField" name="up_date"><None></None></field>
<field type="CharField" name="name">a layer</field>
@@ -351,7 +358,7 @@
<field to="orm.build" name="build" rel="ManyToOneRel">1</field>
<field to="orm.layer" name="layer" rel="ManyToOneRel">1</field>
<field type="DateTimeField" name="up_date"><None></None></field>
- <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+ <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
<field type="CharField" name="branch">master</field>
<field type="CharField" name="commit">abcdef123</field>
<field type="CharField" name="dirpath">/tmp/</field>
@@ -363,7 +370,7 @@
<field to="orm.build" name="build" rel="ManyToOneRel"><None></None></field>
<field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
<field type="DateTimeField" name="up_date"><None></None></field>
- <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+ <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
<field type="CharField" name="branch">testing-branch</field>
<field type="CharField" name="commit">9876fedcba</field>
<field type="CharField" name="dirpath"><None></None></field>
@@ -375,7 +382,7 @@
<field to="orm.build" name="build" rel="ManyToOneRel">1</field>
<field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
<field type="DateTimeField" name="up_date"><None></None></field>
- <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+ <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
<field type="CharField" name="branch">testing-branch</field>
<field type="CharField" name="commit">9876fedcba</field>
<field type="CharField" name="dirpath"><None></None></field>
diff --git a/bitbake/lib/toaster/toastergui/tests.py b/bitbake/lib/toaster/toastergui/tests.py
index 03b73a9..61ac477 100644
--- a/bitbake/lib/toaster/toastergui/tests.py
+++ b/bitbake/lib/toaster/toastergui/tests.py
@@ -31,7 +31,7 @@ from orm.models import Project, Release, BitbakeVersion, Package, LogMessage
from orm.models import LayerSource, Layer, Build
from orm.models import Layer_Version, Recipe, Machine, ProjectLayer, Target
from orm.models import CustomImageRecipe, ProjectVariable
-from orm.models import Branch, CustomImagePackage
+from orm.models import CustomImagePackage
import toastermain
import inspect
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index c76e63a..5533e61 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -160,7 +160,7 @@ def _lv_to_dict(prj, x = None):
return {"id": x.pk,
"name": x.layer.name,
"tooltip": "%s | %s" % (x.layer.vcs_url,x.get_vcs_reference()),
- "detail": "(%s" % x.layer.vcs_url + (")" if x.up_branch == None else " | "+x.get_vcs_reference()+")"),
+ "detail": "(%s" % x.layer.vcs_url + (")" if x.release == None else " | "+x.get_vcs_reference()+")"),
"giturl": x.layer.vcs_url,
"layerdetailurl" : reverse('layerdetails', args=(prj.id,x.pk)),
"revision" : x.get_vcs_reference(),
@@ -1275,7 +1275,7 @@ if True:
from django.contrib.auth import authenticate, login
from django.contrib.auth.decorators import login_required
- from orm.models import Branch, LayerSource, ToasterSetting, Release, Machine, LayerVersionDependency
+ from orm.models import LayerSource, ToasterSetting, Release, Machine, LayerVersionDependency
from bldcontrol.models import BuildRequest
import traceback
@@ -1654,9 +1654,6 @@ if True:
post_data[key] = val.strip()
- # We need to know what release the current project is so that we
- # can set the imported layer's up_branch_id
-
try:
layer, layer_created = Layer.objects.get_or_create(name=post_data['name'])
except MultipleObjectsReturned:
@@ -1746,7 +1743,6 @@ if True:
return HttpResponse(jsonfilter(json_response), content_type = "application/json")
-
@xhr_response
def xhr_customrecipe(request):
"""
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 9/9] toaster: admin Add Layer_Version to the admin-able models
2016-07-07 17:21 [PATCH v2 0/9] database clean ups Michael Wood
` (7 preceding siblings ...)
2016-07-07 17:21 ` [PATCH v2 8/9] toaster: orm Remove the layerindex specific up_branch fields Michael Wood
@ 2016-07-07 17:21 ` Michael Wood
8 siblings, 0 replies; 11+ messages in thread
From: Michael Wood @ 2016-07-07 17:21 UTC (permalink / raw)
To: toaster
If the migration didn't get the release conversion right for say, a
local or imported layer it would be handy to be able to edit this
in the django admin page.
Also useful for developers to be able to tweak layers on the fly.
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
---
bitbake/lib/toaster/bldcollector/admin.py | 18 +++++++++++-------
bitbake/lib/toaster/orm/models.py | 11 +++++++++++
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/bitbake/lib/toaster/bldcollector/admin.py b/bitbake/lib/toaster/bldcollector/admin.py
index 7b69ecd..1f2e07f 100644
--- a/bitbake/lib/toaster/bldcollector/admin.py
+++ b/bitbake/lib/toaster/bldcollector/admin.py
@@ -1,29 +1,33 @@
from django.contrib import admin
-from django.contrib.admin.filters import RelatedFieldListFilter
-from orm.models import BitbakeVersion, Release, ToasterSetting
-from django.forms.widgets import Textarea
+from orm.models import BitbakeVersion, Release, ToasterSetting, Layer_Version
from django import forms
import django.db.models as models
-from django.contrib.admin import widgets, helpers
class BitbakeVersionAdmin(admin.ModelAdmin):
- # we override the formfield for db URLField because of broken URL validation
+ # we override the formfield for db URLField
+ # because of broken URL validation
def formfield_for_dbfield(self, db_field, **kwargs):
if isinstance(db_field, models.fields.URLField):
return forms.fields.CharField()
- return super(BitbakeVersionAdmin, self).formfield_for_dbfield(db_field, **kwargs)
-
+ return super(BitbakeVersionAdmin, self).formfield_for_dbfield(
+ db_field, **kwargs)
class ReleaseAdmin(admin.ModelAdmin):
pass
+
class ToasterSettingAdmin(admin.ModelAdmin):
pass
+
+class LayerVersionsAdmin(admin.ModelAdmin):
+ pass
+
+admin.site.register(Layer_Version, LayerVersionsAdmin)
admin.site.register(BitbakeVersion, BitbakeVersionAdmin)
admin.site.register(Release, ReleaseAdmin)
admin.site.register(ToasterSetting, ToasterSettingAdmin)
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index b42071b..515d911 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -1097,6 +1097,9 @@ class Release(models.Model):
def __unicode__(self):
return "%s (%s)" % (self.name, self.branch_name)
+ def __str__(self):
+ return self.name
+
class ReleaseDefaultLayer(models.Model):
release = models.ForeignKey(Release)
layer_name = models.CharField(max_length=100, default="")
@@ -1264,6 +1267,14 @@ class Layer_Version(models.Model):
def __unicode__(self):
return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
+ def __str__(self):
+ if self.release:
+ release = self.release.name
+ else:
+ release = "No release set"
+
+ return "%d %s (%s)" % (self.pk, self.layer.name, release)
+
class LayerVersionDependency(models.Model):
--
2.7.4
^ permalink raw reply related [flat|nested] 11+ messages in thread