From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: by yocto-www.yoctoproject.org (Postfix, from userid 118) id 4D8D3E00E84; Fri, 8 Jul 2016 03:45:06 -0700 (PDT) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on yocto-www.yoctoproject.org X-Spam-Level: X-Spam-Status: No, score=-2.6 required=5.0 tests=BAYES_00,DKIM_SIGNED, DKIM_VALID,RCVD_IN_DNSWL_LOW autolearn=ham version=3.3.1 X-Spam-HAM-Report: * -0.7 RCVD_IN_DNSWL_LOW RBL: Sender listed at http://www.dnswl.org/, low * trust * [209.85.192.170 listed in list.dnswl.org] * -1.9 BAYES_00 BODY: Bayes spam probability is 0 to 1% * [score: 0.0000] * 0.1 DKIM_SIGNED Message has a DKIM or DK signature, not necessarily * valid * -0.1 DKIM_VALID Message has at least one valid DKIM or DK signature Received: from mail-pf0-f170.google.com (mail-pf0-f170.google.com [209.85.192.170]) by yocto-www.yoctoproject.org (Postfix) with ESMTP id 3CD9EE00575 for ; Fri, 8 Jul 2016 03:45:03 -0700 (PDT) Received: by mail-pf0-f170.google.com with SMTP id t190so13022204pfb.3 for ; Fri, 08 Jul 2016 03:45:03 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=intel-com.20150623.gappssmtp.com; s=20150623; h=subject:to:references:from:message-id:date:user-agent:mime-version :in-reply-to:content-transfer-encoding; bh=wuXIPDErBzsWAM/+TKOPzUwTsn1kklE6kD8NzZebmgc=; b=AjYHPcmokxpAR1OhFEmSurWfbz2hfI9hHCBz1QLpKfh58jTWMRQYtbktrquY9ADR/w oszfV35fje57cJBekSlCjz4DXM3cYBF9DDVZLAZiJkzSRFxtKqzq2ZJTPLinLY1uVPGX lMRWDBbcmRgn1V4UskxqcActOY6sitbYMW0v9EaAnfUptaI/+guvLpqQ1ZeJpoxJY9yl A2wtm6H+eRirNNxqmWbdW4OselrmB3xZ9qf1iR2ETx3Bb04sMsG/HERrlCekqOuRv2T9 ESu6RS9KeTzZfrXoppmQPnIyqLs/vZ08WwMEtbXqBMcdC2btAlckdLawnFVc3o/Eazlv E54Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:subject:to:references:from:message-id:date :user-agent:mime-version:in-reply-to:content-transfer-encoding; bh=wuXIPDErBzsWAM/+TKOPzUwTsn1kklE6kD8NzZebmgc=; b=hc0sfW06TVI/j5fbN9gI0tJ9F44nvlYk0+T9VB0o8G+xRNon+5YGgxbbhSXFkczkKu CGHQSFh6AkhE7bMKsntojsum7a/UVdSxu96oAWiAN5ovasMJybE4MhXuz8UnHXS2h3Xy 7xH9NWmF4/3RZQ80YUK2Io0bsh8oy0bPzANEtuLZ/sfbh67Hpd3FqGK03Pdvj1LsQyvN PVGjJqybyAEYOLvgaZHCPrxwLvy1pYF930AC4PgznSBdxMp+gUczb6ApL+HSiZUBaAU4 Bfwb0IRkSYNoA+KzeWddGEWPOdubW+hfLAUiipHAnK0B+HW4DLZfia1GD05epONXrLd5 WbrQ== X-Gm-Message-State: ALyK8tJtsYQzCjCvb/J04TbajBpRwDaVPULRF/9dkFS3l/W+uoYrSTgBBqOMlneH7hihEefO X-Received: by 10.98.21.149 with SMTP id 143mr7303437pfv.92.1467974703161; Fri, 08 Jul 2016 03:45:03 -0700 (PDT) Received: from [192.168.0.78] (host86-169-85-169.range86-169.btcentralplus.com. [86.169.85.169]) by smtp.googlemail.com with ESMTPSA id um1sm3858647pac.41.2016.07.08.03.45.01 for (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Fri, 08 Jul 2016 03:45:02 -0700 (PDT) To: "toaster@yoctoproject.org" References: <1467912108-24659-1-git-send-email-michael.g.wood@intel.com> <1467912108-24659-5-git-send-email-michael.g.wood@intel.com> From: Michael Wood Message-ID: <577F842C.9000007@intel.com> Date: Fri, 8 Jul 2016 11:45:00 +0100 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Thunderbird/38.8.0 MIME-Version: 1.0 In-Reply-To: <1467912108-24659-5-git-send-email-michael.g.wood@intel.com> Subject: Re: [PATCH v2 4/9] toaster: lsupdates Add layerindex fetcher X-BeenThere: toaster@yoctoproject.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: Web based interface for BitBake List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 08 Jul 2016 10:45:06 -0000 Content-Type: text/plain; charset=windows-1252; format=flowed Content-Transfer-Encoding: 7bit On 07/07/16 18:21, Michael Wood wrote: > Move and refactor the layerindex layer source update mechanism so that > we don't have to track the layerindex objects in the toaster database. > Move this out of the orm and into the management command. > > Paves the way for future improvement to allow you to specify a layer > index server as an argument to the command. > > Signed-off-by: Michael Wood > --- > .../toaster/orm/management/commands/lsupdates.py | 250 ++++++++++++++++++++- > 1 file changed, 244 insertions(+), 6 deletions(-) > > diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py > index 75e9513..f4eba81 100644 > --- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py > +++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py > @@ -1,12 +1,250 @@ > -from django.core.management.base import NoArgsCommand, CommandError > -from orm.models import LayerSource > +# > +# ex:ts=4:sw=4:sts=4:et > +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- > +# > +# BitBake Toaster Implementation > +# > +# Copyright (C) 2016 Intel Corporation > +# > +# This program is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License version 2 as > +# published by the Free Software Foundation. > +# > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License along > +# with this program; if not, write to the Free Software Foundation, Inc., > +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. > + > +from django.core.management.base import NoArgsCommand > + > +from orm.models import LayerSource, Layer, Release, Layer_Version > +from orm.models import LayerVersionDependency, Machine, Recipe > + > import os > +import json > +import logging > +logger = logging.getLogger("toaster") > + > +DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/" > + > > class Command(NoArgsCommand): > - args = "" > - help = "Updates locally cached information from all LayerSources" > + args = "" > + help = "Updates locally cached information from a layerindex server" > + > + def update(self): > + """ > + Fetches layer, recipe and machine information from a layerindex > + server > + """ > + > + self.apiurl = DEFAULT_LAYERINDEX_SERVER > + > + assert self.apiurl is not None > + try: > + from urllib.request import urlopen, URLError > + from urllib.parse import urlparse > + except ImportError: > + from urllib2 import urlopen, URLError > + from urlparse import urlparse > + > + proxy_settings = os.environ.get("http_proxy", None) > + oe_core_layer = 'openembedded-core' > + > + def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER): > + _parsedurl = urlparse(apiurl) > + path = _parsedurl.path > + > + # logger.debug("Fetching %s", apiurl) > + try: > + res = urlopen(apiurl) > + except URLError as e: > + raise Exception("Failed to read %s: %s" % (path, e.reason)) > + > + return json.loads(res.read().decode('utf-8')) > + > + # verify we can get the basic api > + try: > + apilinks = _get_json_response() > + except Exception as e: > + import traceback > + if proxy_settings is not None: > + logger.info("EE: Using proxy %s" % proxy_settings) > + logger.warning("EE: could not connect to %s, skipping update:" > + "%s\n%s" % (self.apiurl, e, traceback.format_exc())) > + return > + > + # update branches; only those that we already have names listed in the > + # Releases table > + whitelist_branch_names = [rel.branch_name > + for rel in Release.objects.all()] > + if len(whitelist_branch_names) == 0: > + raise Exception("Failed to make list of branches to fetch") > + > + logger.debug("Fetching branches") > + > + # keep a track of the id mappings so that layer_versions can be created > + # for these layers later on > + li_layer_id_to_toaster_layer_id = {} > + > + # We may need this? TODO > + #branches_info = _get_json_response(apilinks['branches'] + > + # "?filter=name:%s" > + # % "OR".join(whitelist_branch_names)) > + > + # update layers > + layers_info = _get_json_response(apilinks['layerItems']) > + > + for li in layers_info: > + # Special case for the openembedded-core layer > + if li['name'] == oe_core_layer: > + try: > + # If we have an existing openembedded-core for example > + # from the toasterconf.json augment the info using the > + # layerindex rather than duplicate it > + oe_core_l = Layer.objects.get(name=oe_core_layer) > + # Take ownership of the layer as now coming from the > + # layerindex > + oe_core_l.summary = li['summary'] > + oe_core_l.description = li['description'] > + oe_core_l.save() > + li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk > + continue > + > + except Layer.DoesNotExist: > + pass > + > + l, created = Layer.objects.get_or_create(name=li['name']) > + l.up_date = li['updated'] > + l.vcs_url = li['vcs_url'] > + l.vcs_web_url = li['vcs_web_url'] > + l.vcs_web_tree_base_url = li['vcs_web_tree_base_url'] > + l.vcs_web_file_base_url = li['vcs_web_file_base_url'] > + l.summary = li['summary'] > + l.description = li['description'] > + l.save() > + > + li_layer_id_to_toaster_layer_id[li['id']] = l.pk > + > + # update layerbranches/layer_versions > + logger.debug("Fetching layer information") > + layerbranches_info = _get_json_response( > + apilinks['layerBranches'] + "?filter=branch__name:%s" % > + "OR".join(whitelist_branch_names)) > + > + # Map Layer index layer_branch object id to > + # layer_version toaster object id > + li_layer_branch_id_to_toaster_lv_id = {} > + > + for lbi in layerbranches_info: > + > + try: > + lv, created = Layer_Version.objects.get_or_create( > + layer_source=LayerSource.TYPE_LAYERINDEX, > + layer=Layer.objects.get( > + pk=li_layer_id_to_toaster_layer_id[lbi['layer']]) > + ) > + except KeyError: > + print("No such layerindex layer referenced by layerbranch %d" % > + lbi['layer']) > + continue > + > + lv.up_date = lbi['updated'] > + lv.commit = lbi['actual_branch'] > + lv.dirpath = lbi['vcs_subdir'] > + lv.save() > + > + li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\ > + lv.pk > + > + # update layer dependencies > + layerdependencies_info = _get_json_response( > + apilinks['layerDependencies'] + > + "?filter=layerbranch__branch__name:%s" % > + "OR".join(whitelist_branch_names)) > + > + dependlist = {} > + for ldi in layerdependencies_info: > + try: > + lv = Layer_Version.objects.get( > + pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']]) > + except Layer_Version.DoesNotExist as e: > + continue > + > + if lv not in dependlist: > + dependlist[lv] = [] > + try: > + layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']] > + > + dependlist[lv].append( > + Layer_Version.objects.get( > + layer_source=LayerSource.TYPE_LAYERINDEX, > + layer__pk=layer_id)) > + > + except Layer_Version.DoesNotExist: > + logger.warning("Cannot find layer version (ls:%s)," > + "up_id:%s lv:%s" % > + (self, ldi['dependency'], lv)) > + > + for lv in dependlist: > + LayerVersionDependency.objects.filter(layer_version=lv).delete() > + for lvd in dependlist[lv]: > + LayerVersionDependency.objects.get_or_create(layer_version=lv, > + depends_on=lvd) > + > + # update machines > + logger.debug("Fetching machine information") > + machines_info = _get_json_response( > + apilinks['machines'] + "?filter=layerbranch__branch__name:%s" % > + "OR".join(whitelist_branch_names)) > + > + for mi in machines_info: > + mo, created = Machine.objects.get_or_create( > + layer_version=Layer_Version.objects.get( > + pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']])) This was missing the name attribute and should have been: + mo, created = Machine.objects.get_or_create( + name=mi['name'], + layer_version=Layer_Version.objects.get( + pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']])) It has been corrected on the branch > + mo.up_date = mi['updated'] > + mo.name = mi['name'] > + mo.description = mi['description'] > + mo.save() > + > + # update recipes; paginate by layer version / layer branch > + logger.debug("Fetching target information") > + recipes_info = _get_json_response( > + apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" % > + "OR".join(whitelist_branch_names)) > + > + for ri in recipes_info: > + try: > + lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']] > + lv = Layer_Version.objects.get(pk=lv_id) > + > + ro, created = Recipe.objects.get_or_create( > + layer_version=lv, > + name=ri['pn'] > + ) > > + ro.layer_version = lv > + ro.up_date = ri['updated'] > + ro.name = ri['pn'] > + ro.version = ri['pv'] > + ro.summary = ri['summary'] > + ro.description = ri['description'] > + ro.section = ri['section'] > + ro.license = ri['license'] > + ro.homepage = ri['homepage'] > + ro.bugtracker = ri['bugtracker'] > + ro.file_path = ri['filepath'] + "/" + ri['filename'] > + if 'inherits' in ri: > + ro.is_image = 'image' in ri['inherits'].split() > + else: # workaround for old style layer index > + ro.is_image = "-image-" in ri['pn'] > + ro.save() > + except Exception as e: > + logger.debug("Failed saving recipe %s", e) > > def handle_noargs(self, **options): > - for ls in LayerSource.objects.all(): > - ls.update() > + self.update()