[PATCH 0/5] bug fixes for Toaster

[PATCH 0/5] bug fixes for Toaster

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

Hello,

This patchset contains a number of bugfixes for Toaster.
This should be merged on both origin/master and origin/dora-toaster of OE-Core.

Cheers,
Alex

Alexandru DAMIAN (5):
  cooker, toaster: variable definition tracking
  toaster: fix tasks showing as NoExec
  toaster: remove author field
  toaster: fix path to buildstats file
  toasterui: mark failed sceneQueue tasks as failed

 lib/bb/cooker.py                                   | 15 +++++++++++---
 lib/bb/cookerdata.py                               |  1 +
 lib/bb/ui/buildinfohelper.py                       | 23 +++++++++++++++-------
 lib/bb/ui/toasterui.py                             |  2 +-
 lib/toaster/bldviewer/templates/configuration.html |  6 ++++--
 lib/toaster/bldviewer/templates/recipe.html        |  1 -
 lib/toaster/orm/models.py                          | 17 ++++++++++------
 7 files changed, 45 insertions(+), 20 deletions(-)

-- 
1.8.3.2

[PATCH 1/5] cooker, toaster: variable definition tracking

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

In order to track the file where a configuration
variable was defined, this patch bring these changes:

* a new feature is defined in CookerFeatures, named
VARIABLE_TRACKING. When a UI requests VARIABLE_TRACKING,
the variable definition are tracked when configuration
is parsed.

* getAllKeysWithFlags now includes variable history in the
data dump

* toaster_ui.py will record the operation, file path
and line number where the variable was changes

* toaster Simple UI will display the file path
and line number for Configuration page

There is a change in the models to accomodate the recording
of variable change history.

[YOCTO #5227]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 lib/bb/cooker.py                                   | 15 ++++++++++++---
 lib/bb/cookerdata.py                               |  1 +
 lib/bb/ui/buildinfohelper.py                       | 14 ++++++++++----
 lib/bb/ui/toasterui.py                             |  2 +-
 lib/toaster/bldviewer/templates/configuration.html |  6 ++++--
 lib/toaster/orm/models.py                          |  6 +++++-
 6 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 0580cd5..4c9067b 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -81,7 +81,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
+    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING] = range(3)
 
     def __init__(self):
         self._features=set()
@@ -177,6 +177,9 @@ class BBCooker:
         self.data.disableTracking()
 
     def loadConfigurationData(self):
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.enableDataTracking()
+
         self.initConfigurationData()
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
@@ -189,6 +192,10 @@ class BBCooker:
         bb.data.update_data(self.event_data)
         bb.parse.init_parser(self.event_data)
 
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.disableDataTracking()
+
+
     def modifyConfigurationVar(self, var, val, default_file, op):
         if op == "append":
             self.appendConfigurationVar(var, val, default_file)
@@ -320,7 +327,6 @@ class BBCooker:
         open(confpath, 'w').close()
 
     def parseConfiguration(self):
-
         # Set log file verbosity
         verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
         if verboselogs:
@@ -1175,7 +1181,10 @@ class BBCooker:
             try:
                 v = self.data.getVar(k, True)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
-                    dump[k] = { 'v' : v }
+                    dump[k] = {
+    'v' : v ,
+    'history' : self.data.varhistory.variable(k),
+                    }
                     for d in flaglist:
                         dump[k][d] = self.data.getVarFlag(k, d)
             except Exception as e:
diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index e640ed0..1cc7bc2 100644
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -127,6 +127,7 @@ class CookerConfiguration(object):
         self.dump_signatures = False
         self.dry_run = False
         self.tracking = False
+        self.server_only = False
 
         self.env = {}
 
diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index 2b0298e..863a20e 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -27,8 +27,10 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
 
 import toaster.toastermain.settings as toaster_django_settings
 from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
-from toaster.orm.models import Target_Package, Build_Package, Variable, Build_File
-from toaster.orm.models import Task_Dependency, Build_Package_Dependency, Target_Package_Dependency, Recipe_Dependency
+from toaster.orm.models import Variable, VariableHistory
+from toaster.orm.models import Target_Package, Build_Package, Build_File
+from toaster.orm.models import Task_Dependency, Build_Package_Dependency
+from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency
 from bb.msg import BBLogFormatter as format
 
 class ORMWrapper(object):
@@ -232,11 +234,15 @@ class ORMWrapper(object):
                         desc = vardump[root_var]['doc']
                 if desc is None:
                     desc = ''
-                Variable.objects.create( build = build_obj,
+                variable_obj = Variable.objects.create( build = build_obj,
                     variable_name = k,
                     variable_value = value,
                     description = desc)
-
+                for vh in vardump[k]['history']:
+                    VariableHistory.objects.create( variable = variable_obj,
+                            file_name = vh['file'],
+                            line_number = vh['line'],
+                            operation = vh['op'])
 
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it
diff --git a/lib/bb/ui/toasterui.py b/lib/bb/ui/toasterui.py
index 6c5b152..d2dba25 100644
--- a/lib/bb/ui/toasterui.py
+++ b/lib/bb/ui/toasterui.py
@@ -41,7 +41,7 @@ import sys
 import time
 import xmlrpclib
 
-featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE]
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
diff --git a/lib/toaster/bldviewer/templates/configuration.html b/lib/toaster/bldviewer/templates/configuration.html
index 052c37c..8db35e0 100644
--- a/lib/toaster/bldviewer/templates/configuration.html
+++ b/lib/toaster/bldviewer/templates/configuration.html
@@ -5,16 +5,18 @@
 
             <tr>
             <th>Name</th>
-            <th>Value</th>
             <th>Description</th>
+            <th>Definition history</th>
+            <th>Value</th>
             </tr>
 
             {% for variable in configuration %}
 
             <tr class="data">
                 <td>{{variable.variable_name}}</td>
-                <td>{{variable.variable_value}}</td>
                 <td>{% if variable.description %}{{variable.description}}{% endif %}</td>
+                <td>{% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}<br/>{%endfor%}</td>
+                <td>{{variable.variable_value}}</td>
             {% endfor %}
 
 {% endblock %}
diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 53b9e3a..2ae0b51 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -238,11 +238,15 @@ class Variable(models.Model):
     build = models.ForeignKey(Build, related_name='variable_build')
     variable_name = models.CharField(max_length=100)
     variable_value = models.TextField(blank=True)
-    file = models.FilePathField(max_length=255)
     changed = models.BooleanField(default=False)
     human_readable_name = models.CharField(max_length=200)
     description = models.TextField(blank=True)
 
+class VariableHistory(models.Model):
+    variable = models.ForeignKey(Variable)
+    file_name = models.FilePathField(max_length=255)
+    line_number = models.IntegerField(null=True)
+    operation = models.CharField(max_length=16)
 
 class LogMessage(models.Model):
     INFO = 0
-- 
1.8.3.2

[PATCH 2/5] toaster: fix tasks showing as NoExec

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

Tasks without script type information showed by default
as NoExec; this happens for all Prebuild or Covered
tasks, as script type information comes only on TaskStarted
event. Such a default value may drive confusion, as NoExec value
should be reserved for the NoExec-flagged tasks.

This patch adds a new default value named Unknown that will be
used for all tasks that don't have script type information
available.

    [YOCTO #5327]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 lib/toaster/orm/models.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 2ae0b51..a3df990 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -74,11 +74,13 @@ class Task(models.Model):
         (SSTATE_RESTORED, 'Restored'), # succesfully restored
     )
 
-    CODING_NOEXEC = 0
-    CODING_PYTHON = 1
-    CODING_SHELL = 2
+    CODING_NA = 0
+    CODING_NOEXEC = 1
+    CODING_PYTHON = 2
+    CODING_SHELL = 3
 
     TASK_CODING = (
+        (CODING_NA, 'N/A'),
         (CODING_NOEXEC, 'NoExec'),
         (CODING_PYTHON, 'Python'),
         (CODING_SHELL, 'Shell'),
@@ -110,7 +112,7 @@ class Task(models.Model):
     task_name = models.CharField(max_length=100)
     source_url = models.FilePathField(max_length=255, blank=True)
     work_directory = models.FilePathField(max_length=255, blank=True)
-    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NOEXEC)
+    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
     line_number = models.IntegerField(default=0)
     disk_io = models.IntegerField(null=True)
     cpu_usage = models.DecimalField(max_digits=6, decimal_places=2, null=True)
-- 
1.8.3.2

[PATCH 3/5] toaster: remove author field

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

The AUTHOR field in most recipes is not defined,
or it's not really consistently set in the metadata,
Also does it seem particularly useful.

This patch removes the AUTHOR variable from the
toaster system

    [YOCTO #5449]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 lib/bb/ui/buildinfohelper.py                | 1 -
 lib/toaster/bldviewer/templates/recipe.html | 1 -
 lib/toaster/orm/models.py                   | 1 -
 3 files changed, 3 deletions(-)

diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index 863a20e..c1ffa26 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -624,7 +624,6 @@ class BuildInfoHelper(object):
             recipe_info['licensing_info'] = 'Not Available'
             recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
             recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
-            recipe_info['author'] = 'Not Available'
             recipe_info['file_path'] = file_name
             recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
             if 'inherits' in event._depgraph['pn'][pn].keys():
diff --git a/lib/toaster/bldviewer/templates/recipe.html b/lib/toaster/bldviewer/templates/recipe.html
index a624370..e9a1c10 100644
--- a/lib/toaster/bldviewer/templates/recipe.html
+++ b/lib/toaster/bldviewer/templates/recipe.html
@@ -38,7 +38,6 @@
             <td>{{recipe.licensing_info}}</td>
             <td>{{recipe.homepage}}</td>
             <td>{{recipe.bugtracker}}</td>
-            <td>{{recipe.author}}</td>
             <td>{{recipe.file_path}}</td>
             <td>
         <div style="height: 5em; overflow:auto">
diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index a3df990..cac8367 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -207,7 +207,6 @@ class Recipe(models.Model):
     licensing_info = models.TextField(blank=True)
     homepage = models.URLField(blank=True)
     bugtracker = models.URLField(blank=True)
-    author = models.CharField(max_length=100, blank=True)
     file_path = models.FilePathField(max_length=255)
 
 
-- 
1.8.3.2

[PATCH 4/5] toaster: fix path to buildstats file

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

The buildstats file path changes based on the
optional PE variable that may be defined for a
recipe.

The toasterui simply ignored the PE value, and
as such it didn't correctly reach buildstats files
for some of the tasks.

This patch fixes the issue.

    [YOCTO #5073]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 lib/bb/ui/buildinfohelper.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index c1ffa26..92d6bbe 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -392,7 +392,11 @@ class BuildInfoHelper(object):
             target = t.target
             machine = self.internal_state['build'].machine
             buildname = self.internal_state['build'].build_name
-            package = task_object.recipe.name + "-" + task_object.recipe.version.strip(":")
+            pe, pv = task_object.recipe.version.split(":",1)
+            if len(pe) > 0:
+                package = task_object.recipe.name + "-" + pe + "_" + pv
+            else:
+                package = task_object.recipe.name + "-" + pv
 
             build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, target=target,
                                                      machine=machine, buildname=buildname,
-- 
1.8.3.2

[PATCH 5/5] toasterui: mark failed sceneQueue tasks as failed

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

This patch addresses an issue where a failed sceneQueue task
entry was not updated on the Fail event. As a result, it
always showed the task as not-available.

    [YOCTO #5216]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 lib/bb/ui/buildinfohelper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index 92d6bbe..38e89ec 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -537,7 +537,7 @@ class BuildInfoHelper(object):
             task_information['disk_io'] = task_build_stats['disk_io']
             del self.internal_state[identifier]
 
-        if isinstance(event, bb.runqueue.runQueueTaskFailed):
+        if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runCommand.sceneQueueTaskFailed)):
             task_information['outcome'] = Task.OUTCOME_FAILED
             del self.internal_state[identifier]
 
-- 
1.8.3.2

[PATCH v2] cooker, toaster: variable definition tracking

[Alex DAMIAN]

From: Alexandru DAMIAN <alexandru.damian@intel.com>

In order to track the file where a configuration
variable was defined, this patch bring these changes:

* a new feature is defined in CookerFeatures, named
BASEDATASTORE_TRACKING. When a UI requests BASEDATASTORE_TRACKING,
the base variable definition are tracked when configuration
is parsed.

* getAllKeysWithFlags now includes variable history in the
data dump

* toaster_ui.py will record the operation, file path
and line number where the variable was changes

* toaster Simple UI will display the file path
and line number for Configuration page

There is a change in the models to accomodate the recording
of variable change history.

[YOCTO #5227]

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
---
 lib/bb/cooker.py                                   | 15 ++++++++++++---
 lib/bb/cookerdata.py                               |  1 +
 lib/bb/ui/buildinfohelper.py                       | 14 ++++++++++----
 lib/bb/ui/toasterui.py                             |  2 +-
 lib/toaster/bldviewer/templates/configuration.html |  6 ++++--
 lib/toaster/orm/models.py                          |  6 +++++-
 6 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 0580cd5..4c9067b 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -81,7 +81,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
+    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING] = range(3)
 
     def __init__(self):
         self._features=set()
@@ -177,6 +177,9 @@ class BBCooker:
         self.data.disableTracking()
 
     def loadConfigurationData(self):
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.enableDataTracking()
+
         self.initConfigurationData()
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
@@ -189,6 +192,10 @@ class BBCooker:
         bb.data.update_data(self.event_data)
         bb.parse.init_parser(self.event_data)
 
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.disableDataTracking()
+
+
     def modifyConfigurationVar(self, var, val, default_file, op):
         if op == "append":
             self.appendConfigurationVar(var, val, default_file)
@@ -320,7 +327,6 @@ class BBCooker:
         open(confpath, 'w').close()
 
     def parseConfiguration(self):
-
         # Set log file verbosity
         verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
         if verboselogs:
@@ -1175,7 +1181,10 @@ class BBCooker:
             try:
                 v = self.data.getVar(k, True)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
-                    dump[k] = { 'v' : v }
+                    dump[k] = {
+    'v' : v ,
+    'history' : self.data.varhistory.variable(k),
+                    }
                     for d in flaglist:
                         dump[k][d] = self.data.getVarFlag(k, d)
             except Exception as e:
diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index e640ed0..1cc7bc2 100644
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -127,6 +127,7 @@ class CookerConfiguration(object):
         self.dump_signatures = False
         self.dry_run = False
         self.tracking = False
+        self.server_only = False
 
         self.env = {}
 
diff --git a/lib/bb/ui/buildinfohelper.py b/lib/bb/ui/buildinfohelper.py
index 2b0298e..863a20e 100644
--- a/lib/bb/ui/buildinfohelper.py
+++ b/lib/bb/ui/buildinfohelper.py
@@ -27,8 +27,10 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
 
 import toaster.toastermain.settings as toaster_django_settings
 from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
-from toaster.orm.models import Target_Package, Build_Package, Variable, Build_File
-from toaster.orm.models import Task_Dependency, Build_Package_Dependency, Target_Package_Dependency, Recipe_Dependency
+from toaster.orm.models import Variable, VariableHistory
+from toaster.orm.models import Target_Package, Build_Package, Build_File
+from toaster.orm.models import Task_Dependency, Build_Package_Dependency
+from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency
 from bb.msg import BBLogFormatter as format
 
 class ORMWrapper(object):
@@ -232,11 +234,15 @@ class ORMWrapper(object):
                         desc = vardump[root_var]['doc']
                 if desc is None:
                     desc = ''
-                Variable.objects.create( build = build_obj,
+                variable_obj = Variable.objects.create( build = build_obj,
                     variable_name = k,
                     variable_value = value,
                     description = desc)
-
+                for vh in vardump[k]['history']:
+                    VariableHistory.objects.create( variable = variable_obj,
+                            file_name = vh['file'],
+                            line_number = vh['line'],
+                            operation = vh['op'])
 
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it
diff --git a/lib/bb/ui/toasterui.py b/lib/bb/ui/toasterui.py
index 6c5b152..d2dba25 100644
--- a/lib/bb/ui/toasterui.py
+++ b/lib/bb/ui/toasterui.py
@@ -41,7 +41,7 @@ import sys
 import time
 import xmlrpclib
 
-featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE]
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
diff --git a/lib/toaster/bldviewer/templates/configuration.html b/lib/toaster/bldviewer/templates/configuration.html
index 052c37c..8db35e0 100644
--- a/lib/toaster/bldviewer/templates/configuration.html
+++ b/lib/toaster/bldviewer/templates/configuration.html
@@ -5,16 +5,18 @@
 
             <tr>
             <th>Name</th>
-            <th>Value</th>
             <th>Description</th>
+            <th>Definition history</th>
+            <th>Value</th>
             </tr>
 
             {% for variable in configuration %}
 
             <tr class="data">
                 <td>{{variable.variable_name}}</td>
-                <td>{{variable.variable_value}}</td>
                 <td>{% if variable.description %}{{variable.description}}{% endif %}</td>
+                <td>{% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}<br/>{%endfor%}</td>
+                <td>{{variable.variable_value}}</td>
             {% endfor %}
 
 {% endblock %}
diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 53b9e3a..2ae0b51 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -238,11 +238,15 @@ class Variable(models.Model):
     build = models.ForeignKey(Build, related_name='variable_build')
     variable_name = models.CharField(max_length=100)
     variable_value = models.TextField(blank=True)
-    file = models.FilePathField(max_length=255)
     changed = models.BooleanField(default=False)
     human_readable_name = models.CharField(max_length=200)
     description = models.TextField(blank=True)
 
+class VariableHistory(models.Model):
+    variable = models.ForeignKey(Variable)
+    file_name = models.FilePathField(max_length=255)
+    line_number = models.IntegerField(null=True)
+    operation = models.CharField(max_length=16)
 
 class LogMessage(models.Model):
     INFO = 0
-- 
1.8.3.2

Re: [PATCH v2] cooker, toaster: variable definition tracking

[Richard Purdie]

On Thu, 2013-11-14 at 13:56 +0000, Alex DAMIAN wrote:
> From: Alexandru DAMIAN <alexandru.damian@intel.com>
> 
> In order to track the file where a configuration
> variable was defined, this patch bring these changes:
> 
> * a new feature is defined in CookerFeatures, named
> BASEDATASTORE_TRACKING. When a UI requests BASEDATASTORE_TRACKING,
> the base variable definition are tracked when configuration
> is parsed.
> 
> * getAllKeysWithFlags now includes variable history in the
> data dump
> 
> * toaster_ui.py will record the operation, file path
> and line number where the variable was changes
> 
> * toaster Simple UI will display the file path
> and line number for Configuration page
> 
> There is a change in the models to accomodate the recording
> of variable change history.
> 
> [YOCTO #5227]
> 
> Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> ---
>  lib/bb/cooker.py                                   | 15 ++++++++++++---
>  lib/bb/cookerdata.py                               |  1 +
>  lib/bb/ui/buildinfohelper.py                       | 14 ++++++++++----
>  lib/bb/ui/toasterui.py                             |  2 +-
>  lib/toaster/bldviewer/templates/configuration.html |  6 ++++--
>  lib/toaster/orm/models.py                          |  6 +++++-
>  6 files changed, 33 insertions(+), 11 deletions(-)
> 
> diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> index 0580cd5..4c9067b 100644
> --- a/lib/bb/cooker.py
> +++ b/lib/bb/cooker.py
> @@ -81,7 +81,7 @@ class SkippedPackage:
>  
> 
>  class CookerFeatures(object):
> -    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
> +    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING] = range(3)
>  
>      def __init__(self):
>          self._features=set()
> @@ -177,6 +177,9 @@ class BBCooker:
>          self.data.disableTracking()
>  
>      def loadConfigurationData(self):
> +        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
> +            self.enableDataTracking()
> +
>          self.initConfigurationData()
>          self.databuilder.parseBaseConfiguration()
>          self.data = self.databuilder.data
> @@ -189,6 +192,10 @@ class BBCooker:
>          bb.data.update_data(self.event_data)
>          bb.parse.init_parser(self.event_data)
>  
> +        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
> +            self.disableDataTracking()
> +
> +
>      def modifyConfigurationVar(self, var, val, default_file, op):
>          if op == "append":
>              self.appendConfigurationVar(var, val, default_file)
> @@ -320,7 +327,6 @@ class BBCooker:
>          open(confpath, 'w').close()
>  
>      def parseConfiguration(self):
> -
>          # Set log file verbosity
>          verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
>          if verboselogs:
> @@ -1175,7 +1181,10 @@ class BBCooker:
>              try:
>                  v = self.data.getVar(k, True)
>                  if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
> -                    dump[k] = { 'v' : v }
> +                    dump[k] = {
> +    'v' : v ,
> +    'history' : self.data.varhistory.variable(k),
> +                    }
>                      for d in flaglist:
>                          dump[k][d] = self.data.getVarFlag(k, d)
>              except Exception as e:
> diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
> index e640ed0..1cc7bc2 100644
> --- a/lib/bb/cookerdata.py
> +++ b/lib/bb/cookerdata.py
> @@ -127,6 +127,7 @@ class CookerConfiguration(object):
>          self.dump_signatures = False
>          self.dry_run = False
>          self.tracking = False
> +        self.server_only = False
>  
>          self.env = {}
>  

Patch looks ok in general but I think the above is leftover and
unneeded?

Cheers,

Richard

Re: [PATCH v2] cooker, toaster: variable definition tracking

[Damian, Alexandru]

[-- Attachment #1: Type: text/plain, Size: 4422 bytes --]

Not sure about what leftover you're referring to... the commit message is
not right ?


Alex


On Fri, Nov 15, 2013 at 11:28 AM, Richard Purdie <
richard.purdie@linuxfoundation.org> wrote:

> On Thu, 2013-11-14 at 13:56 +0000, Alex DAMIAN wrote:
> > From: Alexandru DAMIAN <alexandru.damian@intel.com>
> >
> > In order to track the file where a configuration
> > variable was defined, this patch bring these changes:
> >
> > * a new feature is defined in CookerFeatures, named
> > BASEDATASTORE_TRACKING. When a UI requests BASEDATASTORE_TRACKING,
> > the base variable definition are tracked when configuration
> > is parsed.
> >
> > * getAllKeysWithFlags now includes variable history in the
> > data dump
> >
> > * toaster_ui.py will record the operation, file path
> > and line number where the variable was changes
> >
> > * toaster Simple UI will display the file path
> > and line number for Configuration page
> >
> > There is a change in the models to accomodate the recording
> > of variable change history.
> >
> > [YOCTO #5227]
> >
> > Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
> > ---
> >  lib/bb/cooker.py                                   | 15 ++++++++++++---
> >  lib/bb/cookerdata.py                               |  1 +
> >  lib/bb/ui/buildinfohelper.py                       | 14 ++++++++++----
> >  lib/bb/ui/toasterui.py                             |  2 +-
> >  lib/toaster/bldviewer/templates/configuration.html |  6 ++++--
> >  lib/toaster/orm/models.py                          |  6 +++++-
> >  6 files changed, 33 insertions(+), 11 deletions(-)
> >
> > diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> > index 0580cd5..4c9067b 100644
> > --- a/lib/bb/cooker.py
> > +++ b/lib/bb/cooker.py
> > @@ -81,7 +81,7 @@ class SkippedPackage:
> >
> >
> >  class CookerFeatures(object):
> > -    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
> > +    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE,
> BASEDATASTORE_TRACKING] = range(3)
> >
> >      def __init__(self):
> >          self._features=set()
> > @@ -177,6 +177,9 @@ class BBCooker:
> >          self.data.disableTracking()
> >
> >      def loadConfigurationData(self):
> > +        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
> > +            self.enableDataTracking()
> > +
> >          self.initConfigurationData()
> >          self.databuilder.parseBaseConfiguration()
> >          self.data = self.databuilder.data
> > @@ -189,6 +192,10 @@ class BBCooker:
> >          bb.data.update_data(self.event_data)
> >          bb.parse.init_parser(self.event_data)
> >
> > +        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
> > +            self.disableDataTracking()
> > +
> > +
> >      def modifyConfigurationVar(self, var, val, default_file, op):
> >          if op == "append":
> >              self.appendConfigurationVar(var, val, default_file)
> > @@ -320,7 +327,6 @@ class BBCooker:
> >          open(confpath, 'w').close()
> >
> >      def parseConfiguration(self):
> > -
> >          # Set log file verbosity
> >          verboselogs =
> bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
> >          if verboselogs:
> > @@ -1175,7 +1181,10 @@ class BBCooker:
> >              try:
> >                  v = self.data.getVar(k, True)
> >                  if not k.startswith("__") and not isinstance(v,
> bb.data_smart.DataSmart):
> > -                    dump[k] = { 'v' : v }
> > +                    dump[k] = {
> > +    'v' : v ,
> > +    'history' : self.data.varhistory.variable(k),
> > +                    }
> >                      for d in flaglist:
> >                          dump[k][d] = self.data.getVarFlag(k, d)
> >              except Exception as e:
> > diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
> > index e640ed0..1cc7bc2 100644
> > --- a/lib/bb/cookerdata.py
> > +++ b/lib/bb/cookerdata.py
> > @@ -127,6 +127,7 @@ class CookerConfiguration(object):
> >          self.dump_signatures = False
> >          self.dry_run = False
> >          self.tracking = False
> > +        self.server_only = False
> >
> >          self.env = {}
> >
>
> Patch looks ok in general but I think the above is leftover and
> unneeded?
>
> Cheers,
>
> Richard
>
>


-- 
Alex Damian
Yocto Project
SSG / OTC

[-- Attachment #2: Type: text/html, Size: 5855 bytes --]