* [layerindex-web][PATCH 1/3] API: fix recipes API performance regression
2019-09-16 5:19 [layerindex-web][PATCH 0/3] REST API fixes Paul Eggleton
@ 2019-09-16 5:19 ` Paul Eggleton
2019-09-16 5:19 ` [layerindex-web][PATCH 2/3] import_layers: use recipesExtended viewset Paul Eggleton
2019-09-16 5:19 ` [layerindex-web][PATCH 3/3] import_layers: fix output when importing layers from scratch Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2019-09-16 5:19 UTC (permalink / raw)
To: yocto
Adding these extra child items to the standard "recipes" viewset (which
we did recently in 684a06a383fd2a8da80490dff5f0dbc47750934e) means that
some current usage is impractical due to the size of the returned list
of items. Instead, create a recipesExtended viewset, move the new child
items to that and add pagination to avoid result size issues.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/restviews.py | 25 ++++++++++++++++++++++---
layerindex/urls.py | 1 +
2 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/layerindex/restviews.py b/layerindex/restviews.py
index 96c24ea3..7de4cd7d 100644
--- a/layerindex/restviews.py
+++ b/layerindex/restviews.py
@@ -1,7 +1,16 @@
+# OpenEmbedded Layer Index REST API implementation
+#
+# Copyright (C) 2014, 2016-2019 Intel Corporation
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
from layerindex.models import Branch, LayerItem, LayerMaintainer, YPCompatibleVersion, LayerNote, LayerBranch, LayerDependency, Recipe, Machine, Distro, BBClass, Source, Patch, PackageConfig, StaticBuildDep, DynamicBuildDep, RecipeFileDependency, BBAppend, IncFile
-from rest_framework import viewsets, serializers
+from rest_framework import viewsets, serializers, pagination
from layerindex.querysethelper import params_to_queryset, get_search_tuple
+class LayerIndexPagination(pagination.PageNumberPagination):
+ page_size = 200
+
class DynamicFieldsModelSerializer(serializers.ModelSerializer):
"""
A ModelSerializer that takes an additional "fields" argument that
@@ -114,6 +123,15 @@ class RecipeSerializer(serializers.ModelSerializer):
model = Recipe
fields = '__all__'
+class RecipeViewSet(ParametricSearchableModelViewSet):
+ queryset = Recipe.objects.all()
+ serializer_class = RecipeSerializer
+
+class RecipeExtendedSerializer(serializers.ModelSerializer):
+ class Meta:
+ model = Recipe
+ fields = '__all__'
+
sources = serializers.SerializerMethodField()
patches = serializers.SerializerMethodField()
package_configs = serializers.SerializerMethodField()
@@ -143,9 +161,10 @@ class RecipeSerializer(serializers.ModelSerializer):
serializer = RecipeFileDependencySerializer(instance=qs, many=True, read_only=True, fields=('layerbranch', 'path'))
return serializer.data
-class RecipeViewSet(ParametricSearchableModelViewSet):
+class RecipeExtendedViewSet(ParametricSearchableModelViewSet):
queryset = Recipe.objects.all()
- serializer_class = RecipeSerializer
+ serializer_class = RecipeExtendedSerializer
+ pagination_class = LayerIndexPagination
class MachineSerializer(serializers.ModelSerializer):
class Meta:
diff --git a/layerindex/urls.py b/layerindex/urls.py
index dec99585..13f42c92 100644
--- a/layerindex/urls.py
+++ b/layerindex/urls.py
@@ -28,6 +28,7 @@ router.register(r'layerDependencies', restviews.LayerDependencyViewSet)
router.register(r'layerMaintainers', restviews.LayerMaintainerViewSet)
router.register(r'layerNotes', restviews.LayerNoteViewSet)
router.register(r'recipes', restviews.RecipeViewSet)
+router.register(r'recipesExtended', restviews.RecipeExtendedViewSet)
router.register(r'machines', restviews.MachineViewSet)
router.register(r'distros', restviews.DistroViewSet)
router.register(r'classes', restviews.ClassViewSet)
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [layerindex-web][PATCH 2/3] import_layers: use recipesExtended viewset
2019-09-16 5:19 [layerindex-web][PATCH 0/3] REST API fixes Paul Eggleton
2019-09-16 5:19 ` [layerindex-web][PATCH 1/3] API: fix recipes API performance regression Paul Eggleton
@ 2019-09-16 5:19 ` Paul Eggleton
2019-09-16 5:19 ` [layerindex-web][PATCH 3/3] import_layers: fix output when importing layers from scratch Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2019-09-16 5:19 UTC (permalink / raw)
To: yocto
The new viewset has a different URL and uses pagination, so we need to
take that into account.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/tools/import_layers.py | 92 ++++++++++++++++++-------------
1 file changed, 55 insertions(+), 37 deletions(-)
diff --git a/layerindex/tools/import_layers.py b/layerindex/tools/import_layers.py
index d40c391a..0ce31968 100755
--- a/layerindex/tools/import_layers.py
+++ b/layerindex/tools/import_layers.py
@@ -89,7 +89,7 @@ def main():
layerbranches_url = jsdata['layerBranches']
layermaintainers_url = jsdata.get('layerMaintainers', None)
layernotes_url = jsdata.get('layerNotes', None)
- recipes_url = jsdata.get('recipes', None)
+ recipes_url = jsdata.get('recipesExtended', None)
machines_url = jsdata.get('machines', None)
distros_url = jsdata.get('distros', None)
classes_url = jsdata.get('classes', None)
@@ -186,55 +186,73 @@ def main():
# The parent field always needs to be part of the keys
keys = key_fields + [parentfield]
+ def fetch_api_url(api_url):
+ rq = urllib.request.Request(api_url)
+ data = urllib.request.urlopen(rq).read()
+ return json.loads(data.decode('utf-8'))
+
if url:
if parent_orig_id is None:
raise Exception('import_child_items: if url is specified then parent_orig_id must also be specified')
- rq = urllib.request.Request(url + '?filter=%s:%s' % (parentfield, parent_orig_id))
- data = urllib.request.urlopen(rq).read()
- childjslist = json.loads(data.decode('utf-8'))
+ childjsdata = fetch_api_url(url + '?filter=%s:%s' % (parentfield, parent_orig_id))
elif childlist is not None:
- childjslist = childlist
+ childjsdata = childlist
else:
raise Exception('import_child_items: either url or childlist must be specified')
manager = getattr(parentobj, objclass.__name__.lower() + '_set')
existing_ids = list(manager.values_list('id', flat=True))
updated_ids = []
- for childjs in childjslist:
- vals = {}
- for key, value in childjs.items():
- if key in exclude:
- continue
- vals[key] = value
- vals[parentfield] = parentobj
- if keys:
- keyvals = {k: vals[k] for k in keys}
- else:
- keyvals = vals
-
- # In the case of multiple records with the same keys (e.g. multiple recipes with same pn),
- # we need to skip ones we've already touched
- obj = None
- created = False
- for entry in manager.filter(**keyvals):
- if entry.id not in updated_ids:
- obj = entry
+ def import_list(childjslist):
+ for childjs in childjslist:
+ vals = {}
+ for key, value in childjs.items():
+ if key in exclude:
+ continue
+ vals[key] = value
+ vals[parentfield] = parentobj
+
+ if keys:
+ keyvals = {k: vals[k] for k in keys}
+ else:
+ keyvals = vals
+
+ # In the case of multiple records with the same keys (e.g. multiple recipes with same pn),
+ # we need to skip ones we've already touched
+ obj = None
+ created = False
+ for entry in manager.filter(**keyvals):
+ if entry.id not in updated_ids:
+ obj = entry
+ break
+ else:
+ created = True
+ obj = objclass(**keyvals)
+
+ for key, value in vals.items():
+ setattr(obj, key, value)
+ # Need to have saved before calling custom_field_cb since the function might be adding child objects
+ obj.save()
+ updated_ids.append(obj.id)
+ if custom_field_cb is not None:
+ custom_field_cb(obj, childjs)
+ if not created:
+ if obj.id in existing_ids:
+ existing_ids.remove(obj.id)
+
+ if 'results' in childjsdata:
+ while True:
+ import_list(childjsdata['results'])
+ if childjsdata.get('next', None):
+ childjsdata = fetch_api_url(childjsdata['next'])
+ if not 'results' in childjsdata:
+ break
+ else:
break
- else:
- created = True
- obj = objclass(**keyvals)
+ else:
+ import_list(childjsdata)
- for key, value in vals.items():
- setattr(obj, key, value)
- # Need to have saved before calling custom_field_cb since the function might be adding child objects
- obj.save()
- updated_ids.append(obj.id)
- if custom_field_cb is not None:
- custom_field_cb(obj, childjs)
- if not created:
- if obj.id in existing_ids:
- existing_ids.remove(obj.id)
for idv in existing_ids:
objclass.objects.filter(id=idv).delete()
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [layerindex-web][PATCH 3/3] import_layers: fix output when importing layers from scratch
2019-09-16 5:19 [layerindex-web][PATCH 0/3] REST API fixes Paul Eggleton
2019-09-16 5:19 ` [layerindex-web][PATCH 1/3] API: fix recipes API performance regression Paul Eggleton
2019-09-16 5:19 ` [layerindex-web][PATCH 2/3] import_layers: use recipesExtended viewset Paul Eggleton
@ 2019-09-16 5:19 ` Paul Eggleton
2 siblings, 0 replies; 4+ messages in thread
From: Paul Eggleton @ 2019-09-16 5:19 UTC (permalink / raw)
To: yocto
Fixes "Importing None" message since we had not yet initialised the
layerbranch variable.
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
layerindex/tools/import_layers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/layerindex/tools/import_layers.py b/layerindex/tools/import_layers.py
index 0ce31968..1e49bbdf 100755
--- a/layerindex/tools/import_layers.py
+++ b/layerindex/tools/import_layers.py
@@ -329,10 +329,10 @@ def main():
continue
logger.info('Updating %s (%d/%d)' % (layerbranch, i+1, layercount))
else:
- logger.info('Importing %s (%d/%d)' % (layerbranch, i+1, layercount))
layerbranch = LayerBranch()
layerbranch.branch = branch
layerbranch.layer = layer
+ logger.info('Importing %s (%d/%d)' % (layerbranch, i+1, layercount))
for key, value in layerbranchjs.items():
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread