[Feature]add MT2731_MP2_MR2_SVN388 baseline version
Change-Id: Ief04314834b31e27effab435d3ca8ba33b499059
diff --git a/meta/poky/bitbake/lib/toaster/toastergui/api.py b/meta/poky/bitbake/lib/toaster/toastergui/api.py
new file mode 100644
index 0000000..564d595
--- /dev/null
+++ b/meta/poky/bitbake/lib/toaster/toastergui/api.py
@@ -0,0 +1,1226 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Please run flake8 on this file before sending patches
+
+import os
+import re
+import logging
+import json
+import subprocess
+from collections import Counter
+from shutil import copyfile
+
+from orm.models import Project, ProjectTarget, Build, Layer_Version
+from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
+from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
+from orm.models import Layer, Target, Package, Package_Dependency
+from orm.models import ProjectVariable
+from bldcontrol.models import BuildRequest, BuildEnvironment
+from bldcontrol import bbcontroller
+
+from django.http import HttpResponse, JsonResponse
+from django.views.generic import View
+from django.core.urlresolvers import reverse
+from django.db.models import Q, F
+from django.db import Error
+from toastergui.templatetags.projecttags import filtered_filesizeformat
+from django.utils import timezone
+import pytz
+
+# development/debugging support
+verbose = 2
+def _log(msg):
+ if 1 == verbose:
+ print(msg)
+ elif 2 == verbose:
+ f1=open('/tmp/toaster.log', 'a')
+ f1.write("|" + msg + "|\n" )
+ f1.close()
+
+logger = logging.getLogger("toaster")
+
+
+def error_response(error):
+ return JsonResponse({"error": error})
+
+
+class XhrBuildRequest(View):
+
+ def get(self, request, *args, **kwargs):
+ return HttpResponse()
+
+ @staticmethod
+ def cancel_build(br):
+ """Cancel a build request"""
+ try:
+ bbctrl = bbcontroller.BitbakeController(br.environment)
+ bbctrl.forceShutDown()
+ except:
+ # We catch a bunch of exceptions here because
+ # this is where the server has not had time to start up
+ # and the build request or build is in transit between
+ # processes.
+ # We can safely just set the build as cancelled
+ # already as it never got started
+ build = br.build
+ build.outcome = Build.CANCELLED
+ build.save()
+
+ # We now hand over to the buildinfohelper to update the
+ # build state once we've finished cancelling
+ br.state = BuildRequest.REQ_CANCELLING
+ br.save()
+
+ def post(self, request, *args, **kwargs):
+ """
+ Build control
+
+ Entry point: /xhr_buildrequest/<project_id>
+ Method: POST
+
+ Args:
+ id: id of build to change
+ buildCancel = build_request_id ...
+ buildDelete = id ...
+ targets = recipe_name ...
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+
+ project = Project.objects.get(pk=kwargs['pid'])
+
+ if 'buildCancel' in request.POST:
+ for i in request.POST['buildCancel'].strip().split(" "):
+ try:
+ br = BuildRequest.objects.get(project=project, pk=i)
+ self.cancel_build(br)
+ except BuildRequest.DoesNotExist:
+ return error_response('No such build request id %s' % i)
+
+ return error_response('ok')
+
+ if 'buildDelete' in request.POST:
+ for i in request.POST['buildDelete'].strip().split(" "):
+ try:
+ BuildRequest.objects.select_for_update().get(
+ project=project,
+ pk=i,
+ state__lte=BuildRequest.REQ_DELETED).delete()
+
+ except BuildRequest.DoesNotExist:
+ pass
+ return error_response("ok")
+
+ if 'targets' in request.POST:
+ ProjectTarget.objects.filter(project=project).delete()
+ s = str(request.POST['targets'])
+ for t in re.sub(r'[;%|"]', '', s).split(" "):
+ if ":" in t:
+ target, task = t.split(":")
+ else:
+ target = t
+ task = ""
+ ProjectTarget.objects.create(project=project,
+ target=target,
+ task=task)
+ project.schedule_build()
+
+ return error_response('ok')
+
+ response = HttpResponse()
+ response.status_code = 500
+ return response
+
+
+class XhrProjectUpdate(View):
+
+ def get(self, request, *args, **kwargs):
+ return HttpResponse()
+
+ def post(self, request, *args, **kwargs):
+ """
+ Project Update
+
+ Entry point: /xhr_projectupdate/<project_id>
+ Method: POST
+
+ Args:
+ pid: pid of project to update
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+
+ project = Project.objects.get(pk=kwargs['pid'])
+ logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
+
+ if 'do_update' in request.POST:
+
+ # Extract any default image recipe
+ if 'default_image' in request.POST:
+ project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image']))
+ else:
+ project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'')
+
+ logger.debug("ProjectUpdateCallback:Chain to the build request")
+
+ # Chain to the build request
+ xhrBuildRequest = XhrBuildRequest()
+ return xhrBuildRequest.post(request, *args, **kwargs)
+
+ logger.warning("ERROR:XhrProjectUpdate")
+ response = HttpResponse()
+ response.status_code = 500
+ return response
+
+class XhrSetDefaultImageUrl(View):
+
+ def get(self, request, *args, **kwargs):
+ return HttpResponse()
+
+ def post(self, request, *args, **kwargs):
+ """
+ Project Update
+
+ Entry point: /xhr_setdefaultimage/<project_id>
+ Method: POST
+
+ Args:
+ pid: pid of project to update default image
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+
+ project = Project.objects.get(pk=kwargs['pid'])
+ logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk))
+
+ # set any default image recipe
+ if 'targets' in request.POST:
+ default_target = str(request.POST['targets'])
+ project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target)
+ logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
+ return error_response('ok')
+
+ logger.warning("ERROR:XhrSetDefaultImageUrl")
+ response = HttpResponse()
+ response.status_code = 500
+ return response
+
+
+#
+# Layer Management
+#
+# Rules for 'local_source_dir' layers
+# * Layers must have a unique name in the Layers table
+# * A 'local_source_dir' layer is supposed to be shared
+# by all projects that use it, so that it can have the
+# same logical name
+# * Each project that uses a layer will have its own
+# LayerVersion and Project Layer for it
+# * During the Paroject delete process, when the last
+# LayerVersion for a 'local_source_dir' layer is deleted
+# then the Layer record is deleted to remove orphans
+#
+
+def scan_layer_content(layer,layer_version):
+ # if this is a local layer directory, we can immediately scan its content
+ if layer.local_source_dir:
+ try:
+ # recipes-*/*/*.bb
+ cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
+ recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
+ recipes_list = recipes_list.decode("utf-8").strip()
+ if recipes_list and 'No such' not in recipes_list:
+ for recipe in recipes_list.split('\n'):
+ recipe_path = recipe[recipe.rfind('recipes-'):]
+ recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
+ recipe_ver = recipe_name.rfind('_')
+ if recipe_ver > 0:
+ recipe_name = recipe_name[0:recipe_ver]
+ if recipe_name:
+ ro, created = Recipe.objects.get_or_create(
+ layer_version=layer_version,
+ name=recipe_name
+ )
+ if created:
+ ro.file_path = recipe_path
+ ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name)
+ ro.description = ro.summary
+ ro.save()
+
+ except Exception as e:
+ logger.warning("ERROR:scan_layer_content: %s" % e)
+
+class XhrLayer(View):
+ """ Delete, Get, Add and Update Layer information
+
+ Methods: GET POST DELETE PUT
+ """
+
+ def get(self, request, *args, **kwargs):
+ """
+ Get layer information
+
+ Method: GET
+ Entry point: /xhr_layer/<project id>/<layerversion_id>
+ """
+
+ try:
+ layer_version = Layer_Version.objects.get(
+ pk=kwargs['layerversion_id'])
+
+ project = Project.objects.get(pk=kwargs['pid'])
+
+ project_layers = ProjectLayer.objects.filter(
+ project=project).values_list("layercommit_id",
+ flat=True)
+
+ ret = {
+ 'error': 'ok',
+ 'id': layer_version.pk,
+ 'name': layer_version.layer.name,
+ 'layerdetailurl':
+ layer_version.get_detailspage_url(project.pk),
+ 'vcs_ref': layer_version.get_vcs_reference(),
+ 'vcs_url': layer_version.layer.vcs_url,
+ 'local_source_dir': layer_version.layer.local_source_dir,
+ 'layerdeps': {
+ "list": [
+ {
+ "id": dep.id,
+ "name": dep.layer.name,
+ "layerdetailurl":
+ dep.get_detailspage_url(project.pk),
+ "vcs_url": dep.layer.vcs_url,
+ "vcs_reference": dep.get_vcs_reference()
+ }
+ for dep in layer_version.get_alldeps(project.id)]
+ },
+ 'projectlayers': list(project_layers)
+ }
+
+ return JsonResponse(ret)
+ except Layer_Version.DoesNotExist:
+ error_response("No such layer")
+
+ def post(self, request, *args, **kwargs):
+ """
+ Update a layer
+
+ Method: POST
+ Entry point: /xhr_layer/<layerversion_id>
+
+ Args:
+ vcs_url, dirpath, commit, up_branch, summary, description,
+ local_source_dir
+
+ add_dep = append a layerversion_id as a dependency
+ rm_dep = remove a layerversion_id as a depedency
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+
+ try:
+ # We currently only allow Imported layers to be edited
+ layer_version = Layer_Version.objects.get(
+ id=kwargs['layerversion_id'],
+ project=kwargs['pid'],
+ layer_source=LayerSource.TYPE_IMPORTED)
+
+ except Layer_Version.DoesNotExist:
+ return error_response("Cannot find imported layer to update")
+
+ if "vcs_url" in request.POST:
+ layer_version.layer.vcs_url = request.POST["vcs_url"]
+ if "dirpath" in request.POST:
+ layer_version.dirpath = request.POST["dirpath"]
+ if "commit" in request.POST:
+ layer_version.commit = request.POST["commit"]
+ layer_version.branch = request.POST["commit"]
+ if "summary" in request.POST:
+ layer_version.layer.summary = request.POST["summary"]
+ if "description" in request.POST:
+ layer_version.layer.description = request.POST["description"]
+ if "local_source_dir" in request.POST:
+ layer_version.layer.local_source_dir = \
+ request.POST["local_source_dir"]
+
+ if "add_dep" in request.POST:
+ lvd = LayerVersionDependency(
+ layer_version=layer_version,
+ depends_on_id=request.POST["add_dep"])
+ lvd.save()
+
+ if "rm_dep" in request.POST:
+ rm_dep = LayerVersionDependency.objects.get(
+ layer_version=layer_version,
+ depends_on_id=request.POST["rm_dep"])
+ rm_dep.delete()
+
+ try:
+ layer_version.layer.save()
+ layer_version.save()
+ except Exception as e:
+ return error_response("Could not update layer version entry: %s"
+ % e)
+
+ return error_response("ok")
+
+ def put(self, request, *args, **kwargs):
+ """ Add a new layer
+
+ Method: PUT
+ Entry point: /xhr_layer/<project id>/
+ Args:
+ project_id, name,
+ [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
+ (csv)]
+
+ """
+
+ try:
+ project = Project.objects.get(pk=kwargs['pid'])
+
+ layer_data = json.loads(request.body.decode('utf-8'))
+
+ # We require a unique layer name as otherwise the lists of layers
+ # becomes very confusing
+ existing_layers = \
+ project.get_all_compatible_layer_versions().values_list(
+ "layer__name",
+ flat=True)
+
+ add_to_project = False
+ layer_deps_added = []
+ if 'add_to_project' in layer_data:
+ add_to_project = True
+
+ if layer_data['name'] in existing_layers:
+ return JsonResponse({"error": "layer-name-exists"})
+
+ if ('local_source_dir' in layer_data):
+ # Local layer can be shared across projects. They have no 'release'
+ # and are not included in get_all_compatible_layer_versions() above
+ layer,created = Layer.objects.get_or_create(name=layer_data['name'])
+ _log("Local Layer created=%s" % created)
+ else:
+ layer = Layer.objects.create(name=layer_data['name'])
+
+ layer_version = Layer_Version.objects.create(
+ layer=layer,
+ project=project,
+ layer_source=LayerSource.TYPE_IMPORTED)
+
+ # Local layer
+ if ('local_source_dir' in layer_data): ### and layer.local_source_dir:
+ layer.local_source_dir = layer_data['local_source_dir']
+ # git layer
+ elif 'vcs_url' in layer_data:
+ layer.vcs_url = layer_data['vcs_url']
+ layer_version.dirpath = layer_data['dir_path']
+ layer_version.commit = layer_data['git_ref']
+ layer_version.branch = layer_data['git_ref']
+
+ layer.save()
+ layer_version.save()
+
+ if add_to_project:
+ ProjectLayer.objects.get_or_create(
+ layercommit=layer_version, project=project)
+
+ # Add the layer dependencies
+ if 'layer_deps' in layer_data:
+ for layer_dep_id in layer_data['layer_deps'].split(","):
+ layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
+ LayerVersionDependency.objects.get_or_create(
+ layer_version=layer_version, depends_on=layer_dep)
+
+ # Add layer deps to the project if specified
+ if add_to_project:
+ created, pl = ProjectLayer.objects.get_or_create(
+ layercommit=layer_dep, project=project)
+ layer_deps_added.append(
+ {'name': layer_dep.layer.name,
+ 'layerdetailurl':
+ layer_dep.get_detailspage_url(project.pk)})
+
+ # Scan the layer's content and update components
+ scan_layer_content(layer,layer_version)
+
+ except Layer_Version.DoesNotExist:
+ return error_response("layer-dep-not-found")
+ except Project.DoesNotExist:
+ return error_response("project-not-found")
+ except KeyError:
+ return error_response("incorrect-parameters")
+
+ return JsonResponse({'error': "ok",
+ 'imported_layer': {
+ 'name': layer.name,
+ 'layerdetailurl':
+ layer_version.get_detailspage_url()},
+ 'deps_added': layer_deps_added})
+
+ def delete(self, request, *args, **kwargs):
+ """ Delete an imported layer
+
+ Method: DELETE
+ Entry point: /xhr_layer/<projed id>/<layerversion_id>
+
+ """
+ try:
+ # We currently only allow Imported layers to be deleted
+ layer_version = Layer_Version.objects.get(
+ id=kwargs['layerversion_id'],
+ project=kwargs['pid'],
+ layer_source=LayerSource.TYPE_IMPORTED)
+ except Layer_Version.DoesNotExist:
+ return error_response("Cannot find imported layer to delete")
+
+ try:
+ ProjectLayer.objects.get(project=kwargs['pid'],
+ layercommit=layer_version).delete()
+ except ProjectLayer.DoesNotExist:
+ pass
+
+ layer_version.layer.delete()
+ layer_version.delete()
+
+ return JsonResponse({
+ "error": "ok",
+ "gotoUrl": reverse('projectlayers', args=(kwargs['pid'],))
+ })
+
+
+class XhrCustomRecipe(View):
+ """ Create a custom image recipe """
+
+ def post(self, request, *args, **kwargs):
+ """
+ Custom image recipe REST API
+
+ Entry point: /xhr_customrecipe/
+ Method: POST
+
+ Args:
+ name: name of custom recipe to create
+ project: target project id of orm.models.Project
+ base: base recipe id of orm.models.Recipe
+
+ Returns:
+ {"error": "ok",
+ "url": <url of the created recipe>}
+ or
+ {"error": <error message>}
+ """
+ # check if request has all required parameters
+ for param in ('name', 'project', 'base'):
+ if param not in request.POST:
+ return error_response("Missing parameter '%s'" % param)
+
+ # get project and baserecipe objects
+ params = {}
+ for name, model in [("project", Project),
+ ("base", Recipe)]:
+ value = request.POST[name]
+ try:
+ params[name] = model.objects.get(id=value)
+ except model.DoesNotExist:
+ return error_response("Invalid %s id %s" % (name, value))
+
+ # create custom recipe
+ try:
+
+ # Only allowed chars in name are a-z, 0-9 and -
+ if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
+ return error_response("invalid-name")
+
+ custom_images = CustomImageRecipe.objects.all()
+
+ # Are there any recipes with this name already in our project?
+ existing_image_recipes_in_project = custom_images.filter(
+ name=request.POST["name"], project=params["project"])
+
+ if existing_image_recipes_in_project.count() > 0:
+ return error_response("image-already-exists")
+
+ # Are there any recipes with this name which aren't custom
+ # image recipes?
+ custom_image_ids = custom_images.values_list('id', flat=True)
+ existing_non_image_recipes = Recipe.objects.filter(
+ Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
+ )
+
+ if existing_non_image_recipes.count() > 0:
+ return error_response("recipe-already-exists")
+
+ # create layer 'Custom layer' and verion if needed
+ layer, l_created = Layer.objects.get_or_create(
+ name=CustomImageRecipe.LAYER_NAME,
+ summary="Layer for custom recipes")
+
+ if l_created:
+ layer.local_source_dir = "toaster_created_layer"
+ layer.save()
+
+ # Check if we have a layer version already
+ # We don't use get_or_create here because the dirpath will change
+ # and is a required field
+ lver = Layer_Version.objects.filter(Q(project=params['project']) &
+ Q(layer=layer) &
+ Q(build=None)).last()
+ if lver is None:
+ lver, lv_created = Layer_Version.objects.get_or_create(
+ project=params['project'],
+ layer=layer,
+ layer_source=LayerSource.TYPE_LOCAL,
+ dirpath="toaster_created_layer")
+
+ # Add a dependency on our layer to the base recipe's layer
+ LayerVersionDependency.objects.get_or_create(
+ layer_version=lver,
+ depends_on=params["base"].layer_version)
+
+ # Add it to our current project if needed
+ ProjectLayer.objects.get_or_create(project=params['project'],
+ layercommit=lver,
+ optional=False)
+
+ # Create the actual recipe
+ recipe, r_created = CustomImageRecipe.objects.get_or_create(
+ name=request.POST["name"],
+ base_recipe=params["base"],
+ project=params["project"],
+ layer_version=lver,
+ is_image=True)
+
+ # If we created the object then setup these fields. They may get
+ # overwritten later on and cause the get_or_create to create a
+ # duplicate if they've changed.
+ if r_created:
+ recipe.file_path = request.POST["name"]
+ recipe.license = "MIT"
+ recipe.version = "0.1"
+ recipe.save()
+
+ except Error as err:
+ return error_response("Can't create custom recipe: %s" % err)
+
+ # Find the package list from the last build of this recipe/target
+ target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
+ Q(build__project=params['project']) &
+ (Q(target=params['base'].name) |
+ Q(target=recipe.name))).last()
+ if target:
+ # Copy in every package
+ # We don't want these packages to be linked to anything because
+ # that underlying data may change e.g. delete a build
+ for tpackage in target.target_installed_package_set.all():
+ try:
+ built_package = tpackage.package
+ # The package had no recipe information so is a ghost
+ # package skip it
+ if built_package.recipe is None:
+ continue
+
+ config_package = CustomImagePackage.objects.get(
+ name=built_package.name)
+
+ recipe.includes_set.add(config_package)
+ except Exception as e:
+ logger.warning("Error adding package %s %s" %
+ (tpackage.package.name, e))
+ pass
+
+ # pre-create layer directory structure, so that other builds
+ # are not blocked by this new recipe dependecy
+ # NOTE: this is parallel code to 'localhostbecontroller.py'
+ be = BuildEnvironment.objects.all()[0]
+ layerpath = os.path.join(be.builddir,
+ CustomImageRecipe.LAYER_NAME)
+ for name in ("conf", "recipes"):
+ path = os.path.join(layerpath, name)
+ if not os.path.isdir(path):
+ os.makedirs(path)
+ # pre-create layer.conf
+ config = os.path.join(layerpath, "conf", "layer.conf")
+ if not os.path.isfile(config):
+ with open(config, "w") as conf:
+ conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
+ # pre-create new image's recipe file
+ recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
+ recipe.name)
+ with open(recipe_path, "w") as recipef:
+ content = recipe.generate_recipe_file_contents()
+ if not content:
+ # Delete this incomplete image recipe object
+ recipe.delete()
+ return error_response("recipe-parent-not-exist")
+ else:
+ recipef.write(recipe.generate_recipe_file_contents())
+
+ return JsonResponse(
+ {"error": "ok",
+ "packages": recipe.get_all_packages().count(),
+ "url": reverse('customrecipe', args=(params['project'].pk,
+ recipe.id))})
+
+
+class XhrCustomRecipeId(View):
+ """
+ Set of ReST API processors working with recipe id.
+
+ Entry point: /xhr_customrecipe/<recipe_id>
+
+ Methods:
+ GET - Get details of custom image recipe
+ DELETE - Delete custom image recipe
+
+ Returns:
+ GET:
+ {"error": "ok",
+ "info": dictionary of field name -> value pairs
+ of the CustomImageRecipe model}
+ DELETE:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+ @staticmethod
+ def _get_ci_recipe(recipe_id):
+ """ Get Custom Image recipe or return an error response"""
+ try:
+ custom_recipe = \
+ CustomImageRecipe.objects.get(pk=recipe_id)
+ return custom_recipe, None
+
+ except CustomImageRecipe.DoesNotExist:
+ return None, error_response("Custom recipe with id=%s "
+ "not found" % recipe_id)
+
+ def get(self, request, *args, **kwargs):
+ custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
+ if error:
+ return error
+
+ if request.method == 'GET':
+ info = {"id": custom_recipe.id,
+ "name": custom_recipe.name,
+ "base_recipe_id": custom_recipe.base_recipe.id,
+ "project_id": custom_recipe.project.id}
+
+ return JsonResponse({"error": "ok", "info": info})
+
+ def delete(self, request, *args, **kwargs):
+ custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
+ if error:
+ return error
+
+ project = custom_recipe.project
+
+ custom_recipe.delete()
+ return JsonResponse({"error": "ok",
+ "gotoUrl": reverse("projectcustomimages",
+ args=(project.pk,))})
+
+
+class XhrCustomRecipePackages(View):
+ """
+ ReST API to add/remove packages to/from custom recipe.
+
+ Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
+ Methods:
+ PUT - Add package to the recipe
+ DELETE - Delete package from the recipe
+ GET - Get package information
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+ @staticmethod
+ def _get_package(package_id):
+ try:
+ package = CustomImagePackage.objects.get(pk=package_id)
+ return package, None
+ except Package.DoesNotExist:
+ return None, error_response("Package with id=%s "
+ "not found" % package_id)
+
+ def _traverse_dependents(self, next_package_id,
+ rev_deps, all_current_packages, tree_level=0):
+ """
+ Recurse through reverse dependency tree for next_package_id.
+ Limit the reverse dependency search to packages not already scanned,
+ that is, not already in rev_deps.
+ Limit the scan to a depth (tree_level) not exceeding the count of
+ all packages in the custom image, and if that depth is exceeded
+ return False, pop out of the recursion, and write a warning
+ to the log, but this is unlikely, suggesting a dependency loop
+ not caught by bitbake.
+ On return, the input/output arg rev_deps is appended with queryset
+ dictionary elements, annotated for use in the customimage template.
+ The list has unsorted, but unique elements.
+ """
+ max_dependency_tree_depth = all_current_packages.count()
+ if tree_level >= max_dependency_tree_depth:
+ logger.warning(
+ "The number of reverse dependencies "
+ "for this package exceeds " + max_dependency_tree_depth +
+ " and the remaining reverse dependencies will not be removed")
+ return True
+
+ package = CustomImagePackage.objects.get(id=next_package_id)
+ dependents = \
+ package.package_dependencies_target.annotate(
+ name=F('package__name'),
+ pk=F('package__pk'),
+ size=F('package__size'),
+ ).values("name", "pk", "size").exclude(
+ ~Q(pk__in=all_current_packages)
+ )
+
+ for pkg in dependents:
+ if pkg in rev_deps:
+ # already seen, skip dependent search
+ continue
+
+ rev_deps.append(pkg)
+ if (self._traverse_dependents(pkg["pk"], rev_deps,
+ all_current_packages,
+ tree_level+1)):
+ return True
+
+ return False
+
+ def _get_all_dependents(self, package_id, all_current_packages):
+ """
+ Returns sorted list of recursive reverse dependencies for package_id,
+ as a list of dictionary items, by recursing through dependency
+ relationships.
+ """
+ rev_deps = []
+ self._traverse_dependents(package_id, rev_deps, all_current_packages)
+ rev_deps = sorted(rev_deps, key=lambda x: x["name"])
+ return rev_deps
+
+ def get(self, request, *args, **kwargs):
+ recipe, error = XhrCustomRecipeId._get_ci_recipe(
+ kwargs['recipe_id'])
+ if error:
+ return error
+
+ # If no package_id then list all the current packages
+ if not kwargs['package_id']:
+ total_size = 0
+ packages = recipe.get_all_packages().values("id",
+ "name",
+ "version",
+ "size")
+ for package in packages:
+ package['size_formatted'] = \
+ filtered_filesizeformat(package['size'])
+ total_size += package['size']
+
+ return JsonResponse({"error": "ok",
+ "packages": list(packages),
+ "total": len(packages),
+ "total_size": total_size,
+ "total_size_formatted":
+ filtered_filesizeformat(total_size)})
+ else:
+ package, error = XhrCustomRecipePackages._get_package(
+ kwargs['package_id'])
+ if error:
+ return error
+
+ all_current_packages = recipe.get_all_packages()
+
+ # Dependencies for package which aren't satisfied by the
+ # current packages in the custom image recipe
+ deps = package.package_dependencies_source.for_target_or_none(
+ recipe.name)['packages'].annotate(
+ name=F('depends_on__name'),
+ pk=F('depends_on__pk'),
+ size=F('depends_on__size'),
+ ).values("name", "pk", "size").filter(
+ # There are two depends types we don't know why
+ (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
+ Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
+ ~Q(pk__in=all_current_packages)
+ )
+
+ # Reverse dependencies which are needed by packages that are
+ # in the image. Recursive search providing all dependents,
+ # not just immediate dependents.
+ reverse_deps = self._get_all_dependents(kwargs['package_id'],
+ all_current_packages)
+ total_size_deps = 0
+ total_size_reverse_deps = 0
+
+ for dep in deps:
+ dep['size_formatted'] = \
+ filtered_filesizeformat(dep['size'])
+ total_size_deps += dep['size']
+
+ for dep in reverse_deps:
+ dep['size_formatted'] = \
+ filtered_filesizeformat(dep['size'])
+ total_size_reverse_deps += dep['size']
+
+ return JsonResponse(
+ {"error": "ok",
+ "id": package.pk,
+ "name": package.name,
+ "version": package.version,
+ "unsatisfied_dependencies": list(deps),
+ "unsatisfied_dependencies_size": total_size_deps,
+ "unsatisfied_dependencies_size_formatted":
+ filtered_filesizeformat(total_size_deps),
+ "reverse_dependencies": list(reverse_deps),
+ "reverse_dependencies_size": total_size_reverse_deps,
+ "reverse_dependencies_size_formatted":
+ filtered_filesizeformat(total_size_reverse_deps)})
+
+ def put(self, request, *args, **kwargs):
+ recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
+ package, error = self._get_package(kwargs['package_id'])
+ if error:
+ return error
+
+ included_packages = recipe.includes_set.values_list('pk',
+ flat=True)
+
+ # If we're adding back a package which used to be included in this
+ # image all we need to do is remove it from the excludes
+ if package.pk in included_packages:
+ try:
+ recipe.excludes_set.remove(package)
+ return {"error": "ok"}
+ except Package.DoesNotExist:
+ return error_response("Package %s not found in excludes"
+ " but was in included list" %
+ package.name)
+ else:
+ recipe.appends_set.add(package)
+ # Make sure that package is not in the excludes set
+ try:
+ recipe.excludes_set.remove(package)
+ except:
+ pass
+
+ # Add the dependencies we think will be added to the recipe
+ # as a result of appending this package.
+ # TODO this should recurse down the entire deps tree
+ for dep in package.package_dependencies_source.all_depends():
+ try:
+ cust_package = CustomImagePackage.objects.get(
+ name=dep.depends_on.name)
+
+ recipe.includes_set.add(cust_package)
+ try:
+ # When adding the pre-requisite package, make
+ # sure it's not in the excluded list from a
+ # prior removal.
+ recipe.excludes_set.remove(cust_package)
+ except package.DoesNotExist:
+ # Don't care if the package had never been excluded
+ pass
+ except:
+ logger.warning("Could not add package's suggested"
+ "dependencies to the list")
+ return JsonResponse({"error": "ok"})
+
+ def delete(self, request, *args, **kwargs):
+ recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
+ package, error = self._get_package(kwargs['package_id'])
+ if error:
+ return error
+
+ try:
+ included_packages = recipe.includes_set.values_list('pk',
+ flat=True)
+ # If we're deleting a package which is included we need to
+ # Add it to the excludes list.
+ if package.pk in included_packages:
+ recipe.excludes_set.add(package)
+ else:
+ recipe.appends_set.remove(package)
+
+ # remove dependencies as well
+ all_current_packages = recipe.get_all_packages()
+
+ reverse_deps_dictlist = self._get_all_dependents(
+ package.pk,
+ all_current_packages)
+
+ ids = [entry['pk'] for entry in reverse_deps_dictlist]
+ reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
+ for r in reverse_deps:
+ try:
+ if r.id in included_packages:
+ recipe.excludes_set.add(r)
+ else:
+ recipe.appends_set.remove(r)
+ except:
+ pass
+
+ return JsonResponse({"error": "ok"})
+ except CustomImageRecipe.DoesNotExist:
+ return error_response("Tried to remove package that wasn't"
+ " present")
+
+
+class XhrProject(View):
+ """ Create, delete or edit a project
+
+ Entry point: /xhr_project/<project_id>
+ """
+ def post(self, request, *args, **kwargs):
+ """
+ Edit project control
+
+ Args:
+ layerAdd = layer_version_id layer_version_id ...
+ layerDel = layer_version_id layer_version_id ...
+ projectName = new_project_name
+ machineName = new_machine_name
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+ try:
+ prj = Project.objects.get(pk=kwargs['project_id'])
+ except Project.DoesNotExist:
+ return error_response("No such project")
+
+ # Add layers
+ if 'layerAdd' in request.POST and len(request.POST['layerAdd']) > 0:
+ for layer_version_id in request.POST['layerAdd'].split(','):
+ try:
+ lv = Layer_Version.objects.get(pk=int(layer_version_id))
+ ProjectLayer.objects.get_or_create(project=prj,
+ layercommit=lv)
+ except Layer_Version.DoesNotExist:
+ return error_response("Layer version %s asked to add "
+ "doesn't exist" % layer_version_id)
+
+ # Remove layers
+ if 'layerDel' in request.POST and len(request.POST['layerDel']) > 0:
+ layer_version_ids = request.POST['layerDel'].split(',')
+ ProjectLayer.objects.filter(
+ project=prj,
+ layercommit_id__in=layer_version_ids).delete()
+
+ # Project name change
+ if 'projectName' in request.POST:
+ prj.name = request.POST['projectName']
+ prj.save()
+
+ # Machine name change
+ if 'machineName' in request.POST:
+ machinevar = prj.projectvariable_set.get(name="MACHINE")
+ machinevar.value = request.POST['machineName']
+ machinevar.save()
+
+ # Distro name change
+ if 'distroName' in request.POST:
+ distrovar = prj.projectvariable_set.get(name="DISTRO")
+ distrovar.value = request.POST['distroName']
+ distrovar.save()
+
+ return JsonResponse({"error": "ok"})
+
+ def get(self, request, *args, **kwargs):
+ """
+ Returns:
+ json object representing the current project
+ or:
+ {"error": <error message>}
+ """
+
+ try:
+ project = Project.objects.get(pk=kwargs['project_id'])
+ except Project.DoesNotExist:
+ return error_response("Project %s does not exist" %
+ kwargs['project_id'])
+
+ # Create the frequently built targets list
+
+ freqtargets = Counter(Target.objects.filter(
+ Q(build__project=project),
+ ~Q(build__outcome=Build.IN_PROGRESS)
+ ).order_by("target").values_list("target", flat=True))
+
+ freqtargets = freqtargets.most_common(5)
+
+ # We now have the targets in order of frequency but if there are two
+ # with the same frequency then we need to make sure those are in
+ # alphabetical order without losing the frequency ordering
+
+ tmp = []
+ switch = None
+ for i, freqtartget in enumerate(freqtargets):
+ target, count = freqtartget
+ try:
+ target_next, count_next = freqtargets[i+1]
+ if count == count_next and target > target_next:
+ switch = target
+ continue
+ except IndexError:
+ pass
+
+ tmp.append(target)
+
+ if switch:
+ tmp.append(switch)
+ switch = None
+
+ freqtargets = tmp
+
+ layers = []
+ for layer in project.projectlayer_set.all():
+ layers.append({
+ "id": layer.layercommit.pk,
+ "name": layer.layercommit.layer.name,
+ "vcs_url": layer.layercommit.layer.vcs_url,
+ "local_source_dir": layer.layercommit.layer.local_source_dir,
+ "vcs_reference": layer.layercommit.get_vcs_reference(),
+ "url": layer.layercommit.layer.layer_index_url,
+ "layerdetailurl": layer.layercommit.get_detailspage_url(
+ project.pk),
+ "xhrLayerUrl": reverse("xhr_layer",
+ args=(project.pk,
+ layer.layercommit.pk)),
+ "layersource": layer.layercommit.layer_source
+ })
+
+ data = {
+ "name": project.name,
+ "layers": layers,
+ "freqtargets": freqtargets,
+ }
+
+ if project.release is not None:
+ data['release'] = {
+ "id": project.release.pk,
+ "name": project.release.name,
+ "description": project.release.description
+ }
+
+ try:
+ data["machine"] = {"name":
+ project.projectvariable_set.get(
+ name="MACHINE").value}
+ except ProjectVariable.DoesNotExist:
+ data["machine"] = None
+ try:
+ data["distro"] = {"name":
+ project.projectvariable_set.get(
+ name="DISTRO").value}
+ except ProjectVariable.DoesNotExist:
+ data["distro"] = None
+
+ data['error'] = "ok"
+
+ return JsonResponse(data)
+
+ def put(self, request, *args, **kwargs):
+ # TODO create new project api
+ return HttpResponse()
+
+ def delete(self, request, *args, **kwargs):
+ """Delete a project. Cancels any builds in progress"""
+ try:
+ project = Project.objects.get(pk=kwargs['project_id'])
+ # Cancel any builds in progress
+ for br in BuildRequest.objects.filter(
+ project=project,
+ state=BuildRequest.REQ_INPROGRESS):
+ XhrBuildRequest.cancel_build(br)
+
+ # gather potential orphaned local layers attached to this project
+ project_local_layer_list = []
+ for pl in ProjectLayer.objects.filter(project=project):
+ if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED:
+ project_local_layer_list.append(pl.layercommit.layer)
+
+ # deep delete the project and its dependencies
+ project.delete()
+
+ # delete any local layers now orphaned
+ _log("LAYER_ORPHAN_CHECK:Check for orphaned layers")
+ for layer in project_local_layer_list:
+ layer_refs = Layer_Version.objects.filter(layer=layer)
+ _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs)))
+ if 0 == len(layer_refs):
+ _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name))
+ Layer.objects.filter(pk=layer.id).delete()
+
+ except Project.DoesNotExist:
+ return error_response("Project %s does not exist" %
+ kwargs['project_id'])
+
+ return JsonResponse({
+ "error": "ok",
+ "gotoUrl": reverse("all-projects", args=[])
+ })
+
+
+class XhrBuild(View):
+ """ Delete a build object
+
+ Entry point: /xhr_build/<build_id>
+ """
+ def delete(self, request, *args, **kwargs):
+ """
+ Delete build data
+
+ Args:
+ build_id = build_id
+
+ Returns:
+ {"error": "ok"}
+ or
+ {"error": <error message>}
+ """
+ try:
+ build = Build.objects.get(pk=kwargs['build_id'])
+ project = build.project
+ build.delete()
+ except Build.DoesNotExist:
+ return error_response("Build %s does not exist" %
+ kwargs['build_id'])
+ return JsonResponse({
+ "error": "ok",
+ "gotoUrl": reverse("projectbuilds", args=(project.pk,))
+ })