rjw | 1f88458 | 2022-01-06 17:20:42 +0800 | [diff] [blame^] | 1 | #!/usr/bin/env python |
| 2 | # ex:ts=4:sw=4:sts=4:et |
| 3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- |
| 4 | # |
| 5 | # Copyright (C) 2003, 2004 Chris Larson |
| 6 | # Copyright (C) 2003, 2004 Phil Blundell |
| 7 | # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer |
| 8 | # Copyright (C) 2005 Holger Hans Peter Freyther |
| 9 | # Copyright (C) 2005 ROAD GmbH |
| 10 | # Copyright (C) 2006 Richard Purdie |
| 11 | # |
| 12 | # This program is free software; you can redistribute it and/or modify |
| 13 | # it under the terms of the GNU General Public License version 2 as |
| 14 | # published by the Free Software Foundation. |
| 15 | # |
| 16 | # This program is distributed in the hope that it will be useful, |
| 17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 19 | # GNU General Public License for more details. |
| 20 | # |
| 21 | # You should have received a copy of the GNU General Public License along |
| 22 | # with this program; if not, write to the Free Software Foundation, Inc., |
| 23 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 24 | |
| 25 | import logging |
| 26 | import os |
| 27 | import re |
| 28 | import sys |
| 29 | import hashlib |
| 30 | from functools import wraps |
| 31 | import bb |
| 32 | from bb import data |
| 33 | import bb.parse |
| 34 | |
| 35 | logger = logging.getLogger("BitBake") |
| 36 | parselog = logging.getLogger("BitBake.Parsing") |
| 37 | |
| 38 | class ConfigParameters(object): |
| 39 | def __init__(self, argv=sys.argv): |
| 40 | self.options, targets = self.parseCommandLine(argv) |
| 41 | self.environment = self.parseEnvironment() |
| 42 | |
| 43 | self.options.pkgs_to_build = targets or [] |
| 44 | |
| 45 | for key, val in self.options.__dict__.items(): |
| 46 | setattr(self, key, val) |
| 47 | |
| 48 | def parseCommandLine(self, argv=sys.argv): |
| 49 | raise Exception("Caller must implement commandline option parsing") |
| 50 | |
| 51 | def parseEnvironment(self): |
| 52 | return os.environ.copy() |
| 53 | |
| 54 | def updateFromServer(self, server): |
| 55 | if not self.options.cmd: |
| 56 | defaulttask, error = server.runCommand(["getVariable", "BB_DEFAULT_TASK"]) |
| 57 | if error: |
| 58 | raise Exception("Unable to get the value of BB_DEFAULT_TASK from the server: %s" % error) |
| 59 | self.options.cmd = defaulttask or "build" |
| 60 | _, error = server.runCommand(["setConfig", "cmd", self.options.cmd]) |
| 61 | if error: |
| 62 | raise Exception("Unable to set configuration option 'cmd' on the server: %s" % error) |
| 63 | |
| 64 | if not self.options.pkgs_to_build: |
| 65 | bbpkgs, error = server.runCommand(["getVariable", "BBTARGETS"]) |
| 66 | if error: |
| 67 | raise Exception("Unable to get the value of BBTARGETS from the server: %s" % error) |
| 68 | if bbpkgs: |
| 69 | self.options.pkgs_to_build.extend(bbpkgs.split()) |
| 70 | |
| 71 | def updateToServer(self, server, environment): |
| 72 | options = {} |
| 73 | for o in ["abort", "force", "invalidate_stamp", |
| 74 | "verbose", "debug", "dry_run", "dump_signatures", |
| 75 | "debug_domains", "extra_assume_provided", "profile", |
| 76 | "prefile", "postfile", "server_timeout"]: |
| 77 | options[o] = getattr(self.options, o) |
| 78 | |
| 79 | ret, error = server.runCommand(["updateConfig", options, environment, sys.argv]) |
| 80 | if error: |
| 81 | raise Exception("Unable to update the server configuration with local parameters: %s" % error) |
| 82 | |
| 83 | def parseActions(self): |
| 84 | # Parse any commandline into actions |
| 85 | action = {'action':None, 'msg':None} |
| 86 | if self.options.show_environment: |
| 87 | if 'world' in self.options.pkgs_to_build: |
| 88 | action['msg'] = "'world' is not a valid target for --environment." |
| 89 | elif 'universe' in self.options.pkgs_to_build: |
| 90 | action['msg'] = "'universe' is not a valid target for --environment." |
| 91 | elif len(self.options.pkgs_to_build) > 1: |
| 92 | action['msg'] = "Only one target can be used with the --environment option." |
| 93 | elif self.options.buildfile and len(self.options.pkgs_to_build) > 0: |
| 94 | action['msg'] = "No target should be used with the --environment and --buildfile options." |
| 95 | elif len(self.options.pkgs_to_build) > 0: |
| 96 | action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build] |
| 97 | else: |
| 98 | action['action'] = ["showEnvironment", self.options.buildfile] |
| 99 | elif self.options.buildfile is not None: |
| 100 | action['action'] = ["buildFile", self.options.buildfile, self.options.cmd] |
| 101 | elif self.options.revisions_changed: |
| 102 | action['action'] = ["compareRevisions"] |
| 103 | elif self.options.show_versions: |
| 104 | action['action'] = ["showVersions"] |
| 105 | elif self.options.parse_only: |
| 106 | action['action'] = ["parseFiles"] |
| 107 | elif self.options.dot_graph: |
| 108 | if self.options.pkgs_to_build: |
| 109 | action['action'] = ["generateDotGraph", self.options.pkgs_to_build, self.options.cmd] |
| 110 | else: |
| 111 | action['msg'] = "Please specify a package name for dependency graph generation." |
| 112 | else: |
| 113 | if self.options.pkgs_to_build: |
| 114 | action['action'] = ["buildTargets", self.options.pkgs_to_build, self.options.cmd] |
| 115 | else: |
| 116 | #action['msg'] = "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information." |
| 117 | action = None |
| 118 | self.options.initialaction = action |
| 119 | return action |
| 120 | |
| 121 | class CookerConfiguration(object): |
| 122 | """ |
| 123 | Manages build options and configurations for one run |
| 124 | """ |
| 125 | |
| 126 | def __init__(self): |
| 127 | self.debug_domains = [] |
| 128 | self.extra_assume_provided = [] |
| 129 | self.prefile = [] |
| 130 | self.postfile = [] |
| 131 | self.debug = 0 |
| 132 | self.cmd = None |
| 133 | self.abort = True |
| 134 | self.force = False |
| 135 | self.profile = False |
| 136 | self.nosetscene = False |
| 137 | self.setsceneonly = False |
| 138 | self.invalidate_stamp = False |
| 139 | self.dump_signatures = [] |
| 140 | self.dry_run = False |
| 141 | self.tracking = False |
| 142 | self.xmlrpcinterface = [] |
| 143 | self.server_timeout = None |
| 144 | self.writeeventlog = False |
| 145 | self.server_only = False |
| 146 | self.limited_deps = False |
| 147 | self.runall = [] |
| 148 | self.runonly = [] |
| 149 | |
| 150 | self.env = {} |
| 151 | |
| 152 | def setConfigParameters(self, parameters): |
| 153 | for key in self.__dict__.keys(): |
| 154 | if key in parameters.options.__dict__: |
| 155 | setattr(self, key, parameters.options.__dict__[key]) |
| 156 | self.env = parameters.environment.copy() |
| 157 | |
| 158 | def setServerRegIdleCallback(self, srcb): |
| 159 | self.server_register_idlecallback = srcb |
| 160 | |
| 161 | def __getstate__(self): |
| 162 | state = {} |
| 163 | for key in self.__dict__.keys(): |
| 164 | if key == "server_register_idlecallback": |
| 165 | state[key] = None |
| 166 | else: |
| 167 | state[key] = getattr(self, key) |
| 168 | return state |
| 169 | |
| 170 | def __setstate__(self,state): |
| 171 | for k in state: |
| 172 | setattr(self, k, state[k]) |
| 173 | |
| 174 | |
| 175 | def catch_parse_error(func): |
| 176 | """Exception handling bits for our parsing""" |
| 177 | @wraps(func) |
| 178 | def wrapped(fn, *args): |
| 179 | try: |
| 180 | return func(fn, *args) |
| 181 | except IOError as exc: |
| 182 | import traceback |
| 183 | parselog.critical(traceback.format_exc()) |
| 184 | parselog.critical("Unable to parse %s: %s" % (fn, exc)) |
| 185 | sys.exit(1) |
| 186 | except bb.data_smart.ExpansionError as exc: |
| 187 | import traceback |
| 188 | |
| 189 | bbdir = os.path.dirname(__file__) + os.sep |
| 190 | exc_class, exc, tb = sys.exc_info() |
| 191 | for tb in iter(lambda: tb.tb_next, None): |
| 192 | # Skip frames in bitbake itself, we only want the metadata |
| 193 | fn, _, _, _ = traceback.extract_tb(tb, 1)[0] |
| 194 | if not fn.startswith(bbdir): |
| 195 | break |
| 196 | parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb)) |
| 197 | sys.exit(1) |
| 198 | except bb.parse.ParseError as exc: |
| 199 | parselog.critical(str(exc)) |
| 200 | sys.exit(1) |
| 201 | return wrapped |
| 202 | |
| 203 | @catch_parse_error |
| 204 | def parse_config_file(fn, data, include=True): |
| 205 | return bb.parse.handle(fn, data, include) |
| 206 | |
| 207 | @catch_parse_error |
| 208 | def _inherit(bbclass, data): |
| 209 | bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data) |
| 210 | return data |
| 211 | |
| 212 | def findConfigFile(configfile, data): |
| 213 | search = [] |
| 214 | bbpath = data.getVar("BBPATH") |
| 215 | if bbpath: |
| 216 | for i in bbpath.split(":"): |
| 217 | search.append(os.path.join(i, "conf", configfile)) |
| 218 | path = os.getcwd() |
| 219 | while path != "/": |
| 220 | search.append(os.path.join(path, "conf", configfile)) |
| 221 | path, _ = os.path.split(path) |
| 222 | |
| 223 | for i in search: |
| 224 | if os.path.exists(i): |
| 225 | return i |
| 226 | |
| 227 | return None |
| 228 | |
| 229 | # |
| 230 | # We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working |
| 231 | # up to /. If that fails, we search for a conf/bitbake.conf in BBPATH. |
| 232 | # |
| 233 | |
| 234 | def findTopdir(): |
| 235 | d = bb.data.init() |
| 236 | bbpath = None |
| 237 | if 'BBPATH' in os.environ: |
| 238 | bbpath = os.environ['BBPATH'] |
| 239 | d.setVar('BBPATH', bbpath) |
| 240 | |
| 241 | layerconf = findConfigFile("bblayers.conf", d) |
| 242 | if layerconf: |
| 243 | return os.path.dirname(os.path.dirname(layerconf)) |
| 244 | if bbpath: |
| 245 | bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf") |
| 246 | if bitbakeconf: |
| 247 | return os.path.dirname(os.path.dirname(bitbakeconf)) |
| 248 | return None |
| 249 | |
| 250 | class CookerDataBuilder(object): |
| 251 | |
| 252 | def __init__(self, cookercfg, worker = False): |
| 253 | |
| 254 | self.prefiles = cookercfg.prefile |
| 255 | self.postfiles = cookercfg.postfile |
| 256 | self.tracking = cookercfg.tracking |
| 257 | |
| 258 | bb.utils.set_context(bb.utils.clean_context()) |
| 259 | bb.event.set_class_handlers(bb.event.clean_class_handlers()) |
| 260 | self.basedata = bb.data.init() |
| 261 | if self.tracking: |
| 262 | self.basedata.enableTracking() |
| 263 | |
| 264 | # Keep a datastore of the initial environment variables and their |
| 265 | # values from when BitBake was launched to enable child processes |
| 266 | # to use environment variables which have been cleaned from the |
| 267 | # BitBake processes env |
| 268 | self.savedenv = bb.data.init() |
| 269 | for k in cookercfg.env: |
| 270 | self.savedenv.setVar(k, cookercfg.env[k]) |
| 271 | |
| 272 | filtered_keys = bb.utils.approved_variables() |
| 273 | bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys) |
| 274 | self.basedata.setVar("BB_ORIGENV", self.savedenv) |
| 275 | |
| 276 | if worker: |
| 277 | self.basedata.setVar("BB_WORKERCONTEXT", "1") |
| 278 | |
| 279 | self.data = self.basedata |
| 280 | self.mcdata = {} |
| 281 | |
| 282 | def parseBaseConfiguration(self): |
| 283 | data_hash = hashlib.sha256() |
| 284 | try: |
| 285 | bb.parse.init_parser(self.basedata) |
| 286 | self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) |
| 287 | |
| 288 | if self.data.getVar("BB_WORKERCONTEXT", False) is None: |
| 289 | bb.fetch.fetcher_init(self.data) |
| 290 | bb.codeparser.parser_cache_init(self.data) |
| 291 | |
| 292 | bb.event.fire(bb.event.ConfigParsed(), self.data) |
| 293 | |
| 294 | reparse_cnt = 0 |
| 295 | while self.data.getVar("BB_INVALIDCONF", False) is True: |
| 296 | if reparse_cnt > 20: |
| 297 | logger.error("Configuration has been re-parsed over 20 times, " |
| 298 | "breaking out of the loop...") |
| 299 | raise Exception("Too deep config re-parse loop. Check locations where " |
| 300 | "BB_INVALIDCONF is being set (ConfigParsed event handlers)") |
| 301 | self.data.setVar("BB_INVALIDCONF", False) |
| 302 | self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) |
| 303 | reparse_cnt += 1 |
| 304 | bb.event.fire(bb.event.ConfigParsed(), self.data) |
| 305 | |
| 306 | bb.parse.init_parser(self.data) |
| 307 | data_hash.update(self.data.get_hash().encode('utf-8')) |
| 308 | self.mcdata[''] = self.data |
| 309 | |
| 310 | multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() |
| 311 | for config in multiconfig: |
| 312 | mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) |
| 313 | bb.event.fire(bb.event.ConfigParsed(), mcdata) |
| 314 | self.mcdata[config] = mcdata |
| 315 | data_hash.update(mcdata.get_hash().encode('utf-8')) |
| 316 | if multiconfig: |
| 317 | bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data) |
| 318 | |
| 319 | self.data_hash = data_hash.hexdigest() |
| 320 | except (SyntaxError, bb.BBHandledException): |
| 321 | raise bb.BBHandledException |
| 322 | except bb.data_smart.ExpansionError as e: |
| 323 | logger.error(str(e)) |
| 324 | raise bb.BBHandledException |
| 325 | except Exception: |
| 326 | logger.exception("Error parsing configuration files") |
| 327 | raise bb.BBHandledException |
| 328 | |
| 329 | # Create a copy so we can reset at a later date when UIs disconnect |
| 330 | self.origdata = self.data |
| 331 | self.data = bb.data.createCopy(self.origdata) |
| 332 | self.mcdata[''] = self.data |
| 333 | |
| 334 | def reset(self): |
| 335 | # We may not have run parseBaseConfiguration() yet |
| 336 | if not hasattr(self, 'origdata'): |
| 337 | return |
| 338 | self.data = bb.data.createCopy(self.origdata) |
| 339 | self.mcdata[''] = self.data |
| 340 | |
| 341 | def _findLayerConf(self, data): |
| 342 | return findConfigFile("bblayers.conf", data) |
| 343 | |
| 344 | def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"): |
| 345 | data = bb.data.createCopy(self.basedata) |
| 346 | data.setVar("BB_CURRENT_MC", mc) |
| 347 | |
| 348 | # Parse files for loading *before* bitbake.conf and any includes |
| 349 | for f in prefiles: |
| 350 | data = parse_config_file(f, data) |
| 351 | |
| 352 | layerconf = self._findLayerConf(data) |
| 353 | if layerconf: |
| 354 | parselog.debug(2, "Found bblayers.conf (%s)", layerconf) |
| 355 | # By definition bblayers.conf is in conf/ of TOPDIR. |
| 356 | # We may have been called with cwd somewhere else so reset TOPDIR |
| 357 | data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf))) |
| 358 | data = parse_config_file(layerconf, data) |
| 359 | |
| 360 | layers = (data.getVar('BBLAYERS') or "").split() |
| 361 | |
| 362 | data = bb.data.createCopy(data) |
| 363 | approved = bb.utils.approved_variables() |
| 364 | for layer in layers: |
| 365 | if not os.path.isdir(layer): |
| 366 | parselog.critical("Layer directory '%s' does not exist! " |
| 367 | "Please check BBLAYERS in %s" % (layer, layerconf)) |
| 368 | sys.exit(1) |
| 369 | parselog.debug(2, "Adding layer %s", layer) |
| 370 | if 'HOME' in approved and '~' in layer: |
| 371 | layer = os.path.expanduser(layer) |
| 372 | if layer.endswith('/'): |
| 373 | layer = layer.rstrip('/') |
| 374 | data.setVar('LAYERDIR', layer) |
| 375 | data.setVar('LAYERDIR_RE', re.escape(layer)) |
| 376 | data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data) |
| 377 | data.expandVarref('LAYERDIR') |
| 378 | data.expandVarref('LAYERDIR_RE') |
| 379 | |
| 380 | data.delVar('LAYERDIR_RE') |
| 381 | data.delVar('LAYERDIR') |
| 382 | |
| 383 | bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split() |
| 384 | collections = (data.getVar('BBFILE_COLLECTIONS') or "").split() |
| 385 | invalid = [] |
| 386 | for entry in bbfiles_dynamic: |
| 387 | parts = entry.split(":", 1) |
| 388 | if len(parts) != 2: |
| 389 | invalid.append(entry) |
| 390 | continue |
| 391 | l, f = parts |
| 392 | if l in collections: |
| 393 | data.appendVar("BBFILES", " " + f) |
| 394 | if invalid: |
| 395 | bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) |
| 396 | |
| 397 | layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split()) |
| 398 | for c in collections: |
| 399 | compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split()) |
| 400 | if compat and not (compat & layerseries): |
| 401 | bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)" |
| 402 | % (c, " ".join(layerseries), " ".join(compat))) |
| 403 | elif not compat and not data.getVar("BB_WORKERCONTEXT"): |
| 404 | bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c)) |
| 405 | |
| 406 | if not data.getVar("BBPATH"): |
| 407 | msg = "The BBPATH variable is not set" |
| 408 | if not layerconf: |
| 409 | msg += (" and bitbake did not find a conf/bblayers.conf file in" |
| 410 | " the expected location.\nMaybe you accidentally" |
| 411 | " invoked bitbake from the wrong directory?") |
| 412 | raise SystemExit(msg) |
| 413 | |
| 414 | data = parse_config_file(os.path.join("conf", "bitbake.conf"), data) |
| 415 | |
| 416 | # Parse files for loading *after* bitbake.conf and any includes |
| 417 | for p in postfiles: |
| 418 | data = parse_config_file(p, data) |
| 419 | |
| 420 | # Handle any INHERITs and inherit the base class |
| 421 | bbclasses = ["base"] + (data.getVar('INHERIT') or "").split() |
| 422 | for bbclass in bbclasses: |
| 423 | data = _inherit(bbclass, data) |
| 424 | |
| 425 | # Nomally we only register event handlers at the end of parsing .bb files |
| 426 | # We register any handlers we've found so far here... |
| 427 | for var in data.getVar('__BBHANDLERS', False) or []: |
| 428 | handlerfn = data.getVarFlag(var, "filename", False) |
| 429 | if not handlerfn: |
| 430 | parselog.critical("Undefined event handler function '%s'" % var) |
| 431 | sys.exit(1) |
| 432 | handlerln = int(data.getVarFlag(var, "lineno", False)) |
| 433 | bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln) |
| 434 | |
| 435 | data.setVar('BBINCLUDED',bb.parse.get_file_depends(data)) |
| 436 | |
| 437 | return data |
| 438 | |