blob: d9887c714b152667d215692ef6feca6ef484b13b [file] [log] [blame]
rjw1f884582022-01-06 17:20:42 +08001#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright (C) 2003, 2004 Chris Larson
6# Copyright (C) 2003, 2004 Phil Blundell
7# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8# Copyright (C) 2005 Holger Hans Peter Freyther
9# Copyright (C) 2005 ROAD GmbH
10# Copyright (C) 2006 Richard Purdie
11#
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License version 2 as
14# published by the Free Software Foundation.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License along
22# with this program; if not, write to the Free Software Foundation, Inc.,
23# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24
25import logging
26import os
27import re
28import sys
29import hashlib
30from functools import wraps
31import bb
32from bb import data
33import bb.parse
34
35logger = logging.getLogger("BitBake")
36parselog = logging.getLogger("BitBake.Parsing")
37
38class ConfigParameters(object):
39 def __init__(self, argv=sys.argv):
40 self.options, targets = self.parseCommandLine(argv)
41 self.environment = self.parseEnvironment()
42
43 self.options.pkgs_to_build = targets or []
44
45 for key, val in self.options.__dict__.items():
46 setattr(self, key, val)
47
48 def parseCommandLine(self, argv=sys.argv):
49 raise Exception("Caller must implement commandline option parsing")
50
51 def parseEnvironment(self):
52 return os.environ.copy()
53
54 def updateFromServer(self, server):
55 if not self.options.cmd:
56 defaulttask, error = server.runCommand(["getVariable", "BB_DEFAULT_TASK"])
57 if error:
58 raise Exception("Unable to get the value of BB_DEFAULT_TASK from the server: %s" % error)
59 self.options.cmd = defaulttask or "build"
60 _, error = server.runCommand(["setConfig", "cmd", self.options.cmd])
61 if error:
62 raise Exception("Unable to set configuration option 'cmd' on the server: %s" % error)
63
64 if not self.options.pkgs_to_build:
65 bbpkgs, error = server.runCommand(["getVariable", "BBTARGETS"])
66 if error:
67 raise Exception("Unable to get the value of BBTARGETS from the server: %s" % error)
68 if bbpkgs:
69 self.options.pkgs_to_build.extend(bbpkgs.split())
70
71 def updateToServer(self, server, environment):
72 options = {}
73 for o in ["abort", "force", "invalidate_stamp",
74 "verbose", "debug", "dry_run", "dump_signatures",
75 "debug_domains", "extra_assume_provided", "profile",
76 "prefile", "postfile", "server_timeout"]:
77 options[o] = getattr(self.options, o)
78
79 ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
80 if error:
81 raise Exception("Unable to update the server configuration with local parameters: %s" % error)
82
83 def parseActions(self):
84 # Parse any commandline into actions
85 action = {'action':None, 'msg':None}
86 if self.options.show_environment:
87 if 'world' in self.options.pkgs_to_build:
88 action['msg'] = "'world' is not a valid target for --environment."
89 elif 'universe' in self.options.pkgs_to_build:
90 action['msg'] = "'universe' is not a valid target for --environment."
91 elif len(self.options.pkgs_to_build) > 1:
92 action['msg'] = "Only one target can be used with the --environment option."
93 elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
94 action['msg'] = "No target should be used with the --environment and --buildfile options."
95 elif len(self.options.pkgs_to_build) > 0:
96 action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
97 else:
98 action['action'] = ["showEnvironment", self.options.buildfile]
99 elif self.options.buildfile is not None:
100 action['action'] = ["buildFile", self.options.buildfile, self.options.cmd]
101 elif self.options.revisions_changed:
102 action['action'] = ["compareRevisions"]
103 elif self.options.show_versions:
104 action['action'] = ["showVersions"]
105 elif self.options.parse_only:
106 action['action'] = ["parseFiles"]
107 elif self.options.dot_graph:
108 if self.options.pkgs_to_build:
109 action['action'] = ["generateDotGraph", self.options.pkgs_to_build, self.options.cmd]
110 else:
111 action['msg'] = "Please specify a package name for dependency graph generation."
112 else:
113 if self.options.pkgs_to_build:
114 action['action'] = ["buildTargets", self.options.pkgs_to_build, self.options.cmd]
115 else:
116 #action['msg'] = "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information."
117 action = None
118 self.options.initialaction = action
119 return action
120
121class CookerConfiguration(object):
122 """
123 Manages build options and configurations for one run
124 """
125
126 def __init__(self):
127 self.debug_domains = []
128 self.extra_assume_provided = []
129 self.prefile = []
130 self.postfile = []
131 self.debug = 0
132 self.cmd = None
133 self.abort = True
134 self.force = False
135 self.profile = False
136 self.nosetscene = False
137 self.setsceneonly = False
138 self.invalidate_stamp = False
139 self.dump_signatures = []
140 self.dry_run = False
141 self.tracking = False
142 self.xmlrpcinterface = []
143 self.server_timeout = None
144 self.writeeventlog = False
145 self.server_only = False
146 self.limited_deps = False
147 self.runall = []
148 self.runonly = []
149
150 self.env = {}
151
152 def setConfigParameters(self, parameters):
153 for key in self.__dict__.keys():
154 if key in parameters.options.__dict__:
155 setattr(self, key, parameters.options.__dict__[key])
156 self.env = parameters.environment.copy()
157
158 def setServerRegIdleCallback(self, srcb):
159 self.server_register_idlecallback = srcb
160
161 def __getstate__(self):
162 state = {}
163 for key in self.__dict__.keys():
164 if key == "server_register_idlecallback":
165 state[key] = None
166 else:
167 state[key] = getattr(self, key)
168 return state
169
170 def __setstate__(self,state):
171 for k in state:
172 setattr(self, k, state[k])
173
174
175def catch_parse_error(func):
176 """Exception handling bits for our parsing"""
177 @wraps(func)
178 def wrapped(fn, *args):
179 try:
180 return func(fn, *args)
181 except IOError as exc:
182 import traceback
183 parselog.critical(traceback.format_exc())
184 parselog.critical("Unable to parse %s: %s" % (fn, exc))
185 sys.exit(1)
186 except bb.data_smart.ExpansionError as exc:
187 import traceback
188
189 bbdir = os.path.dirname(__file__) + os.sep
190 exc_class, exc, tb = sys.exc_info()
191 for tb in iter(lambda: tb.tb_next, None):
192 # Skip frames in bitbake itself, we only want the metadata
193 fn, _, _, _ = traceback.extract_tb(tb, 1)[0]
194 if not fn.startswith(bbdir):
195 break
196 parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
197 sys.exit(1)
198 except bb.parse.ParseError as exc:
199 parselog.critical(str(exc))
200 sys.exit(1)
201 return wrapped
202
203@catch_parse_error
204def parse_config_file(fn, data, include=True):
205 return bb.parse.handle(fn, data, include)
206
207@catch_parse_error
208def _inherit(bbclass, data):
209 bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
210 return data
211
212def findConfigFile(configfile, data):
213 search = []
214 bbpath = data.getVar("BBPATH")
215 if bbpath:
216 for i in bbpath.split(":"):
217 search.append(os.path.join(i, "conf", configfile))
218 path = os.getcwd()
219 while path != "/":
220 search.append(os.path.join(path, "conf", configfile))
221 path, _ = os.path.split(path)
222
223 for i in search:
224 if os.path.exists(i):
225 return i
226
227 return None
228
229#
230# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
231# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
232#
233
234def findTopdir():
235 d = bb.data.init()
236 bbpath = None
237 if 'BBPATH' in os.environ:
238 bbpath = os.environ['BBPATH']
239 d.setVar('BBPATH', bbpath)
240
241 layerconf = findConfigFile("bblayers.conf", d)
242 if layerconf:
243 return os.path.dirname(os.path.dirname(layerconf))
244 if bbpath:
245 bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf")
246 if bitbakeconf:
247 return os.path.dirname(os.path.dirname(bitbakeconf))
248 return None
249
250class CookerDataBuilder(object):
251
252 def __init__(self, cookercfg, worker = False):
253
254 self.prefiles = cookercfg.prefile
255 self.postfiles = cookercfg.postfile
256 self.tracking = cookercfg.tracking
257
258 bb.utils.set_context(bb.utils.clean_context())
259 bb.event.set_class_handlers(bb.event.clean_class_handlers())
260 self.basedata = bb.data.init()
261 if self.tracking:
262 self.basedata.enableTracking()
263
264 # Keep a datastore of the initial environment variables and their
265 # values from when BitBake was launched to enable child processes
266 # to use environment variables which have been cleaned from the
267 # BitBake processes env
268 self.savedenv = bb.data.init()
269 for k in cookercfg.env:
270 self.savedenv.setVar(k, cookercfg.env[k])
271
272 filtered_keys = bb.utils.approved_variables()
273 bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
274 self.basedata.setVar("BB_ORIGENV", self.savedenv)
275
276 if worker:
277 self.basedata.setVar("BB_WORKERCONTEXT", "1")
278
279 self.data = self.basedata
280 self.mcdata = {}
281
282 def parseBaseConfiguration(self):
283 data_hash = hashlib.sha256()
284 try:
285 bb.parse.init_parser(self.basedata)
286 self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
287
288 if self.data.getVar("BB_WORKERCONTEXT", False) is None:
289 bb.fetch.fetcher_init(self.data)
290 bb.codeparser.parser_cache_init(self.data)
291
292 bb.event.fire(bb.event.ConfigParsed(), self.data)
293
294 reparse_cnt = 0
295 while self.data.getVar("BB_INVALIDCONF", False) is True:
296 if reparse_cnt > 20:
297 logger.error("Configuration has been re-parsed over 20 times, "
298 "breaking out of the loop...")
299 raise Exception("Too deep config re-parse loop. Check locations where "
300 "BB_INVALIDCONF is being set (ConfigParsed event handlers)")
301 self.data.setVar("BB_INVALIDCONF", False)
302 self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
303 reparse_cnt += 1
304 bb.event.fire(bb.event.ConfigParsed(), self.data)
305
306 bb.parse.init_parser(self.data)
307 data_hash.update(self.data.get_hash().encode('utf-8'))
308 self.mcdata[''] = self.data
309
310 multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
311 for config in multiconfig:
312 mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
313 bb.event.fire(bb.event.ConfigParsed(), mcdata)
314 self.mcdata[config] = mcdata
315 data_hash.update(mcdata.get_hash().encode('utf-8'))
316 if multiconfig:
317 bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
318
319 self.data_hash = data_hash.hexdigest()
320 except (SyntaxError, bb.BBHandledException):
321 raise bb.BBHandledException
322 except bb.data_smart.ExpansionError as e:
323 logger.error(str(e))
324 raise bb.BBHandledException
325 except Exception:
326 logger.exception("Error parsing configuration files")
327 raise bb.BBHandledException
328
329 # Create a copy so we can reset at a later date when UIs disconnect
330 self.origdata = self.data
331 self.data = bb.data.createCopy(self.origdata)
332 self.mcdata[''] = self.data
333
334 def reset(self):
335 # We may not have run parseBaseConfiguration() yet
336 if not hasattr(self, 'origdata'):
337 return
338 self.data = bb.data.createCopy(self.origdata)
339 self.mcdata[''] = self.data
340
341 def _findLayerConf(self, data):
342 return findConfigFile("bblayers.conf", data)
343
344 def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
345 data = bb.data.createCopy(self.basedata)
346 data.setVar("BB_CURRENT_MC", mc)
347
348 # Parse files for loading *before* bitbake.conf and any includes
349 for f in prefiles:
350 data = parse_config_file(f, data)
351
352 layerconf = self._findLayerConf(data)
353 if layerconf:
354 parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
355 # By definition bblayers.conf is in conf/ of TOPDIR.
356 # We may have been called with cwd somewhere else so reset TOPDIR
357 data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
358 data = parse_config_file(layerconf, data)
359
360 layers = (data.getVar('BBLAYERS') or "").split()
361
362 data = bb.data.createCopy(data)
363 approved = bb.utils.approved_variables()
364 for layer in layers:
365 if not os.path.isdir(layer):
366 parselog.critical("Layer directory '%s' does not exist! "
367 "Please check BBLAYERS in %s" % (layer, layerconf))
368 sys.exit(1)
369 parselog.debug(2, "Adding layer %s", layer)
370 if 'HOME' in approved and '~' in layer:
371 layer = os.path.expanduser(layer)
372 if layer.endswith('/'):
373 layer = layer.rstrip('/')
374 data.setVar('LAYERDIR', layer)
375 data.setVar('LAYERDIR_RE', re.escape(layer))
376 data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data)
377 data.expandVarref('LAYERDIR')
378 data.expandVarref('LAYERDIR_RE')
379
380 data.delVar('LAYERDIR_RE')
381 data.delVar('LAYERDIR')
382
383 bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
384 collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
385 invalid = []
386 for entry in bbfiles_dynamic:
387 parts = entry.split(":", 1)
388 if len(parts) != 2:
389 invalid.append(entry)
390 continue
391 l, f = parts
392 if l in collections:
393 data.appendVar("BBFILES", " " + f)
394 if invalid:
395 bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid))
396
397 layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
398 for c in collections:
399 compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
400 if compat and not (compat & layerseries):
401 bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
402 % (c, " ".join(layerseries), " ".join(compat)))
403 elif not compat and not data.getVar("BB_WORKERCONTEXT"):
404 bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
405
406 if not data.getVar("BBPATH"):
407 msg = "The BBPATH variable is not set"
408 if not layerconf:
409 msg += (" and bitbake did not find a conf/bblayers.conf file in"
410 " the expected location.\nMaybe you accidentally"
411 " invoked bitbake from the wrong directory?")
412 raise SystemExit(msg)
413
414 data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
415
416 # Parse files for loading *after* bitbake.conf and any includes
417 for p in postfiles:
418 data = parse_config_file(p, data)
419
420 # Handle any INHERITs and inherit the base class
421 bbclasses = ["base"] + (data.getVar('INHERIT') or "").split()
422 for bbclass in bbclasses:
423 data = _inherit(bbclass, data)
424
425 # Nomally we only register event handlers at the end of parsing .bb files
426 # We register any handlers we've found so far here...
427 for var in data.getVar('__BBHANDLERS', False) or []:
428 handlerfn = data.getVarFlag(var, "filename", False)
429 if not handlerfn:
430 parselog.critical("Undefined event handler function '%s'" % var)
431 sys.exit(1)
432 handlerln = int(data.getVarFlag(var, "lineno", False))
433 bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
434
435 data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
436
437 return data
438