Gitiles
Code Review Sign In
192.168.1.100 / T103 / 1f884588077ad3476b9c91cbb0ee9ee0332bbb68 / . / meta / poky / scripts / lib / recipetool / create_npm.py
blob: 0b09ed0b04353d8b33395d4d5bff5b1e27686ff9 [file] [log] [blame]
rjw1f884582022-01-06 17:20:42 +0800[diff] [blame^]1# Recipe creation tool - node.js NPM module support plugin
2#
3# Copyright (C) 2016 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import os
19import sys
20import logging
21import subprocess
22import tempfile
23import shutil
24import json
25from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars
26
27logger = logging.getLogger('recipetool')
28
29
30tinfoil = None
31
32def tinfoil_init(instance):
33 global tinfoil
34 tinfoil = instance
35
36
37class NpmRecipeHandler(RecipeHandler):
38 lockdownpath = None
39
40 def _ensure_npm(self, fixed_setup=False):
41 if not tinfoil.recipes_parsed:
42 tinfoil.parse_recipes()
43 try:
44 rd = tinfoil.parse_recipe('nodejs-native')
45 except bb.providers.NoProvider:
46 if fixed_setup:
47 msg = 'nodejs-native is required for npm but is not available within this SDK'
48 else:
49 msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
50 logger.error(msg)
51 return None
52 bindir = rd.getVar('STAGING_BINDIR_NATIVE')
53 npmpath = os.path.join(bindir, 'npm')
54 if not os.path.exists(npmpath):
55 tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
56 if not os.path.exists(npmpath):
57 logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
58 return None
59 return bindir
60
61 def _handle_license(self, data):
62 '''
63 Handle the license value from an npm package.json file
64 '''
65 license = None
66 if 'license' in data:
67 license = data['license']
68 if isinstance(license, dict):
69 license = license.get('type', None)
70 if license:
71 if 'OR' in license:
72 license = license.replace('OR', '|')
73 license = license.replace('AND', '&')
74 license = license.replace(' ', '_')
75 if not license[0] == '(':
76 license = '(' + license + ')'
77 else:
78 license = license.replace('AND', '&')
79 if license[0] == '(':
80 license = license[1:]
81 if license[-1] == ')':
82 license = license[:-1]
83 license = license.replace('MIT/X11', 'MIT')
84 license = license.replace('Public Domain', 'PD')
85 license = license.replace('SEE LICENSE IN EULA',
86 'SEE-LICENSE-IN-EULA')
87 return license
88
89 def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
90 try:
91 runenv = dict(os.environ, PATH=d.getVar('PATH'))
92 bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
93 except bb.process.ExecutionError as e:
94 logger.warning('npm shrinkwrap failed:\n%s' % e.stdout)
95 return
96
97 tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
98 shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
99 extravalues.setdefault('extrafiles', {})
100 extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
101 lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
102
103 def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
104 runenv = dict(os.environ, PATH=d.getVar('PATH'))
105 if not NpmRecipeHandler.lockdownpath:
106 NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
107 bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
108 cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
109 relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
110 if not os.path.exists(relockbin):
111 logger.warning('Could not find relock.js within lockdown directory; skipping lockdown')
112 return
113 try:
114 bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
115 except bb.process.ExecutionError as e:
116 logger.warning('lockdown-relock failed:\n%s' % e.stdout)
117 return
118
119 tmpfile = os.path.join(localfilesdir, 'lockdown.json')
120 shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
121 extravalues.setdefault('extrafiles', {})
122 extravalues['extrafiles']['lockdown.json'] = tmpfile
123 lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
124
125 def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
126 import scriptutils
127 # If this isn't a single module we need to get the dependencies
128 # and add them to SRC_URI
129 def varfunc(varname, origvalue, op, newlines):
130 if varname == 'SRC_URI':
131 if not origvalue.startswith('npm://'):
132 src_uri = origvalue.split()
133 deplist = {}
134 for dep, depver in optdeps.items():
135 depdata = self.get_npm_data(dep, depver, d)
136 if self.check_npm_optional_dependency(depdata):
137 deplist[dep] = depdata
138 for dep, depver in devdeps.items():
139 depdata = self.get_npm_data(dep, depver, d)
140 if self.check_npm_optional_dependency(depdata):
141 deplist[dep] = depdata
142 for dep, depver in deps.items():
143 depdata = self.get_npm_data(dep, depver, d)
144 deplist[dep] = depdata
145
146 extra_urls = []
147 for dep, depdata in deplist.items():
148 version = depdata.get('version', None)
149 if version:
150 url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
151 extra_urls.append(url)
152 if extra_urls:
153 scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
154 src_uri.extend(extra_urls)
155 return src_uri, None, -1, True
156 return origvalue, None, 0, True
157 updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
158 if updated:
159 del lines_before[:]
160 for line in newlines:
161 # Hack to avoid newlines that edit_metadata inserts
162 if line.endswith('\n'):
163 line = line[:-1]
164 lines_before.append(line)
165 return updated
166
167 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
168 import bb.utils
169 import oe.package
170 from collections import OrderedDict
171
172 if 'buildsystem' in handled:
173 return False
174
175 def read_package_json(fn):
176 with open(fn, 'r', errors='surrogateescape') as f:
177 return json.loads(f.read())
178
179 files = RecipeHandler.checkfiles(srctree, ['package.json'])
180 if files:
181 d = bb.data.createCopy(tinfoil.config_data)
182 npm_bindir = self._ensure_npm()
183 if not npm_bindir:
184 sys.exit(14)
185 d.prependVar('PATH', '%s:' % npm_bindir)
186
187 data = read_package_json(files[0])
188 if 'name' in data and 'version' in data:
189 extravalues['PN'] = data['name']
190 extravalues['PV'] = data['version']
191 classes.append('npm')
192 handled.append('buildsystem')
193 if 'description' in data:
194 extravalues['SUMMARY'] = data['description']
195 if 'homepage' in data:
196 extravalues['HOMEPAGE'] = data['homepage']
197
198 fetchdev = extravalues['fetchdev'] or None
199 deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
200 self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
201
202 # Shrinkwrap
203 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
204 self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
205
206 # Lockdown
207 self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
208
209 # Split each npm module out to is own package
210 npmpackages = oe.package.npm_split_package_dirs(srctree)
211 licvalues = None
212 for item in handled:
213 if isinstance(item, tuple):
214 if item[0] == 'license':
215 licvalues = item[1]
216 break
217 if not licvalues:
218 licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
219 if licvalues:
220 # Augment the license list with information we have in the packages
221 licenses = {}
222 license = self._handle_license(data)
223 if license:
224 licenses['${PN}'] = license
225 for pkgname, pkgitem in npmpackages.items():
226 _, pdata = pkgitem
227 license = self._handle_license(pdata)
228 if license:
229 licenses[pkgname] = license
230 # Now write out the package-specific license values
231 # We need to strip out the json data dicts for this since split_pkg_licenses
232 # isn't expecting it
233 packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
234 packages['${PN}'] = ''
235 pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
236 all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
237 if '&' in all_licenses:
238 all_licenses.remove('&')
239 extravalues['LICENSE'] = ' & '.join(all_licenses)
240
241 # Need to move S setting after inherit npm
242 for i, line in enumerate(lines_before):
243 if line.startswith('S ='):
244 lines_before.pop(i)
245 lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
246 lines_after.insert(1, line)
247 break
248
249 return True
250
251 return False
252
253 # FIXME this is duplicated from lib/bb/fetch2/npm.py
254 def _parse_view(self, output):
255 '''
256 Parse the output of npm view --json; the last JSON result
257 is assumed to be the one that we're interested in.
258 '''
259 pdata = None
260 outdeps = {}
261 datalines = []
262 bracelevel = 0
263 for line in output.splitlines():
264 if bracelevel:
265 datalines.append(line)
266 elif '{' in line:
267 datalines = []
268 datalines.append(line)
269 bracelevel = bracelevel + line.count('{') - line.count('}')
270 if datalines:
271 pdata = json.loads('\n'.join(datalines))
272 return pdata
273
274 # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
275 # (split out from _getdependencies())
276 def get_npm_data(self, pkg, version, d):
277 import bb.fetch2
278 pkgfullname = pkg
279 if version != '*' and not '/' in version:
280 pkgfullname += "@'%s'" % version
281 logger.debug(2, "Calling getdeps on %s" % pkg)
282 runenv = dict(os.environ, PATH=d.getVar('PATH'))
283 fetchcmd = "npm view %s --json" % pkgfullname
284 output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
285 data = self._parse_view(output)
286 return data
287
288 # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
289 # (split out from _getdependencies())
290 def get_npm_package_dependencies(self, pdata, fetchdev):
291 dependencies = pdata.get('dependencies', {})
292 optionalDependencies = pdata.get('optionalDependencies', {})
293 dependencies.update(optionalDependencies)
294 if fetchdev:
295 devDependencies = pdata.get('devDependencies', {})
296 dependencies.update(devDependencies)
297 else:
298 devDependencies = {}
299 depsfound = {}
300 optdepsfound = {}
301 devdepsfound = {}
302 for dep in dependencies:
303 if dep in optionalDependencies:
304 optdepsfound[dep] = dependencies[dep]
305 elif dep in devDependencies:
306 devdepsfound[dep] = dependencies[dep]
307 else:
308 depsfound[dep] = dependencies[dep]
309 return depsfound, optdepsfound, devdepsfound
310
311 # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
312 # (split out from _getdependencies())
313 def check_npm_optional_dependency(self, pdata):
314 pkg_os = pdata.get('os', None)
315 if pkg_os:
316 if not isinstance(pkg_os, list):
317 pkg_os = [pkg_os]
318 blacklist = False
319 for item in pkg_os:
320 if item.startswith('!'):
321 blacklist = True
322 break
323 if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
324 pkg = pdata.get('name', 'Unnamed package')
325 logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
326 return False
327 return True
328
329
330def register_recipe_handlers(handlers):
331 handlers.append((NpmRecipeHandler(), 60))