| lh | 9ed821d | 2023-04-07 01:36:19 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # SPDX-License-Identifier: GPL-2.0-only |
| 4 | # |
| 5 | |
| 6 | import os |
| 7 | import sys |
| 8 | import shutil |
| 9 | import errno |
| 10 | import time |
| 11 | |
| 12 | def mkdir(d): |
| 13 | try: |
| 14 | os.makedirs(d) |
| 15 | except OSError as e: |
| 16 | if e.errno != errno.EEXIST: |
| 17 | raise e |
| 18 | |
| 19 | # extract the hash from past the last colon to last underscore |
| 20 | def extract_sha(filename): |
| 21 | return filename.split(':')[7].split('_')[0] |
| 22 | |
| 23 | # get all files in a directory, extract hash and make |
| 24 | # a map from hash to list of file with that hash |
| 25 | def map_sha_to_files(dir_, prefix, sha_map): |
| 26 | sstate_prefix_path = dir_ + '/' + prefix + '/' |
| 27 | if not os.path.exists(sstate_prefix_path): |
| 28 | return |
| 29 | sstate_files = os.listdir(sstate_prefix_path) |
| 30 | for f in sstate_files: |
| 31 | try: |
| 32 | sha = extract_sha(f) |
| 33 | if sha not in sha_map: |
| 34 | sha_map[sha] = [] |
| 35 | sha_map[sha].append(sstate_prefix_path + f) |
| 36 | except IndexError: |
| 37 | continue |
| 38 | |
| 39 | # given a prefix build a map of hash to list of files |
| 40 | def build_sha_cache(prefix): |
| 41 | sha_map = {} |
| 42 | |
| 43 | sstate_dir = sys.argv[2] |
| 44 | map_sha_to_files(sstate_dir, prefix, sha_map) |
| 45 | |
| 46 | native_sstate_dir = sys.argv[2] + '/' + sys.argv[4] |
| 47 | map_sha_to_files(native_sstate_dir, prefix, sha_map) |
| 48 | |
| 49 | return sha_map |
| 50 | |
| 51 | if len(sys.argv) < 5: |
| 52 | print("Incorrect number of arguments specified") |
| 53 | print("syntax: gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> <nativelsbstring> [filterfile]") |
| 54 | sys.exit(1) |
| 55 | |
| 56 | filterlist = [] |
| 57 | if len(sys.argv) > 5: |
| 58 | print('Reading filter file %s' % sys.argv[5]) |
| 59 | with open(sys.argv[5]) as f: |
| 60 | for l in f.readlines(): |
| 61 | if ":" in l: |
| 62 | filterlist.append(l.rstrip()) |
| 63 | |
| 64 | print('Reading %s' % sys.argv[1]) |
| 65 | sigs = [] |
| 66 | with open(sys.argv[1]) as f: |
| 67 | for l in f.readlines(): |
| 68 | if ":" in l: |
| 69 | task, sig = l.split()[0].rsplit(':', 1) |
| 70 | if filterlist and not task in filterlist: |
| 71 | print('Filtering out %s' % task) |
| 72 | else: |
| 73 | sigs.append(sig) |
| 74 | |
| 75 | print('Gathering file list') |
| 76 | start_time = time.perf_counter() |
| 77 | files = set() |
| 78 | sstate_content_cache = {} |
| 79 | for s in sigs: |
| 80 | prefix = s[:2] |
| 81 | prefix2 = s[2:4] |
| 82 | if prefix not in sstate_content_cache: |
| 83 | sstate_content_cache[prefix] = {} |
| 84 | if prefix2 not in sstate_content_cache[prefix]: |
| 85 | sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2) |
| 86 | |
| 87 | if s in sstate_content_cache[prefix][prefix2]: |
| 88 | for f in sstate_content_cache[prefix][prefix2][s]: |
| 89 | files.add(f) |
| 90 | |
| 91 | elapsed = time.perf_counter() - start_time |
| 92 | print("Gathering file list took %.1fs" % elapsed) |
| 93 | |
| 94 | print('Processing files') |
| 95 | for f in files: |
| 96 | sys.stdout.write('Processing %s... ' % f) |
| 97 | _, ext = os.path.splitext(f) |
| 98 | if not ext in ['.tgz', '.siginfo', '.sig']: |
| 99 | # Most likely a temp file, skip it |
| 100 | print('skipping') |
| 101 | continue |
| 102 | dst = os.path.join(sys.argv[3], os.path.relpath(f, sys.argv[2])) |
| 103 | destdir = os.path.dirname(dst) |
| 104 | mkdir(destdir) |
| 105 | |
| 106 | src = os.path.realpath(f) |
| 107 | if os.path.exists(dst): |
| 108 | os.remove(dst) |
| 109 | if (os.stat(src).st_dev == os.stat(destdir).st_dev): |
| 110 | print('linking') |
| 111 | try: |
| 112 | os.link(src, dst) |
| 113 | except OSError as e: |
| 114 | print('hard linking failed, copying') |
| 115 | shutil.copyfile(src, dst) |
| 116 | else: |
| 117 | print('copying') |
| 118 | shutil.copyfile(src, dst) |
| 119 | |
| 120 | print('Done!') |