Gitiles
Code Review Sign In
192.168.1.100 / T108 / e958203796650e3959a43708d67534bf36f4c83b / . / mgit.sh
blob: 2ed5ae1c778698393d120566add2db8f92fc9f48 [file] [log] [blame]
b.liue9582032025-04-17 19:18:16 +0800[diff] [blame^]1#!/bin/bash
2
3#This is a small script that helps pulling, cloning and running arbitrary commands in multiple repositories.
4
5#Usage examples:
6##Clone all repos:
7##./mgit.sh clone
8
9#Update all repos:
10##./mgit.sh pull
11
12#Fetch (without rebasing or merging or anything):
13##./mgit.sh run "git fetch"
14
15#Make and install :
16##./mgit.sh run "make && make install"
17
18L=1
19BASE_URL="git@10.26.128.140"
20
21# For display color
22RESET="\033[0m"
23RED="\033[0;31m"
24BROWN="\033[0;33m"
25
26# The top repo dir path
27if [ -z ${OWRT_ROOT+x} ]; then
28 if [ -z ${TOP_DIR} ]; then
29 TOP_DIR=`pwd`
30 fi
31else
32 TOP_DIR=${OWRT_ROOT};
33fi
34
35#The list of repos other than the top repo (openwrt) which contains the script
36#Note: REPOPATHS is path + repo name, the path is relative to TOP_DIR
37# git server will only use the repo name without path info for simple.
38
39#repos in openwrt top folder
40REPOPATHS="dl"
41#Feeds repos
42REPOPATHS+=" external/management external/routing external/subpack"
43#Marvell repos
44REPOPATHS+=" marvell/linux marvell/lte-telephony marvell/obm marvell/services marvell/swd marvell/uboot marvell/fota marvell/webui"
45
46if [[ "${BASE_URL}" = "git@10.26.128.140" ]]; then
47 REPOPATHS+=" marvell/fastpath"
48fi
49
50###############################################################################
51# Actual code, don't touch:
52
53usage()
54{
55 echo "Usage : $0 <pull|clone|status>"
56 echo "Usage : $0 <checkout> <branch>"
57 echo "Usage : $0 <coman> <Jenkins_manifest_txt> : checkout a manifest points"
58 echo "Usage : $0 <mdiff> <Jenkins_manifest_txt> : create log list start from the manifest points"
59 echo "Usage : $0 <genopen> <Jenkins_manifest_txt> : pack open source code to opensource_<date>.tgz from manifest points"
60 echo "Usage : $0 <PICK> <dev_branch> <Phab_Diff_ID> <sdk_branch> <sdk_branch2> : pick a single patch from dev branch to target SDK branch"
61 echo "Usage : $0 <SYNC> <dev_branch> <Jenkins_build_ID> <sdk_branch> <sdk_branch2> : sync source dev branch to target SDK branch for specified build"
62 echo " $0 run <comand line in double quotation marks>"
63 echo " $0 erun <comand line in double quotation marks> : like run but breaks loop on failure"
64 cd - &>/dev/null
65 exit 1
66}
67
68clone_repos()
69{
70 echo "Using base URL: '${BASE_URL}'"
71
72 TOPREPO="openwrt"
73 if [ ! -d ".git" ]; then
74 if [ -d "$TOPREPO" ]; then
75 cd ${TOPREPO}
76 git pull
77 else
78 mkdir -p "${TOPREPO}"
79 git clone "${BASE_URL}:${TOPREPO}.git" "${TOPREPO}"
80 cd ${TOPREPO}
81 fi
82 else
83 #TOPNAME=$(basename $(git remote show -n origin | grep Fetch | cut -d: -f3-))
84 TOPNAME=$(basename $(git rev-parse --show-toplevel))
85 if [[ "${TOPNAME}" = "${TOPREPO}" ]]; then
86 git pull
87 else
88 if [ -d "$TOPREPO" ]; then
89 cd ${TOPREPO}
90 git pull
91 else
92 mkdir -p "${TOPREPO}"
93 git clone "${BASE_URL}:${TOPREPO}.git" "${TOPREPO}"
94 cd ${TOPREPO}
95 fi
96 fi
97 fi
98
99 for REPOPATH in $REPOPATHS ; do
100 REPO=$(basename "${REPOPATH}")
101 if [ ! -d "$REPOPATH" ]; then
102 mkdir -p "${REPOPATH}"
103 git clone "${BASE_URL}:${REPO}_ow.git" "${REPOPATH}"
104 else
105 echo "Updating existing repo: ${REPOPATH}"
106 pushd ${REPOPATH} &>/dev/null
107 git pull
108 popd &>/dev/null
109 fi
110 done
111}
112
113run_command_in_dirs()
114{
115 CMD=$1
116
117#the top repo
118 echo -e "Processing command in $RED"openwrt"$RESET"
119 eval $CMD
120 echo -e "\r\n"
121
122 for REPOPATH in $REPOPATHS ; do
123 if [ -d "$REPOPATH" ]; then
124 echo -e "Processing command in $RED${REPOPATH}$RESET"
125 pushd ${REPOPATH} &>/dev/null
126 eval $CMD
127 popd &>/dev/null
128 echo -e "\r\n"
129 else
130 echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n"
131 fi
132 done
133}
134
135checkout_bld_point()
136{
137 cp $1 manifest.txt
138 echo -e "Checking out $RED"openwrt"$RESET"
139 git checkout $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1)
140 echo -e "\r\n"
141
142 for REPOPATH in $REPOPATHS ; do
143 if [ -d "$REPOPATH" ]; then
144 echo -e "Checking out $RED${REPOPATH}$RESET"
145 pushd ${REPOPATH} &>/dev/null
146 L=$[L+1]
147 git checkout $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1)
148 popd &>/dev/null
149 echo -e "\r\n"
150 else
151 echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n"
152 fi
153 done
154 rm -rf manifest.txt
155
156}
157
158changes_since_manifest()
159{
160 cp $1 manifest.txt
161 echo -e "Checking out $RED"openwrt"$RESET"
162 branchonly=`git symbolic-ref --short HEAD`
163 local path tab='\t'
164
165 path="openwrt" &&
166 git diff --name-status $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1) | sed "s,$tab,$tab$path/," >${TOP_DIR}/filelist_"$branchonly"
167 if [ -z "$CUR_VER" ]; then
168 CUR_VER="$(git show HEAD:target/linux/mmp/image/version.asr)"
169 fi
170 LAST_VER="$(git show `head -1 "${TOP_DIR}/manifest.txt" | cut -d'-' -f1`:target/linux/mmp/image/version.asr)"
171 echo -e "\t\t\t\t$CUR_VER vs $LAST_VER release notes\n\t\t\t\tuse git log <Commit-id> in related repository to get more information for the change interested\nRedmine\tDifferential\tPriority\tCommit\tSummary\tDetails\tTest Plan\tDependency" >${TOP_DIR}/loglist_"$branchonly"
172 git log --pretty=tformat:'COMMIT %h%n%B' --abbrev=8 $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1).. | awk 'function p(){if(h){if(!t){++t;printf("\t\t\t\t'$path' repo\t\t\t\n")};gsub(/^[ \t\v]+|[ \t\v]+$/,"",S);gsub(/\t/," ",S);gsub(/^[ \t\v]+|[ \t\v]+$/,"",T);gsub(/\t/," ",T);printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",m,d,r,h,s,S,T,D)}};$1=="COMMIT"{p();h=$2;m=d=r=s=S=T=D="";k=0;next};h&&!s{s=$0};$1=="Differential"&&$2=="Revision:"{d=$3;sub("http.*/","",d);k=0;next};$1=="priority:"{r=$2;k=0;next};/[Rr]edmine.*[0-9]/{m=$0;sub(/^.*edmine:?[ \t]*/,"",m);k=0;next};/CVE-[0-9]+-[0-9]+/{v=$0;gsub(/CVE-[0-9]+-[0-9]+/,"\001&\004",v);n=split(v,a,"\001");for(i=1;i<=n;++i){j=index(a[i],"\004");if(j){v=a[i];sub(/\004.*/,"",v);if(!index(m,v)){if(m)m=m" "v;else m=v}}}};/[Ii]mpact:|[Bb]ranch:|URT:|[Ss]implicity:|Reviewers:/{k=0;next};/[Dd]etails:/{S=$0;sub(/^.*[Dd]etails:?[ \t]*/,"",S);k=1;next};/[Tt]est ?[Pp]lan:/{T=$0;sub(/^.*[Tt]est ?[Pp]lan:?[ \t]*/,"",T);k=2;next};/[Dd]ependenc[yi]e?s?:/{D=$0;sub(/^.*[Dd]ependenc[yi]e?s?:?[ \t]*/,"",D);k=0;next};k==1{if(S)S=S"\v"$0;else S=$0;next};k==2{if(T)T=T"\v"$0;else T=$0;next};END{p()}' >>${TOP_DIR}/loglist_"$branchonly"
173 echo -e "\r\n"
174
175 for REPOPATH in $REPOPATHS ; do
176 if [ -d "$REPOPATH" ]; then
177 if [ "$REPOPATH" = "marvell/fastpath" ]; then
178 continue
179 fi
180 echo -e "Checking out $RED${REPOPATH}$RESET"
181 pushd ${REPOPATH} &>/dev/null
182 L=$[L+1]
183 path="openwrt/$REPOPATH" &&
184 git diff --name-status $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1) | sed "s,$tab,$tab$path/," >>${TOP_DIR}/filelist_"$branchonly"
185 git log --pretty=tformat:'COMMIT %h%n%B' --abbrev=8 $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1).. | awk 'function p(){if(h){if(!t){++t;printf("\t\t\t\t'$path' repo\t\t\t\n")};gsub(/^[ \t\v]+|[ \t\v]+$/,"",S);gsub(/\t/," ",S);gsub(/^[ \t\v]+|[ \t\v]+$/,"",T);gsub(/\t/," ",T);printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",m,d,r,h,s,S,T,D)}};$1=="COMMIT"{p();h=$2;m=d=r=s=S=T=D="";k=0;next};h&&!s{s=$0};$1=="Differential"&&$2=="Revision:"{d=$3;sub("http.*/","",d);k=0;next};$1=="priority:"{r=$2;k=0;next};/[Rr]edmine.*[0-9]/{m=$0;sub(/^.*edmine:?[ \t]*/,"",m);k=0;next};/CVE-[0-9]+-[0-9]+/{v=$0;gsub(/CVE-[0-9]+-[0-9]+/,"\001&\004",v);n=split(v,a,"\001");for(i=1;i<=n;++i){j=index(a[i],"\004");if(j){v=a[i];sub(/\004.*/,"",v);if(!index(m,v)){if(m)m=m" "v;else m=v}}}};/[Ii]mpact:|[Bb]ranch:|URT:|[Ss]implicity:|Reviewers:/{k=0;next};/[Dd]etails:/{S=$0;sub(/^.*[Dd]etails:?[ \t]*/,"",S);k=1;next};/[Tt]est ?[Pp]lan:/{T=$0;sub(/^.*[Tt]est ?[Pp]lan:?[ \t]*/,"",T);k=2;next};/[Dd]ependenc[yi]e?s?:/{D=$0;sub(/^.*[Dd]ependenc[yi]e?s?:?[ \t]*/,"",D);k=0;next};k==1{if(S)S=S"\v"$0;else S=$0;next};k==2{if(T)T=T"\v"$0;else T=$0;next};END{p()}' >>${TOP_DIR}/loglist_"$branchonly"
186 popd &>/dev/null
187 echo -e "\r\n"
188 else
189 echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n"
190 fi
191 done
192 rm -rf manifest.txt
193
194 # cf. https://wiki.documentfoundation.org/Documentation/DevGuide/Spreadsheet_Documents#Filter_Options_for_the_CSV_Filter
195 lowriter --headless --convert-to docx:"Office Open XML Text" ${TOP_DIR}/filelist_"$branchonly"
196 localc --headless --infilter="Text - txt - csv (StarCalc):9,34,76,1,1/2/4/2" --convert-to xlsx:"Calc MS Excel 2007 XML" ${TOP_DIR}/loglist_"$branchonly"
197 # cf. https://openpyxl.readthedocs.io/en/stable/api/openpyxl.html
198 python3 -c 'import sys,openpyxl as o,openpyxl.worksheet.copier as owc; from copy import copy; wb=o.load_workbook(sys.argv[1]); ws=wb.active; d=dict(A=16,E=60,F=56,G=30,H=12); [setattr(ws.column_dimensions[c],"width",d[c]) for c in d.keys()]; ws.freeze_panes=ws["A4"]; f=o.styles.PatternFill(start_color="FFFF00",end_color="FFFF00",fill_type="solid"); a=o.styles.Alignment(wrap_text=True,vertical="center"); [([setattr(c,"alignment",a) for c in r],setattr(r[5],"value",r[5].value and type(r[5].value)==str and r[5].value.replace("_x000b_","\r\n") or r[5].value),setattr(r[6],"value",r[6].value and type(r[6].value)==str and r[6].value.replace("_x000b_","\r\n") or r[6].value)) for r in ws.rows]; [setattr(r[4],"fill",f) if not r[3].value else setattr(ws.row_dimensions[r[4].row],"height",None) for r in ws.rows]; f=o.styles.PatternFill(start_color="92D050",end_color="92D050",fill_type="solid"); [setattr(r[4],"fill",f) for _, r in zip(range(2),ws.rows)]; kr=[r[0].row for r in ws.rows if not r[3].value or r[3].value=="Commit" or (r[0].value and str(r[0].value).find("CVE-")>=0)]; skr=set(kr); nr=list(range(1,len(kr)+1)); dkr=dict(zip(kr,nr)); wt=wb.create_sheet("CVE"); cp=owc.WorksheetCopy(ws,wt); [[[setattr(t,"_value",s._value),setattr(t,"data_type",s.data_type),s.has_style and setattr(t,"_style",copy(s._style))] for t in [wt.cell(row=dkr[r],column=c)]] for (r,c),s in ws._cells.items() if r in skr]; cp._copy_dimensions(); [setattr(wt,f,copy(getattr(ws,f))) for f in "sheet_format sheet_properties page_margins page_setup print_options".split()]; wt.freeze_panes=wt["A4"]; wt["A3"]="CVE"; wb.save(sys.argv[1])' ${TOP_DIR}/loglist_"$branchonly".xlsx
199}
200
201check_repo_logs()
202{
203 START=$1
204 END=$2
205 TYPE=$3
206
207#the top repo
208 echo -e "Processing command in $RED"openwrt"$RESET"
209 echo -e "@repo:openwrt\r\n" > ${TOP_DIR}/change_"$START"_"$END".log
210 if [[ "${TYPE}" = "time" ]]; then
211 git log --pretty="format: * %s[%an][%h]" --since="$START" >> ${TOP_DIR}/change_"$START"_"$END".log
212 else
213 git log --pretty="format: * %s[%ad]" --date=short "$START".."$END" >> ${TOP_DIR}/change_"$START"_"$END".log
214 fi
215 echo -e "\r\n" >> ${TOP_DIR}/change_"$START"_"$END".log
216 echo -e "\r\n"
217
218 for REPOPATH in $REPOPATHS ; do
219 REPO=$(basename "${REPOPATH}")
220 if [ -d "$REPOPATH" ]; then
221 echo -e "Processing command in $RED${REPOPATH}$RESET"
222 pushd ${REPOPATH} &>/dev/null
223 echo -e "@repo:${REPO}\r\n" >> ${TOP_DIR}/change_"$START"_"$END".log
224 if [[ "${TYPE}" = "time" ]]; then
225 git log --pretty="format: * %s[%an][%h]" --since="$START" >> ${TOP_DIR}/change_"$START"_"$END".log
226 else
227 git log --pretty="format: * %s[%ad]" --date=short "$START".."$END" >> ${TOP_DIR}/change_"$START"_"$END".log
228 fi
229 echo -e "\r\n" >> ${TOP_DIR}/change_"$START"_"$END".log
230 popd &>/dev/null
231 echo -e "\r\n"
232 else
233 echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n"
234 fi
235 done
236}
237
238remove_private_code()
239{
240 PKGMKFL="$(grep -rl '\<MRVLDIR\>\|\<marvell/\|\<asr' package --include=Makefile | awk -F / '$n!=s;{n=NF-1;s=$n}')"
241 PKGPAT=
242 CFGPAT=
243 LIBPAT=
244 for c in $PKGMKFL; do
245 if [ ! -e $c ]; then
246 continue
247 fi
248 if [ $c == package/boot/uboot-mmp/Makefile ]; then
249 continue
250 fi
251 if [ $c == package/kernel/ugw_wlan/iwlwav-driver-uci/iwlwav-dev-uci/drivers/net/wireless/Makefile ]; then
252 continue
253 fi
254 if [ $c == package/network/utils/ipt_trigger/libipt_trigger/src/Makefile ]; then
255 continue
256 fi
257 PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $c)"
258 PKGDIR="$(dirname $c)"
259 if [ -z "$PKGNAMES" ]; then
260 while ! grep -q 'define \(Kernel\)\?Package' $PKGDIR/Makefile; do
261 d="$(dirname $PKGDIR)"
262 while [ -n "$d" -a ! -e $d/Makefile ]; do
263 d="$(dirname $d)"
264 done
265 if [ "$d" = "." ]; then
266 d=
267 fi
268 if [ -n "$d" ]; then
269 PKGDIR="$d"
270 else
271 break
272 fi
273 done
274 if [ -z "$d" ]; then
275 continue
276 fi
277 PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $PKGDIR/Makefile)"
278 if [ -z "$PKGNAMES" ]; then
279 PKGNAMES="$(basename $PKGDIR)"
280 fi
281 fi
282 for d in $PKGNAMES; do
283 if [ -z "$PKGPAT" ]; then
284 PKGPAT=$d
285 else
286 PKGPAT="$PKGPAT\|$d"
287 fi
288 done
289 KCFGPAT=
290 for d in $(sed 's/^ *config //;t n;d;:n;s/ //g' $PKGDIR/Makefile $PKGDIR/Config.in 2>/dev/null); do
291 if [ "$d" != "${d#KERNEL_}" ]; then
292 KCFGPAT="$KCFGPAT $d"
293 continue
294 fi
295 if [ -z "$CFGPAT" ]; then
296 CFGPAT=$d
297 else
298 CFGPAT="$CFGPAT\|$d"
299 fi
300 done
301 if [ -n "$KCFGPAT" ]; then
302 sed -n '/^define .*\/config/,/^endef/{/^define .*config\|^endef/d;p}' $PKGDIR/Makefile >> config/Config-kernel.in
303 [ ! -e $PKGDIR/Config.in ] || cat $PKGDIR/Config.in >> config/Config-kernel.in
304 fi
305 for d in $(sed 's/^.*\/lib\([a-zA-Z0-9_]\+\)\.so[ .].*$/\1/;t;d' $PKGDIR/Makefile | awk '!x[$0]++'); do
306 LIBPAT="$LIBPAT $d"
307 done
308 echo "Removing $PKGDIR"
309 rm -rf $PKGDIR
310 done
311 if [ -n "$LIBPAT" ]; then
312 LIBPAT="$(tr ' ' '\n' <<< "$LIBPAT" | awk '!x[$0]++' | tr '\n' '|' | sed 's/^|\||$//g;s/|/\\|/g')"
313 PKGMKFL="$(grep -rl "[ =]-l\($LIBPAT\)\>" package --include=Makefile | awk -F / '$n!=s;{n=NF-1;s=$n}')"
314 for c in $PKGMKFL; do
315 if [ ! -e $c ]; then
316 continue
317 fi
318 PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $c)"
319 PKGDIR="$(dirname $c)"
320 if [ -z "$PKGNAMES" ]; then
321 while ! grep -q 'define \(Kernel\)\?Package' $PKGDIR/Makefile; do
322 d="$(dirname $PKGDIR)"
323 while [ -n "$d" -a ! -e $d/Makefile ]; do
324 d="$(dirname $d)"
325 done
326 if [ "$d" = "." ]; then
327 d=
328 fi
329 if [ -n "$d" ]; then
330 PKGDIR="$d"
331 else
332 break
333 fi
334 done
335 if [ -z "$d" ]; then
336 continue
337 fi
338 PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $PKGDIR/Makefile)"
339 if [ -z "$PKGNAMES" ]; then
340 PKGNAMES="$(basename $PKGDIR)"
341 fi
342 fi
343 for d in $PKGNAMES; do
344 if [ -z "$PKGPAT" ]; then
345 PKGPAT=$d
346 else
347 PKGPAT="$PKGPAT\|$d"
348 fi
349 done
350 KCFGPAT=
351 for d in $(sed 's/^ *config //;t n;d;:n;s/ //g' $PKGDIR/Makefile $PKGDIR/Config.in 2>/dev/null); do
352 if [ "$d" != "${d#KERNEL_}" ]; then
353 KCFGPAT="$KCFGPAT $d"
354 continue
355 fi
356 if [ -z "$CFGPAT" ]; then
357 CFGPAT=$d
358 else
359 CFGPAT="$CFGPAT\|$d"
360 fi
361 done
362 if [ -n "$KCFGPAT" ]; then
363 sed -n '/^define .*\/config/,/^endef/{/^define .*config\|^endef/d;p}' $PKGDIR/Makefile >> config/Config-kernel.in
364 [ ! -e $PKGDIR/Config.in ] || cat $PKGDIR/Config.in >> config/Config-kernel.in
365 fi
366 for d in $(sed 's/^.*\/lib\([a-zA-Z0-9_]\+\)\.so[ .].*$/\1/;t;d' $PKGDIR/Makefile | awk '!x[$0]++'); do
367 LIBPAT="$LIBPAT $d"
368 done
369 echo "Removing $PKGDIR"
370 rm -rf $PKGDIR
371 done
372 fi
373 for c in $(grep -rl '\<marvell/\|\<RERRMSG\>' package --include='*.patch'); do
374 rm -fv $c
375 done
376 rm -rfv target/linux/mmp/*/*/base-files package/base-files/profile-files/*
377 git rm -rf docs/marvell mgit.sh ugit.sh scripts/env_tools
378 sed -i "/\<CONFIG_PACKAGE_\\(kmod-\\)\\?\\($PKGPAT\\)\>\|\<CONFIG_\\($CFGPAT\\)\>/d" config/defconfig*
379 NOBLD="BuildSwdownloader\|BuildFota\|BuildSecureSwd\|BuildSecFota\|BuildDiagMdb\|save_manifest"
380 sed -i "/^define \($NOBLD\)/,/^endef/d;/\$(call \($NOBLD\))/d" target/linux/mmp/image/Makefile
381 sed -i '/^if/{N;/^if.*\nendif/d}' target/linux/mmp/image/Makefile
382}
383
384pack_repo()
385{
386 BASEDIR="$(basename $TOP_DIR)"
387 TOPFILE="$(git ls-files | cut -d / -f 1 | uniq | sed "s/^/$BASEDIR\//")"
388 for c in $REPOPATHS ; do
389 if [ "$c" != "${c#marvell/}" -a "$c" != "marvell/linux" -a "$c" != "marvell/uboot" ]; then
390 continue
391 fi
392 if [ "$c" != "${c#marvell/}" ]; then
393 make -C $c mrproper
394 rm -rfv $c/ipkg-*
395 fi
396 TOPFILE="$TOPFILE $BASEDIR/$c"
397 done
398 (
399 set -x
400 tar -C $TOP_DIR/.. -czf $TOP_DIR/$1 --exclude=.git $TOPFILE
401 )
402 echo "Generated archive `du -sh $1`"
403}
404
405cd ${TOP_DIR}
406if [ $# -lt 1 ]; then
407 usage
408fi
409
410# path_cur=${PWD##*/}
411# if [[ $path_cur != openwrt ]]; then
412# usage
413# fi
414
415case "$1" in
416 clone)
417 clone_repos
418 ;;
419 pull)
420#git pull --rebase
421 run_command_in_dirs "git pull"
422 ;;
423 status)
424 run_command_in_dirs "git status"
425 ;;
426 checkout)
427 if [ -z "$2" ]; then
428 echo "'$1' accepts an additional 'command' parameter."
429 cd -
430 exit 1
431 fi
432 run_command_in_dirs "git checkout $2"
433 ;;
434 coman)
435 if [ -z "$2" ]; then
436 echo "'$1' accepts the manifest file for a specific build on Jenkins server"
437 cd -
438 exit 1
439 fi
440 checkout_bld_point "$2"
441 ;;
442 genopen)
443 if [ -z "$2" ]; then
444 echo "'$1' accepts the manifest file for a specific build on Jenkins server"
445 cd -
446 exit 1
447 fi
448 git stash
449 checkout_bld_point "$2"
450 run_command_in_dirs "git rm -f .arcconfig"
451 remove_private_code
452 pack_repo opensource_`date +%F`.tgz
453 git reset --hard
454 run_command_in_dirs "git checkout HEAD .arcconfig; git checkout -"
455 ;;
456 mdiff)
457 if [ -z "$2" ]; then
458 echo "'$1' input a manifest file for a specific build on Jenkins server"
459 cd -
460 exit 1
461 fi
462 git config --global diff.renameLimit 0
463 changes_since_manifest "$2"
464 ;;
465#changes in $2 days
466 dlog)
467 logdate=`date +'%Y-%m-%d'`
468 if [ -z "$2" ]; then
469 startdate=`date +'%Y-%m-%d' --date="-1 day"`
470 else
471 startdate=`date +'%Y-%m-%d' --date="-$2 day"`
472 fi
473 logtype="time"
474 check_repo_logs "$startdate" "$logdate" "$logtype"
475 ;;
476#changes between two tags/releases
477 rlog)
478 if [ -z "$2" ]; then
479 echo "'$1' accepts 1 or 2 additional 'command' parameters to specify the tag interval."
480 cd -
481 exit 1
482 fi
483 logtype="tags"
484 check_repo_logs "$2" "$3" "$logtype"
485 ;;
486 PICK|SYNC)
487 if [ -z "$2" -o -z "$3" -o -z "$4" ]; then
488 echo "'$1' accepts 3 or 4 additional parameters to specify the dev branch, diff / build ID, SDK branch(es)"
489 cd - &>/dev/null
490 exit 1
491 fi
492 JOBNAME=
493 if [ "$2" = "master" ]; then
494 JOBNAME=openwrtnightly
495 elif [ "$2" = "v2102" ]; then
496 JOBNAME=openwrt_v2102
497 else
498 echo "Invalid source dev branch $2"
499 cd - &>/dev/null
500 exit 1
501 fi
502 for BRCH in $4 $5; do
503 if [[ $BRCH != *$2 ]] ; then
504 echo "$BRCH incompatible with $2"
505 cd - &>/dev/null
506 exit 1
507 fi
508 done
509 set -e
510 #clone_repos
511 run_command_in_dirs "git stash && git checkout $2 && git pull"
512 git config --global push.default simple
513 if [ "$1" = "PICK" ]; then
514 ID="${3//D/}"
515 echo "Querying differential with ID $ID"
516 ARC=/phab/web/www/arcanist/bin/arc
517 if [ ! -e $ARC ]; then
518 ARC=`which arc`
519 fi
520 JSON="$($ARC call-conduit -- differential.query <<< "{\"ids\":[$ID]}")"
521 ERR="$(jq .error <<< "$JSON")"
522 if [ "$ERR" = "null" ]; then
523 if [ "$(jq '.response | length' <<< "$JSON")" -ge 1 ]; then
524 declare -A REPOS
525 REPOS[null]=N/A
526 REPOPATH_OW=openwrt
527 REPOPATH_ETL=external/management
528 REPOPATH_ERT=external/routing
529 REPOPATH_ESP=external/subpack
530 REPOPATH_MKL=marvell/linux
531 REPOPATH_OTL=marvell/lte-telephony
532 REPOPATH_BL=marvell/obm
533 REPOPATH_SVR=marvell/services
534 REPOPATH_DL=marvell/swd
535 REPOPATH_UO=marvell/uboot
536 REPOPATH_OTA=marvell/fota
537 REPOPATH_WEB=marvell/webui
538 REPOPATH_FP=$REPOPATH_OW # marvell/fastpath
539 IDLIST=
540 for ID in $(tr ',' '\n' <<< "$ID"); do
541 if [ -n "$(jq '.response | .[] | select(.id == "'$ID'") | length' <<< "$JSON")" ]; then
542 IDLIST="$IDLIST $ID"
543 fi
544 done
545 for ID in $IDLIST; do
546 PART="$(jq '.response | .[] | select(.id == "'$ID'")' <<< "$JSON")"
547 jq -r '.uri + "\t" + .title' <<< "$PART"
548 STAT="$(jq -r .statusName <<< "$PART")"
549 if [ "$STAT" != "Closed" ]; then
550 echo "Patch not closed: $STAT"
551 cd - &>/dev/null
552 exit 1
553 fi
554 REPO="$(jq -r .repositoryPHID <<< "$PART")"
555 RNAM="$(set +e; echo "${REPOS[$REPO]}" 2>/dev/null)"
556 if [ -z "$RNAM" ]; then
557 RJSN="$($ARC call-conduit -- diffusion.repository.search <<< "{\"constraints\":{\"phids\":[\"$REPO\"]}}")"
558 ERR="$(jq .error <<< "$RJSN")"
559 if [ "$ERR" = "null" ]; then
560 if [ "$(jq '.response.data | length' <<< "$RJSN")" -ge 1 ]; then
561 RNAM="$(jq -r '.response.data | .[] | .fields.callsign' <<< "$RJSN")"
562 REPOS[$REPO]=$RNAM
563 else
564 echo "Empty response from server for $REPO"
565 cd - &>/dev/null
566 exit 1
567 fi
568 else
569 jq -r .errorMessage <<< "$RJSN"
570 cd - &>/dev/null
571 exit 1
572 fi
573 elif [ "$RNAM" = "N/A" ]; then
574 echo "Cannot determine repository for D$ID. Manual operation needed."
575 cd - &>/dev/null
576 exit 2
577 fi
578 REPOPATH="$(eval "echo \${REPOPATH_$RNAM}")"
579 if [ -z "$REPOPATH" ]; then
580 echo "Empty repository for D$ID. Manual operation needed."
581 cd - &>/dev/null
582 exit 2
583 fi
584 echo " for repository $REPOPATH"
585 if [ $REPOPATH = openwrt ]; then
586 REPOPATH=.
587 fi
588 CMIT="$(jq -c '.commits' <<< "$PART")"
589 CJSN="$($ARC call-conduit -- diffusion.commit.search <<< "{\"constraints\":{\"phids\":$CMIT}}")"
590 ERR="$(jq .error <<< "$CJSN")"
591 for i in `seq 15`; do
592 if [ "$ERR" = "null" ]; then
593 break
594 fi
595 jq -r .errorMessage <<< "$CJSN"
596 echo "Waiting 2 minutes to retry..."
597 sleep 120
598 JSON="$($ARC call-conduit -- differential.query <<< "{\"ids\":[$ID]}")"
599 ERR="$(jq .error <<< "$JSON")"
600 if [ "$ERR" = "null" ]; then
601 PART="$(jq '.response | .[] | select(.id == "'$ID'")' <<< "$JSON")"
602 CMIT="$(jq -c '.commits' <<< "$PART")"
603 CJSN="$($ARC call-conduit -- diffusion.commit.search <<< "{\"constraints\":{\"phids\":$CMIT}}")"
604 ERR="$(jq .error <<< "$CJSN")"
605 fi
606 done
607 if [ "$ERR" = "null" ]; then
608 if [ "$(jq '.response.data | length' <<< "$CJSN")" -ge 1 ]; then
609 FOUND=
610 TARGET=
611 TARGET2=
612 for CMID in $(jq -r '.response.data | .[] | .fields.identifier' <<< "$CJSN"); do
613 BRCH="$(cd $REPOPATH; git branch -r --contains $CMID | sed '/->/d;s/^.*\///')"
614 echo " commit $CMID on $(tr '\n' ' ' <<< "$BRCH")"
615 if grep -Fqx "$2" <<< "$BRCH"; then
616 FOUND="$CMID"
617 fi
618 if grep -Fqx "$4" <<< "$BRCH"; then
619 TARGET="$CMID"
620 fi
621 if [ .$5 != . ] && grep -Fqx "$5" <<< "$BRCH"; then
622 TARGET2="$CMID"
623 fi
624 done
625 if [ -z "$FOUND" ]; then
626 echo "No commit found on $2"
627 cd - &>/dev/null
628 exit 1
629 fi
630 if [ -z "$TARGET" ]; then
631 echo "Picking commit $FOUND onto $4"
632 (
633 cd $REPOPATH;
634 set -x;
635 git checkout $4;
636 git reset --hard origin/$4;
637 git cherry-pick $FOUND;
638 #git push
639 )
640 TARGET="$(cd $REPOPATH; git show -s --pretty=tformat:%H)"
641 if [ .$5 = .$4 -a -z "$TARGET2" ]; then
642 TARGET2="$TARGET"
643 fi
644 else
645 echo "Already on $4"
646 fi
647 if [ .$5 == . ]; then
648 :
649 elif [ -z "$TARGET2" ]; then
650 echo "Picking commit $FOUND onto $5"
651 (
652 cd $REPOPATH;
653 set -x;
654 git checkout $5;
655 git reset --hard origin/$5;
656 git cherry-pick $FOUND;
657 #git push
658 )
659 TARGET2="$(cd $REPOPATH; git show -s --pretty=tformat:%H)"
660 else
661 echo "Already on $5"
662 fi
663 # find reverting changes
664 RLST="$(cd $REPOPATH; git log --pretty=tformat:'COMMIT: %H%n%B' $FOUND..origin/$2 | awk '/^COMMIT: /{h=$2;next};h&&/([Rr]evert[sed]*|[Bb]ack[ sed]out|[Uu]ndo[es]) .*(commits?|change[set]*|rev[ison]*|diffs?) +'$FOUND'/{print h;h=""}')"
665 if [ -n "$RLST" ]; then
666 echo "Found $(wc -l <<< "$RLST") reverting change(s) on $2"
667 for CMID in $RLST; do
668 (cd $REPOPATH; git show -s --pretty=tformat:' commit %H : %B' $CMID | sed '/^ *$/d')
669 echo
670 done
671 TLST="$(cd $REPOPATH; git log --pretty=tformat:'COMMIT: %H%n%B' $TARGET..origin/$4 | awk '/^COMMIT: /{h=$2;next};h&&/([Rr]evert[sed]*|[Bb]ack[ sed]out|[Uu]ndo[es]) .*(commits?|change[set]*|rev[ison]*|diffs?) +('$FOUND'|'$TARGET')/{print h;h=""}')"
672 if [ -n "$TLST" ]; then
673 echo "Found $(wc -l <<< "$TLST") reverting change(s) on $4"
674 for CMID in $TLST; do
675 (cd $REPOPATH; git show -s --pretty=tformat:' commit %H : %s' $CMID | sed '/^ *$/d')
676 done
677 else
678 for CMID in $RLST; do
679 echo "Picking commit $CMID onto $4"
680 (
681 cd $REPOPATH;
682 set -x;
683 git checkout $4;
684 git cherry-pick $CMID;
685 #git push
686 )
687 done
688 fi
689 if [ .$5 != . -a .$5 != .$4 ]; then
690 TLST="$(cd $REPOPATH; git log --pretty=tformat:'COMMIT: %H%n%B' $TARGET2..origin/$5 | awk '/^COMMIT: /{h=$2;next};h&&/([Rr]evert[sed]*|[Bb]ack[ sed]out|[Uu]ndo[es]) .*(commits?|change[set]*|rev[ison]*|diffs?) +('$FOUND'|'$TARGET2')/{print h;h=""}')"
691 if [ -n "$TLST" ]; then
692 echo "Found $(wc -l <<< "$TLST") reverting change(s) on $5"
693 for CMID in $TLST; do
694 (cd $REPOPATH; git show -s --pretty=tformat:' commit %H : %s' $CMID | sed '/^ *$/d')
695 done
696 else
697 for CMID in $RLST; do
698 echo "Picking commit $CMID onto $5"
699 (
700 cd $REPOPATH;
701 set -x;
702 git checkout $5;
703 git cherry-pick $CMID;
704 #git push
705 )
706 done
707 fi
708 fi
709 fi
710 else
711 echo "Empty response from server for $CMIT"
712 cd - &>/dev/null
713 exit 1
714 fi
715 echo
716 else
717 jq -r .errorMessage <<< "$CJSN"
718 cd - &>/dev/null
719 exit 1
720 fi
721 done
722 else
723 echo "Empty response from server for $3"
724 cd - &>/dev/null
725 exit 1
726 fi
727 else
728 jq -r .errorMessage <<< "$JSON"
729 cd - &>/dev/null
730 exit 1
731 fi
732 else
733 if [ ! -e "$TOP_DIR/.arcconfig" -a -e "$TOP_DIR/openwrt/.arcconfig" ]; then
734 TOP_DIR=$TOP_DIR/openwrt
735 fi
736 HOST="$(jq -r '.conduit_uri' $TOP_DIR/.arcconfig)"
737 if [ "$HOST" = "null" ]; then
738 HOST="$(jq -r '.["phabricator.uri"]' $TOP_DIR/.arcconfig)"
739 fi
740 JOBURL="${HOST%/*}:8080/job/$JOBNAME"
741 echo "Querying artifacts for $JOBNAME build $3"
742 JSON="$(wget $JOBURL/$3/api/json -o /dev/null -O -)"
743 echo "Parsing json..."
744 RMAN="$(jq -r '.artifacts | map(select(.fileName == "manifest.txt")) | .[0] | .relativePath' <<< "$JSON")"
745 echo "Fetching the first of $(jq -r '.artifacts | map(select(.fileName == "manifest.txt")) | length' <<< "$JSON") profiles: $RMAN"
746 MANI="$(wget $JOBURL/$3/artifact/$RMAN -o /dev/null -O -)"
747 echo "Parsing manifest..."
748 MCNT="$(wc -l <<< "$MANI")"
749 RCNT="$(awk "{x+=NF};END{print 0+x}" <<< "$REPOPATHS")"
750 if [ "$MCNT" -lt "$RCNT" ]; then
751 echo "Too few lines in manifest: $RCNT expected, but got $MCNT"
752 cd - &>/dev/null
753 exit 1
754 fi
755 VER="$(cat target/linux/mmp/image/version.asr)"
756 VERCMID=
757 if [ $VER -lt $3 -a "$6" = "true" ]; then
758 echo "Updating version.asr from $VER to $3"
759 echo $3 > target/linux/mmp/image/version.asr
760 git add target/linux/mmp/image/version.asr
761 git commit -m "target:update image version to $3"
762 git push
763 VERCMID="$(git show -s --pretty=tformat:%H)"
764 fi
765 for BRCH in $4 $5; do
766 L=1
767 echo -e "Processing autopick to $BRCH in $RED"openwrt"$RESET"
768 CMID="$(sed -e $L'q;d' <<< "$MANI" | cut -d'-' -f1)"
769 (
770 set -x;
771 git checkout $BRCH;
772 git reset --hard origin/$BRCH;
773 $TOP_DIR/scripts/autopick.sh $CMID;
774 )
775 echo -e "\r\n"
776 for REPOPATH in $REPOPATHS ; do
777 if [ -d "$REPOPATH" ]; then
778 echo -e "Processing autopick to $BRCH in $RED${REPOPATH}$RESET"
779 L=$((L+1))
780 CMID="$(sed -e $L'q;d' <<< "$MANI" | cut -d'-' -f1)"
781 (
782 set -x;
783 cd $REPOPATH;
784 git checkout $BRCH;
785 git reset --hard origin/$BRCH;
786 $TOP_DIR/scripts/autopick.sh $CMID;
787 )
788 echo -e "\r\n"
789 else
790 echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n"
791 fi
792 done
793 if [ -n "$VERCMID" ]; then
794 (
795 set -x;
796 git cherry-pick $VERCMID
797 )
798 else
799 VER="$(cat target/linux/mmp/image/version.asr)"
800 if [ $VER -lt $3 -a "$6" = "true" ]; then
801 echo "Updating version.asr on $BRCH from $VER to $3"
802 echo $3 > target/linux/mmp/image/version.asr
803 git add target/linux/mmp/image/version.asr
804 git commit -m "target:update image version to $3"
805 fi
806 fi
807 L=1
808 #run_command_in_dirs "git push"
809 done
810 fi
811 ;;
812 run)
813 if [ -z "$2" ]; then
814 echo "'$1' accepts an additional 'command' parameter."
815 cd -
816 exit 1
817 fi
818 run_command_in_dirs "$2"
819 ;;
820 erun)
821 if [ -z "$2" ]; then
822 echo "'$1' accepts an additional 'command' parameter."
823 cd -
824 exit 1
825 fi
826 set -e
827 run_command_in_dirs "$2"
828 ;;
829 *)
830 usage
831 ;;
832esac
833cd - &>/dev/null