| #!/bin/bash |
| |
| #This is a small script that helps pulling, cloning and running arbitrary commands in multiple repositories. |
| |
| #Usage examples: |
| ##Clone all repos: |
| ##./mgit.sh clone |
| |
| #Update all repos: |
| ##./mgit.sh pull |
| |
| #Fetch (without rebasing or merging or anything): |
| ##./mgit.sh run "git fetch" |
| |
| #Make and install : |
| ##./mgit.sh run "make && make install" |
| |
| L=1 |
| BASE_URL="git@10.26.128.140" |
| |
| # For display color |
| RESET="\033[0m" |
| RED="\033[0;31m" |
| BROWN="\033[0;33m" |
| |
| # The top repo dir path |
| if [ -z ${OWRT_ROOT+x} ]; then |
| if [ -z ${TOP_DIR} ]; then |
| TOP_DIR=`pwd` |
| fi |
| else |
| TOP_DIR=${OWRT_ROOT}; |
| fi |
| |
| #The list of repos other than the top repo (openwrt) which contains the script |
| #Note: REPOPATHS is path + repo name, the path is relative to TOP_DIR |
| # git server will only use the repo name without path info for simple. |
| |
| #repos in openwrt top folder |
| REPOPATHS="dl" |
| #Feeds repos |
| REPOPATHS+=" external/management external/routing external/subpack" |
| #Marvell repos |
| REPOPATHS+=" marvell/linux marvell/lte-telephony marvell/obm marvell/services marvell/swd marvell/uboot marvell/fota marvell/webui" |
| |
| if [[ "${BASE_URL}" = "git@10.26.128.140" ]]; then |
| REPOPATHS+=" marvell/fastpath" |
| fi |
| |
| ############################################################################### |
| # Actual code, don't touch: |
| |
| usage() |
| { |
| echo "Usage : $0 <pull|clone|status>" |
| echo "Usage : $0 <checkout> <branch>" |
| echo "Usage : $0 <coman> <Jenkins_manifest_txt> : checkout a manifest points" |
| echo "Usage : $0 <mdiff> <Jenkins_manifest_txt> : create log list start from the manifest points" |
| echo "Usage : $0 <genopen> <Jenkins_manifest_txt> : pack open source code to opensource_<date>.tgz from manifest points" |
| echo "Usage : $0 <PICK> <dev_branch> <Phab_Diff_ID> <sdk_branch> <sdk_branch2> : pick a single patch from dev branch to target SDK branch" |
| echo "Usage : $0 <SYNC> <dev_branch> <Jenkins_build_ID> <sdk_branch> <sdk_branch2> : sync source dev branch to target SDK branch for specified build" |
| echo " $0 run <comand line in double quotation marks>" |
| echo " $0 erun <comand line in double quotation marks> : like run but breaks loop on failure" |
| cd - &>/dev/null |
| exit 1 |
| } |
| |
| clone_repos() |
| { |
| echo "Using base URL: '${BASE_URL}'" |
| |
| TOPREPO="openwrt" |
| if [ ! -d ".git" ]; then |
| if [ -d "$TOPREPO" ]; then |
| cd ${TOPREPO} |
| git pull |
| else |
| mkdir -p "${TOPREPO}" |
| git clone "${BASE_URL}:${TOPREPO}.git" "${TOPREPO}" |
| cd ${TOPREPO} |
| fi |
| else |
| #TOPNAME=$(basename $(git remote show -n origin | grep Fetch | cut -d: -f3-)) |
| TOPNAME=$(basename $(git rev-parse --show-toplevel)) |
| if [[ "${TOPNAME}" = "${TOPREPO}" ]]; then |
| git pull |
| else |
| if [ -d "$TOPREPO" ]; then |
| cd ${TOPREPO} |
| git pull |
| else |
| mkdir -p "${TOPREPO}" |
| git clone "${BASE_URL}:${TOPREPO}.git" "${TOPREPO}" |
| cd ${TOPREPO} |
| fi |
| fi |
| fi |
| |
| for REPOPATH in $REPOPATHS ; do |
| REPO=$(basename "${REPOPATH}") |
| if [ ! -d "$REPOPATH" ]; then |
| mkdir -p "${REPOPATH}" |
| git clone "${BASE_URL}:${REPO}_ow.git" "${REPOPATH}" |
| else |
| echo "Updating existing repo: ${REPOPATH}" |
| pushd ${REPOPATH} &>/dev/null |
| git pull |
| popd &>/dev/null |
| fi |
| done |
| } |
| |
| run_command_in_dirs() |
| { |
| CMD=$1 |
| |
| #the top repo |
| echo -e "Processing command in $RED"openwrt"$RESET" |
| eval $CMD |
| echo -e "\r\n" |
| |
| for REPOPATH in $REPOPATHS ; do |
| if [ -d "$REPOPATH" ]; then |
| echo -e "Processing command in $RED${REPOPATH}$RESET" |
| pushd ${REPOPATH} &>/dev/null |
| eval $CMD |
| popd &>/dev/null |
| echo -e "\r\n" |
| else |
| echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n" |
| fi |
| done |
| } |
| |
| checkout_bld_point() |
| { |
| cp $1 manifest.txt |
| echo -e "Checking out $RED"openwrt"$RESET" |
| git checkout $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1) |
| echo -e "\r\n" |
| |
| for REPOPATH in $REPOPATHS ; do |
| if [ -d "$REPOPATH" ]; then |
| echo -e "Checking out $RED${REPOPATH}$RESET" |
| pushd ${REPOPATH} &>/dev/null |
| L=$[L+1] |
| git checkout $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1) |
| popd &>/dev/null |
| echo -e "\r\n" |
| else |
| echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n" |
| fi |
| done |
| rm -rf manifest.txt |
| |
| } |
| |
| changes_since_manifest() |
| { |
| cp $1 manifest.txt |
| echo -e "Checking out $RED"openwrt"$RESET" |
| branchonly=`git symbolic-ref --short HEAD` |
| local path tab='\t' |
| |
| path="openwrt" && |
| git diff --name-status $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1) | sed "s,$tab,$tab$path/," >${TOP_DIR}/filelist_"$branchonly" |
| if [ -z "$CUR_VER" ]; then |
| CUR_VER="$(git show HEAD:target/linux/mmp/image/version.asr)" |
| fi |
| LAST_VER="$(git show `head -1 "${TOP_DIR}/manifest.txt" | cut -d'-' -f1`:target/linux/mmp/image/version.asr)" |
| echo -e "\t\t\t\t$CUR_VER vs $LAST_VER release notes\n\t\t\t\tuse git log <Commit-id> in related repository to get more information for the change interested\nRedmine\tDifferential\tPriority\tCommit\tSummary\tDetails\tTest Plan\tDependency" >${TOP_DIR}/loglist_"$branchonly" |
| git log --pretty=tformat:'COMMIT %h%n%B' --abbrev=8 $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1).. | awk 'function p(){if(h){if(!t){++t;printf("\t\t\t\t'$path' repo\t\t\t\n")};gsub(/^[ \t\v]+|[ \t\v]+$/,"",S);gsub(/\t/," ",S);gsub(/^[ \t\v]+|[ \t\v]+$/,"",T);gsub(/\t/," ",T);printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",m,d,r,h,s,S,T,D)}};$1=="COMMIT"{p();h=$2;m=d=r=s=S=T=D="";k=0;next};h&&!s{s=$0};$1=="Differential"&&$2=="Revision:"{d=$3;sub("http.*/","",d);k=0;next};$1=="priority:"{r=$2;k=0;next};/[Rr]edmine.*[0-9]/{m=$0;sub(/^.*edmine:?[ \t]*/,"",m);k=0;next};/CVE-[0-9]+-[0-9]+/{v=$0;gsub(/CVE-[0-9]+-[0-9]+/,"\001&\004",v);n=split(v,a,"\001");for(i=1;i<=n;++i){j=index(a[i],"\004");if(j){v=a[i];sub(/\004.*/,"",v);if(!index(m,v)){if(m)m=m" "v;else m=v}}}};/[Ii]mpact:|[Bb]ranch:|URT:|[Ss]implicity:|Reviewers:/{k=0;next};/[Dd]etails:/{S=$0;sub(/^.*[Dd]etails:?[ \t]*/,"",S);k=1;next};/[Tt]est ?[Pp]lan:/{T=$0;sub(/^.*[Tt]est ?[Pp]lan:?[ \t]*/,"",T);k=2;next};/[Dd]ependenc[yi]e?s?:/{D=$0;sub(/^.*[Dd]ependenc[yi]e?s?:?[ \t]*/,"",D);k=0;next};k==1{if(S)S=S"\v"$0;else S=$0;next};k==2{if(T)T=T"\v"$0;else T=$0;next};END{p()}' >>${TOP_DIR}/loglist_"$branchonly" |
| echo -e "\r\n" |
| |
| for REPOPATH in $REPOPATHS ; do |
| if [ -d "$REPOPATH" ]; then |
| if [ "$REPOPATH" = "marvell/fastpath" ]; then |
| continue |
| fi |
| echo -e "Checking out $RED${REPOPATH}$RESET" |
| pushd ${REPOPATH} &>/dev/null |
| L=$[L+1] |
| path="openwrt/$REPOPATH" && |
| git diff --name-status $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1) | sed "s,$tab,$tab$path/," >>${TOP_DIR}/filelist_"$branchonly" |
| git log --pretty=tformat:'COMMIT %h%n%B' --abbrev=8 $(sed -e $L'q;d' "${TOP_DIR}/manifest.txt" | cut -d'-' -f1).. | awk 'function p(){if(h){if(!t){++t;printf("\t\t\t\t'$path' repo\t\t\t\n")};gsub(/^[ \t\v]+|[ \t\v]+$/,"",S);gsub(/\t/," ",S);gsub(/^[ \t\v]+|[ \t\v]+$/,"",T);gsub(/\t/," ",T);printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",m,d,r,h,s,S,T,D)}};$1=="COMMIT"{p();h=$2;m=d=r=s=S=T=D="";k=0;next};h&&!s{s=$0};$1=="Differential"&&$2=="Revision:"{d=$3;sub("http.*/","",d);k=0;next};$1=="priority:"{r=$2;k=0;next};/[Rr]edmine.*[0-9]/{m=$0;sub(/^.*edmine:?[ \t]*/,"",m);k=0;next};/CVE-[0-9]+-[0-9]+/{v=$0;gsub(/CVE-[0-9]+-[0-9]+/,"\001&\004",v);n=split(v,a,"\001");for(i=1;i<=n;++i){j=index(a[i],"\004");if(j){v=a[i];sub(/\004.*/,"",v);if(!index(m,v)){if(m)m=m" "v;else m=v}}}};/[Ii]mpact:|[Bb]ranch:|URT:|[Ss]implicity:|Reviewers:/{k=0;next};/[Dd]etails:/{S=$0;sub(/^.*[Dd]etails:?[ \t]*/,"",S);k=1;next};/[Tt]est ?[Pp]lan:/{T=$0;sub(/^.*[Tt]est ?[Pp]lan:?[ \t]*/,"",T);k=2;next};/[Dd]ependenc[yi]e?s?:/{D=$0;sub(/^.*[Dd]ependenc[yi]e?s?:?[ \t]*/,"",D);k=0;next};k==1{if(S)S=S"\v"$0;else S=$0;next};k==2{if(T)T=T"\v"$0;else T=$0;next};END{p()}' >>${TOP_DIR}/loglist_"$branchonly" |
| popd &>/dev/null |
| echo -e "\r\n" |
| else |
| echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n" |
| fi |
| done |
| rm -rf manifest.txt |
| |
| # cf. https://wiki.documentfoundation.org/Documentation/DevGuide/Spreadsheet_Documents#Filter_Options_for_the_CSV_Filter |
| lowriter --headless --convert-to docx:"Office Open XML Text" ${TOP_DIR}/filelist_"$branchonly" |
| localc --headless --infilter="Text - txt - csv (StarCalc):9,34,76,1,1/2/4/2" --convert-to xlsx:"Calc MS Excel 2007 XML" ${TOP_DIR}/loglist_"$branchonly" |
| # cf. https://openpyxl.readthedocs.io/en/stable/api/openpyxl.html |
| python3 -c 'import sys,openpyxl as o,openpyxl.worksheet.copier as owc; from copy import copy; wb=o.load_workbook(sys.argv[1]); ws=wb.active; d=dict(A=16,E=60,F=56,G=30,H=12); [setattr(ws.column_dimensions[c],"width",d[c]) for c in d.keys()]; ws.freeze_panes=ws["A4"]; f=o.styles.PatternFill(start_color="FFFF00",end_color="FFFF00",fill_type="solid"); a=o.styles.Alignment(wrap_text=True,vertical="center"); [([setattr(c,"alignment",a) for c in r],setattr(r[5],"value",r[5].value and type(r[5].value)==str and r[5].value.replace("_x000b_","\r\n") or r[5].value),setattr(r[6],"value",r[6].value and type(r[6].value)==str and r[6].value.replace("_x000b_","\r\n") or r[6].value)) for r in ws.rows]; [setattr(r[4],"fill",f) if not r[3].value else setattr(ws.row_dimensions[r[4].row],"height",None) for r in ws.rows]; f=o.styles.PatternFill(start_color="92D050",end_color="92D050",fill_type="solid"); [setattr(r[4],"fill",f) for _, r in zip(range(2),ws.rows)]; kr=[r[0].row for r in ws.rows if not r[3].value or r[3].value=="Commit" or (r[0].value and str(r[0].value).find("CVE-")>=0)]; skr=set(kr); nr=list(range(1,len(kr)+1)); dkr=dict(zip(kr,nr)); wt=wb.create_sheet("CVE"); cp=owc.WorksheetCopy(ws,wt); [[[setattr(t,"_value",s._value),setattr(t,"data_type",s.data_type),s.has_style and setattr(t,"_style",copy(s._style))] for t in [wt.cell(row=dkr[r],column=c)]] for (r,c),s in ws._cells.items() if r in skr]; cp._copy_dimensions(); [setattr(wt,f,copy(getattr(ws,f))) for f in "sheet_format sheet_properties page_margins page_setup print_options".split()]; wt.freeze_panes=wt["A4"]; wt["A3"]="CVE"; wb.save(sys.argv[1])' ${TOP_DIR}/loglist_"$branchonly".xlsx |
| } |
| |
| check_repo_logs() |
| { |
| START=$1 |
| END=$2 |
| TYPE=$3 |
| |
| #the top repo |
| echo -e "Processing command in $RED"openwrt"$RESET" |
| echo -e "@repo:openwrt\r\n" > ${TOP_DIR}/change_"$START"_"$END".log |
| if [[ "${TYPE}" = "time" ]]; then |
| git log --pretty="format: * %s[%an][%h]" --since="$START" >> ${TOP_DIR}/change_"$START"_"$END".log |
| else |
| git log --pretty="format: * %s[%ad]" --date=short "$START".."$END" >> ${TOP_DIR}/change_"$START"_"$END".log |
| fi |
| echo -e "\r\n" >> ${TOP_DIR}/change_"$START"_"$END".log |
| echo -e "\r\n" |
| |
| for REPOPATH in $REPOPATHS ; do |
| REPO=$(basename "${REPOPATH}") |
| if [ -d "$REPOPATH" ]; then |
| echo -e "Processing command in $RED${REPOPATH}$RESET" |
| pushd ${REPOPATH} &>/dev/null |
| echo -e "@repo:${REPO}\r\n" >> ${TOP_DIR}/change_"$START"_"$END".log |
| if [[ "${TYPE}" = "time" ]]; then |
| git log --pretty="format: * %s[%an][%h]" --since="$START" >> ${TOP_DIR}/change_"$START"_"$END".log |
| else |
| git log --pretty="format: * %s[%ad]" --date=short "$START".."$END" >> ${TOP_DIR}/change_"$START"_"$END".log |
| fi |
| echo -e "\r\n" >> ${TOP_DIR}/change_"$START"_"$END".log |
| popd &>/dev/null |
| echo -e "\r\n" |
| else |
| echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n" |
| fi |
| done |
| } |
| |
| remove_private_code() |
| { |
| PKGMKFL="$(grep -rl '\<MRVLDIR\>\|\<marvell/\|\<asr' package --include=Makefile | awk -F / '$n!=s;{n=NF-1;s=$n}')" |
| PKGPAT= |
| CFGPAT= |
| LIBPAT= |
| for c in $PKGMKFL; do |
| if [ ! -e $c ]; then |
| continue |
| fi |
| if [ $c == package/boot/uboot-mmp/Makefile ]; then |
| continue |
| fi |
| if [ $c == package/kernel/ugw_wlan/iwlwav-driver-uci/iwlwav-dev-uci/drivers/net/wireless/Makefile ]; then |
| continue |
| fi |
| if [ $c == package/network/utils/ipt_trigger/libipt_trigger/src/Makefile ]; then |
| continue |
| fi |
| PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $c)" |
| PKGDIR="$(dirname $c)" |
| if [ -z "$PKGNAMES" ]; then |
| while ! grep -q 'define \(Kernel\)\?Package' $PKGDIR/Makefile; do |
| d="$(dirname $PKGDIR)" |
| while [ -n "$d" -a ! -e $d/Makefile ]; do |
| d="$(dirname $d)" |
| done |
| if [ "$d" = "." ]; then |
| d= |
| fi |
| if [ -n "$d" ]; then |
| PKGDIR="$d" |
| else |
| break |
| fi |
| done |
| if [ -z "$d" ]; then |
| continue |
| fi |
| PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $PKGDIR/Makefile)" |
| if [ -z "$PKGNAMES" ]; then |
| PKGNAMES="$(basename $PKGDIR)" |
| fi |
| fi |
| for d in $PKGNAMES; do |
| if [ -z "$PKGPAT" ]; then |
| PKGPAT=$d |
| else |
| PKGPAT="$PKGPAT\|$d" |
| fi |
| done |
| KCFGPAT= |
| for d in $(sed 's/^ *config //;t n;d;:n;s/ //g' $PKGDIR/Makefile $PKGDIR/Config.in 2>/dev/null); do |
| if [ "$d" != "${d#KERNEL_}" ]; then |
| KCFGPAT="$KCFGPAT $d" |
| continue |
| fi |
| if [ -z "$CFGPAT" ]; then |
| CFGPAT=$d |
| else |
| CFGPAT="$CFGPAT\|$d" |
| fi |
| done |
| if [ -n "$KCFGPAT" ]; then |
| sed -n '/^define .*\/config/,/^endef/{/^define .*config\|^endef/d;p}' $PKGDIR/Makefile >> config/Config-kernel.in |
| [ ! -e $PKGDIR/Config.in ] || cat $PKGDIR/Config.in >> config/Config-kernel.in |
| fi |
| for d in $(sed 's/^.*\/lib\([a-zA-Z0-9_]\+\)\.so[ .].*$/\1/;t;d' $PKGDIR/Makefile | awk '!x[$0]++'); do |
| LIBPAT="$LIBPAT $d" |
| done |
| echo "Removing $PKGDIR" |
| rm -rf $PKGDIR |
| done |
| if [ -n "$LIBPAT" ]; then |
| LIBPAT="$(tr ' ' '\n' <<< "$LIBPAT" | awk '!x[$0]++' | tr '\n' '|' | sed 's/^|\||$//g;s/|/\\|/g')" |
| PKGMKFL="$(grep -rl "[ =]-l\($LIBPAT\)\>" package --include=Makefile | awk -F / '$n!=s;{n=NF-1;s=$n}')" |
| for c in $PKGMKFL; do |
| if [ ! -e $c ]; then |
| continue |
| fi |
| PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $c)" |
| PKGDIR="$(dirname $c)" |
| if [ -z "$PKGNAMES" ]; then |
| while ! grep -q 'define \(Kernel\)\?Package' $PKGDIR/Makefile; do |
| d="$(dirname $PKGDIR)" |
| while [ -n "$d" -a ! -e $d/Makefile ]; do |
| d="$(dirname $d)" |
| done |
| if [ "$d" = "." ]; then |
| d= |
| fi |
| if [ -n "$d" ]; then |
| PKGDIR="$d" |
| else |
| break |
| fi |
| done |
| if [ -z "$d" ]; then |
| continue |
| fi |
| PKGNAMES="$(sed 's/^define \(Kernel\)\?Package\///;t n;d;:n;/\//d;s/\$(.*//' $PKGDIR/Makefile)" |
| if [ -z "$PKGNAMES" ]; then |
| PKGNAMES="$(basename $PKGDIR)" |
| fi |
| fi |
| for d in $PKGNAMES; do |
| if [ -z "$PKGPAT" ]; then |
| PKGPAT=$d |
| else |
| PKGPAT="$PKGPAT\|$d" |
| fi |
| done |
| KCFGPAT= |
| for d in $(sed 's/^ *config //;t n;d;:n;s/ //g' $PKGDIR/Makefile $PKGDIR/Config.in 2>/dev/null); do |
| if [ "$d" != "${d#KERNEL_}" ]; then |
| KCFGPAT="$KCFGPAT $d" |
| continue |
| fi |
| if [ -z "$CFGPAT" ]; then |
| CFGPAT=$d |
| else |
| CFGPAT="$CFGPAT\|$d" |
| fi |
| done |
| if [ -n "$KCFGPAT" ]; then |
| sed -n '/^define .*\/config/,/^endef/{/^define .*config\|^endef/d;p}' $PKGDIR/Makefile >> config/Config-kernel.in |
| [ ! -e $PKGDIR/Config.in ] || cat $PKGDIR/Config.in >> config/Config-kernel.in |
| fi |
| for d in $(sed 's/^.*\/lib\([a-zA-Z0-9_]\+\)\.so[ .].*$/\1/;t;d' $PKGDIR/Makefile | awk '!x[$0]++'); do |
| LIBPAT="$LIBPAT $d" |
| done |
| echo "Removing $PKGDIR" |
| rm -rf $PKGDIR |
| done |
| fi |
| for c in $(grep -rl '\<marvell/\|\<RERRMSG\>' package --include='*.patch'); do |
| rm -fv $c |
| done |
| rm -rfv target/linux/mmp/*/*/base-files package/base-files/profile-files/* |
| git rm -rf docs/marvell mgit.sh ugit.sh scripts/env_tools |
| sed -i "/\<CONFIG_PACKAGE_\\(kmod-\\)\\?\\($PKGPAT\\)\>\|\<CONFIG_\\($CFGPAT\\)\>/d" config/defconfig* |
| NOBLD="BuildSwdownloader\|BuildFota\|BuildSecureSwd\|BuildSecFota\|BuildDiagMdb\|save_manifest" |
| sed -i "/^define \($NOBLD\)/,/^endef/d;/\$(call \($NOBLD\))/d" target/linux/mmp/image/Makefile |
| sed -i '/^if/{N;/^if.*\nendif/d}' target/linux/mmp/image/Makefile |
| } |
| |
| pack_repo() |
| { |
| BASEDIR="$(basename $TOP_DIR)" |
| TOPFILE="$(git ls-files | cut -d / -f 1 | uniq | sed "s/^/$BASEDIR\//")" |
| for c in $REPOPATHS ; do |
| if [ "$c" != "${c#marvell/}" -a "$c" != "marvell/linux" -a "$c" != "marvell/uboot" ]; then |
| continue |
| fi |
| if [ "$c" != "${c#marvell/}" ]; then |
| make -C $c mrproper |
| rm -rfv $c/ipkg-* |
| fi |
| TOPFILE="$TOPFILE $BASEDIR/$c" |
| done |
| ( |
| set -x |
| tar -C $TOP_DIR/.. -czf $TOP_DIR/$1 --exclude=.git $TOPFILE |
| ) |
| echo "Generated archive `du -sh $1`" |
| } |
| |
| cd ${TOP_DIR} |
| if [ $# -lt 1 ]; then |
| usage |
| fi |
| |
| # path_cur=${PWD##*/} |
| # if [[ $path_cur != openwrt ]]; then |
| # usage |
| # fi |
| |
| case "$1" in |
| clone) |
| clone_repos |
| ;; |
| pull) |
| #git pull --rebase |
| run_command_in_dirs "git pull" |
| ;; |
| status) |
| run_command_in_dirs "git status" |
| ;; |
| checkout) |
| if [ -z "$2" ]; then |
| echo "'$1' accepts an additional 'command' parameter." |
| cd - |
| exit 1 |
| fi |
| run_command_in_dirs "git checkout $2" |
| ;; |
| coman) |
| if [ -z "$2" ]; then |
| echo "'$1' accepts the manifest file for a specific build on Jenkins server" |
| cd - |
| exit 1 |
| fi |
| checkout_bld_point "$2" |
| ;; |
| genopen) |
| if [ -z "$2" ]; then |
| echo "'$1' accepts the manifest file for a specific build on Jenkins server" |
| cd - |
| exit 1 |
| fi |
| git stash |
| checkout_bld_point "$2" |
| run_command_in_dirs "git rm -f .arcconfig" |
| remove_private_code |
| pack_repo opensource_`date +%F`.tgz |
| git reset --hard |
| run_command_in_dirs "git checkout HEAD .arcconfig; git checkout -" |
| ;; |
| mdiff) |
| if [ -z "$2" ]; then |
| echo "'$1' input a manifest file for a specific build on Jenkins server" |
| cd - |
| exit 1 |
| fi |
| git config --global diff.renameLimit 0 |
| changes_since_manifest "$2" |
| ;; |
| #changes in $2 days |
| dlog) |
| logdate=`date +'%Y-%m-%d'` |
| if [ -z "$2" ]; then |
| startdate=`date +'%Y-%m-%d' --date="-1 day"` |
| else |
| startdate=`date +'%Y-%m-%d' --date="-$2 day"` |
| fi |
| logtype="time" |
| check_repo_logs "$startdate" "$logdate" "$logtype" |
| ;; |
| #changes between two tags/releases |
| rlog) |
| if [ -z "$2" ]; then |
| echo "'$1' accepts 1 or 2 additional 'command' parameters to specify the tag interval." |
| cd - |
| exit 1 |
| fi |
| logtype="tags" |
| check_repo_logs "$2" "$3" "$logtype" |
| ;; |
| PICK|SYNC) |
| if [ -z "$2" -o -z "$3" -o -z "$4" ]; then |
| echo "'$1' accepts 3 or 4 additional parameters to specify the dev branch, diff / build ID, SDK branch(es)" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| JOBNAME= |
| if [ "$2" = "master" ]; then |
| JOBNAME=openwrtnightly |
| elif [ "$2" = "v2102" ]; then |
| JOBNAME=openwrt_v2102 |
| else |
| echo "Invalid source dev branch $2" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| for BRCH in $4 $5; do |
| if [[ $BRCH != *$2 ]] ; then |
| echo "$BRCH incompatible with $2" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| done |
| set -e |
| #clone_repos |
| run_command_in_dirs "git stash && git checkout $2 && git pull" |
| git config --global push.default simple |
| if [ "$1" = "PICK" ]; then |
| ID="${3//D/}" |
| echo "Querying differential with ID $ID" |
| ARC=/phab/web/www/arcanist/bin/arc |
| if [ ! -e $ARC ]; then |
| ARC=`which arc` |
| fi |
| JSON="$($ARC call-conduit -- differential.query <<< "{\"ids\":[$ID]}")" |
| ERR="$(jq .error <<< "$JSON")" |
| if [ "$ERR" = "null" ]; then |
| if [ "$(jq '.response | length' <<< "$JSON")" -ge 1 ]; then |
| declare -A REPOS |
| REPOS[null]=N/A |
| REPOPATH_OW=openwrt |
| REPOPATH_ETL=external/management |
| REPOPATH_ERT=external/routing |
| REPOPATH_ESP=external/subpack |
| REPOPATH_MKL=marvell/linux |
| REPOPATH_OTL=marvell/lte-telephony |
| REPOPATH_BL=marvell/obm |
| REPOPATH_SVR=marvell/services |
| REPOPATH_DL=marvell/swd |
| REPOPATH_UO=marvell/uboot |
| REPOPATH_OTA=marvell/fota |
| REPOPATH_WEB=marvell/webui |
| REPOPATH_FP=$REPOPATH_OW # marvell/fastpath |
| IDLIST= |
| for ID in $(tr ',' '\n' <<< "$ID"); do |
| if [ -n "$(jq '.response | .[] | select(.id == "'$ID'") | length' <<< "$JSON")" ]; then |
| IDLIST="$IDLIST $ID" |
| fi |
| done |
| for ID in $IDLIST; do |
| PART="$(jq '.response | .[] | select(.id == "'$ID'")' <<< "$JSON")" |
| jq -r '.uri + "\t" + .title' <<< "$PART" |
| STAT="$(jq -r .statusName <<< "$PART")" |
| if [ "$STAT" != "Closed" ]; then |
| echo "Patch not closed: $STAT" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| REPO="$(jq -r .repositoryPHID <<< "$PART")" |
| RNAM="$(set +e; echo "${REPOS[$REPO]}" 2>/dev/null)" |
| if [ -z "$RNAM" ]; then |
| RJSN="$($ARC call-conduit -- diffusion.repository.search <<< "{\"constraints\":{\"phids\":[\"$REPO\"]}}")" |
| ERR="$(jq .error <<< "$RJSN")" |
| if [ "$ERR" = "null" ]; then |
| if [ "$(jq '.response.data | length' <<< "$RJSN")" -ge 1 ]; then |
| RNAM="$(jq -r '.response.data | .[] | .fields.callsign' <<< "$RJSN")" |
| REPOS[$REPO]=$RNAM |
| else |
| echo "Empty response from server for $REPO" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| else |
| jq -r .errorMessage <<< "$RJSN" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| elif [ "$RNAM" = "N/A" ]; then |
| echo "Cannot determine repository for D$ID. Manual operation needed." |
| cd - &>/dev/null |
| exit 2 |
| fi |
| REPOPATH="$(eval "echo \${REPOPATH_$RNAM}")" |
| if [ -z "$REPOPATH" ]; then |
| echo "Empty repository for D$ID. Manual operation needed." |
| cd - &>/dev/null |
| exit 2 |
| fi |
| echo " for repository $REPOPATH" |
| if [ $REPOPATH = openwrt ]; then |
| REPOPATH=. |
| fi |
| CMIT="$(jq -c '.commits' <<< "$PART")" |
| CJSN="$($ARC call-conduit -- diffusion.commit.search <<< "{\"constraints\":{\"phids\":$CMIT}}")" |
| ERR="$(jq .error <<< "$CJSN")" |
| for i in `seq 15`; do |
| if [ "$ERR" = "null" ]; then |
| break |
| fi |
| jq -r .errorMessage <<< "$CJSN" |
| echo "Waiting 2 minutes to retry..." |
| sleep 120 |
| JSON="$($ARC call-conduit -- differential.query <<< "{\"ids\":[$ID]}")" |
| ERR="$(jq .error <<< "$JSON")" |
| if [ "$ERR" = "null" ]; then |
| PART="$(jq '.response | .[] | select(.id == "'$ID'")' <<< "$JSON")" |
| CMIT="$(jq -c '.commits' <<< "$PART")" |
| CJSN="$($ARC call-conduit -- diffusion.commit.search <<< "{\"constraints\":{\"phids\":$CMIT}}")" |
| ERR="$(jq .error <<< "$CJSN")" |
| fi |
| done |
| if [ "$ERR" = "null" ]; then |
| if [ "$(jq '.response.data | length' <<< "$CJSN")" -ge 1 ]; then |
| FOUND= |
| TARGET= |
| TARGET2= |
| for CMID in $(jq -r '.response.data | .[] | .fields.identifier' <<< "$CJSN"); do |
| BRCH="$(cd $REPOPATH; git branch -r --contains $CMID | sed '/->/d;s/^.*\///')" |
| echo " commit $CMID on $(tr '\n' ' ' <<< "$BRCH")" |
| if grep -Fqx "$2" <<< "$BRCH"; then |
| FOUND="$CMID" |
| fi |
| if grep -Fqx "$4" <<< "$BRCH"; then |
| TARGET="$CMID" |
| fi |
| if [ .$5 != . ] && grep -Fqx "$5" <<< "$BRCH"; then |
| TARGET2="$CMID" |
| fi |
| done |
| if [ -z "$FOUND" ]; then |
| echo "No commit found on $2" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| if [ -z "$TARGET" ]; then |
| echo "Picking commit $FOUND onto $4" |
| ( |
| cd $REPOPATH; |
| set -x; |
| git checkout $4; |
| git reset --hard origin/$4; |
| git cherry-pick $FOUND; |
| #git push |
| ) |
| TARGET="$(cd $REPOPATH; git show -s --pretty=tformat:%H)" |
| if [ .$5 = .$4 -a -z "$TARGET2" ]; then |
| TARGET2="$TARGET" |
| fi |
| else |
| echo "Already on $4" |
| fi |
| if [ .$5 == . ]; then |
| : |
| elif [ -z "$TARGET2" ]; then |
| echo "Picking commit $FOUND onto $5" |
| ( |
| cd $REPOPATH; |
| set -x; |
| git checkout $5; |
| git reset --hard origin/$5; |
| git cherry-pick $FOUND; |
| #git push |
| ) |
| TARGET2="$(cd $REPOPATH; git show -s --pretty=tformat:%H)" |
| else |
| echo "Already on $5" |
| fi |
| # find reverting changes |
| RLST="$(cd $REPOPATH; git log --pretty=tformat:'COMMIT: %H%n%B' $FOUND..origin/$2 | awk '/^COMMIT: /{h=$2;next};h&&/([Rr]evert[sed]*|[Bb]ack[ sed]out|[Uu]ndo[es]) .*(commits?|change[set]*|rev[ison]*|diffs?) +'$FOUND'/{print h;h=""}')" |
| if [ -n "$RLST" ]; then |
| echo "Found $(wc -l <<< "$RLST") reverting change(s) on $2" |
| for CMID in $RLST; do |
| (cd $REPOPATH; git show -s --pretty=tformat:' commit %H : %B' $CMID | sed '/^ *$/d') |
| echo |
| done |
| TLST="$(cd $REPOPATH; git log --pretty=tformat:'COMMIT: %H%n%B' $TARGET..origin/$4 | awk '/^COMMIT: /{h=$2;next};h&&/([Rr]evert[sed]*|[Bb]ack[ sed]out|[Uu]ndo[es]) .*(commits?|change[set]*|rev[ison]*|diffs?) +('$FOUND'|'$TARGET')/{print h;h=""}')" |
| if [ -n "$TLST" ]; then |
| echo "Found $(wc -l <<< "$TLST") reverting change(s) on $4" |
| for CMID in $TLST; do |
| (cd $REPOPATH; git show -s --pretty=tformat:' commit %H : %s' $CMID | sed '/^ *$/d') |
| done |
| else |
| for CMID in $RLST; do |
| echo "Picking commit $CMID onto $4" |
| ( |
| cd $REPOPATH; |
| set -x; |
| git checkout $4; |
| git cherry-pick $CMID; |
| #git push |
| ) |
| done |
| fi |
| if [ .$5 != . -a .$5 != .$4 ]; then |
| TLST="$(cd $REPOPATH; git log --pretty=tformat:'COMMIT: %H%n%B' $TARGET2..origin/$5 | awk '/^COMMIT: /{h=$2;next};h&&/([Rr]evert[sed]*|[Bb]ack[ sed]out|[Uu]ndo[es]) .*(commits?|change[set]*|rev[ison]*|diffs?) +('$FOUND'|'$TARGET2')/{print h;h=""}')" |
| if [ -n "$TLST" ]; then |
| echo "Found $(wc -l <<< "$TLST") reverting change(s) on $5" |
| for CMID in $TLST; do |
| (cd $REPOPATH; git show -s --pretty=tformat:' commit %H : %s' $CMID | sed '/^ *$/d') |
| done |
| else |
| for CMID in $RLST; do |
| echo "Picking commit $CMID onto $5" |
| ( |
| cd $REPOPATH; |
| set -x; |
| git checkout $5; |
| git cherry-pick $CMID; |
| #git push |
| ) |
| done |
| fi |
| fi |
| fi |
| else |
| echo "Empty response from server for $CMIT" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| echo |
| else |
| jq -r .errorMessage <<< "$CJSN" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| done |
| else |
| echo "Empty response from server for $3" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| else |
| jq -r .errorMessage <<< "$JSON" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| else |
| if [ ! -e "$TOP_DIR/.arcconfig" -a -e "$TOP_DIR/openwrt/.arcconfig" ]; then |
| TOP_DIR=$TOP_DIR/openwrt |
| fi |
| HOST="$(jq -r '.conduit_uri' $TOP_DIR/.arcconfig)" |
| if [ "$HOST" = "null" ]; then |
| HOST="$(jq -r '.["phabricator.uri"]' $TOP_DIR/.arcconfig)" |
| fi |
| JOBURL="${HOST%/*}:8080/job/$JOBNAME" |
| echo "Querying artifacts for $JOBNAME build $3" |
| JSON="$(wget $JOBURL/$3/api/json -o /dev/null -O -)" |
| echo "Parsing json..." |
| RMAN="$(jq -r '.artifacts | map(select(.fileName == "manifest.txt")) | .[0] | .relativePath' <<< "$JSON")" |
| echo "Fetching the first of $(jq -r '.artifacts | map(select(.fileName == "manifest.txt")) | length' <<< "$JSON") profiles: $RMAN" |
| MANI="$(wget $JOBURL/$3/artifact/$RMAN -o /dev/null -O -)" |
| echo "Parsing manifest..." |
| MCNT="$(wc -l <<< "$MANI")" |
| RCNT="$(awk "{x+=NF};END{print 0+x}" <<< "$REPOPATHS")" |
| if [ "$MCNT" -lt "$RCNT" ]; then |
| echo "Too few lines in manifest: $RCNT expected, but got $MCNT" |
| cd - &>/dev/null |
| exit 1 |
| fi |
| VER="$(cat target/linux/mmp/image/version.asr)" |
| VERCMID= |
| if [ $VER -lt $3 -a "$6" = "true" ]; then |
| echo "Updating version.asr from $VER to $3" |
| echo $3 > target/linux/mmp/image/version.asr |
| git add target/linux/mmp/image/version.asr |
| git commit -m "target:update image version to $3" |
| git push |
| VERCMID="$(git show -s --pretty=tformat:%H)" |
| fi |
| for BRCH in $4 $5; do |
| L=1 |
| echo -e "Processing autopick to $BRCH in $RED"openwrt"$RESET" |
| CMID="$(sed -e $L'q;d' <<< "$MANI" | cut -d'-' -f1)" |
| ( |
| set -x; |
| git checkout $BRCH; |
| git reset --hard origin/$BRCH; |
| $TOP_DIR/scripts/autopick.sh $CMID; |
| ) |
| echo -e "\r\n" |
| for REPOPATH in $REPOPATHS ; do |
| if [ -d "$REPOPATH" ]; then |
| echo -e "Processing autopick to $BRCH in $RED${REPOPATH}$RESET" |
| L=$((L+1)) |
| CMID="$(sed -e $L'q;d' <<< "$MANI" | cut -d'-' -f1)" |
| ( |
| set -x; |
| cd $REPOPATH; |
| git checkout $BRCH; |
| git reset --hard origin/$BRCH; |
| $TOP_DIR/scripts/autopick.sh $CMID; |
| ) |
| echo -e "\r\n" |
| else |
| echo -e "Skipping non-existent dir: $RED${REPOPATH}$RESET\r\n" |
| fi |
| done |
| if [ -n "$VERCMID" ]; then |
| ( |
| set -x; |
| git cherry-pick $VERCMID |
| ) |
| else |
| VER="$(cat target/linux/mmp/image/version.asr)" |
| if [ $VER -lt $3 -a "$6" = "true" ]; then |
| echo "Updating version.asr on $BRCH from $VER to $3" |
| echo $3 > target/linux/mmp/image/version.asr |
| git add target/linux/mmp/image/version.asr |
| git commit -m "target:update image version to $3" |
| fi |
| fi |
| L=1 |
| #run_command_in_dirs "git push" |
| done |
| fi |
| ;; |
| run) |
| if [ -z "$2" ]; then |
| echo "'$1' accepts an additional 'command' parameter." |
| cd - |
| exit 1 |
| fi |
| run_command_in_dirs "$2" |
| ;; |
| erun) |
| if [ -z "$2" ]; then |
| echo "'$1' accepts an additional 'command' parameter." |
| cd - |
| exit 1 |
| fi |
| set -e |
| run_command_in_dirs "$2" |
| ;; |
| *) |
| usage |
| ;; |
| esac |
| cd - &>/dev/null |