From c1ac5805547ec91e7f4c0592dd2520092d10c906 Mon Sep 17 00:00:00 2001 From: mirabilos Date: Sat, 12 Nov 2016 01:54:22 +0000 Subject: [PATCH] update from MirBSD CVS --- mksh/debian-dev/BuildDSC.sh | 30 ++-- mksh/debian-dev/mkdebidx.sh | 338 +++++++++++++++++++++++++++++++------------- 2 files changed, 259 insertions(+), 109 deletions(-) diff --git a/mksh/debian-dev/BuildDSC.sh b/mksh/debian-dev/BuildDSC.sh index 6ee59b0..ce4b0f7 100644 --- a/mksh/debian-dev/BuildDSC.sh +++ b/mksh/debian-dev/BuildDSC.sh @@ -1,10 +1,10 @@ #!/bin/mksh -# $MirOS: contrib/hosted/tg/deb/BuildDSC.sh,v 1.16 2015/05/23 16:57:58 tg Exp $ +# $MirOS: contrib/hosted/tg/deb/BuildDSC.sh,v 1.19 2016/02/23 18:05:35 tg Exp $ #- # Copyright (c) 2010, 2011 # Thorsten Glaser -# Copyright © 2015 -# Thorsten “mirabilos” Glaser +# Copyright © 2015, 2016 +# mirabilos # # Provided that these terms and disclaimer and all copyright notices # are retained or reproduced in an accompanying document, permission @@ -26,7 +26,8 @@ # source package. It will then be renamed to the proper dirname, and # a source package (*.dsc + others) will be created, then it will be # renamed back. -# -d: pass -d to dpkg-buildpackage +# -d: pass -d to dpkg-buildpackage (ignore B-D absence) +# -N: pass -nc to dpkg-buildpackage (do not clean) # -s arg: make a snapshot with “arg” being the version number suffix # -S: build a snapshot with snapshot.YYYYMMDD.HHMMSS (UTC) as suffix # Any further arguments will be passed to debian/rules via MAKEFLAGS @@ -43,18 +44,31 @@ stime_rfc=$(date +"%a, %d %b %Y %H:%M:%S %z") stime_vsn=$(date -u +"%Y%m%d.%H%M%S") optd= +optN= snap=0 ssuf= -while getopts "dSs:" ch; do +while getopts "dNSs:" ch; do case $ch { (d) optd=-d ;; + (+d) optd= + ;; + (N) optN=-nc + ;; + (+N) optN= + ;; (S) snap=1 ssuf=snapshot.$stime_vsn ;; + (+S) snap=0 + ssuf= + ;; (s) snap=1 ssuf=$OPTARG ;; + (+s) snap=0 + ssuf= + ;; (*) print -u2 Syntax error. exit 1 ;; @@ -63,7 +77,7 @@ done shift $((OPTIND - 1)) export MAKEFLAGS="$*" -if (( snap )) && [[ -z $DEBEMAIL ]]; then +if (( snap )) && [[ $DEBEMAIL != +([A-Za-z])*' <'*'>' ]]; then print -u2 'Please set $DEBEMAIL to "First M. Last "' exit 1 fi @@ -130,9 +144,9 @@ curname=${mydir##*/} newname=$pkgstem-$upstreamversion [[ $newname = $curname ]] || mv "$curname" "$newname" cd "$newname" -dpkg-buildpackage -rfakeroot -S -I -i $optd +dpkg-buildpackage -rfakeroot -S -I -i $optd $optN -us -uc rv=$? -fakeroot debian/rules clean +[[ -n $optN ]] || fakeroot debian/rules clean cd .. [[ $newname = $curname ]] || mv "$newname" "$curname" diff --git a/mksh/debian-dev/mkdebidx.sh b/mksh/debian-dev/mkdebidx.sh index 8661e8d..b59749a 100644 --- a/mksh/debian-dev/mkdebidx.sh +++ b/mksh/debian-dev/mkdebidx.sh @@ -1,8 +1,8 @@ #!/bin/mksh -rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.61 2014/05/23 09:44:30 tg Exp $' +rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.74 2016/11/05 16:45:56 tg Exp $' #- -# Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014 -# Thorsten Glaser +# Copyright © 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 +# mirabilos # # Provided that these terms and disclaimer and all copyright notices # are retained or reproduced in an accompanying document, permission @@ -19,23 +19,53 @@ rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.61 2014/05/23 09:44:30 tg E # damage or existence of a defect, except proven that it results out # of said person's immediate fault when using the work as intended. -set -A normarchs -- i386 -repo_keyid=0xAA917C6F +unset normarchs repo_keyid gpg_remote gpg_bin repo_origin repo_label repo_title +unset -f repo_intro repo_description +me=$(dirname "$0"); [[ -s $me/mkdebidx.inc ]] && . "$me/mkdebidx.inc" +unset me + +[[ -n ${normarchs+x} ]] || set -A normarchs -- i386 # either '' (locally) or 'remsign user@host.domain.com' (remote ssh) -gpg_remote= -repo_origin='The MirOS Project' -repo_label=wtf -repo_title='MirDebian “WTF” Repository' -function repo_description { +[[ -n ${repo_keyid+x} ]] || repo_keyid=0xAA917C6F +[[ -n ${gpg_remote+x} ]] || gpg_remote= +case ${gpg_bin:-x} { +(gpg|gpg1|gpg2|gnupg|gnupg1|gnupg2) ;; +(*) + if [[ -n $gpg_remote ]]; then + gpg_bin=gpg + elif ! gpg_bin=$(whence -p gpg1); then + gpg_bin=gpg + fi + ;; +} +[[ -n ${repo_origin+x} ]] || repo_origin='The MirOS Project' +[[ -n ${repo_label+x} ]] || repo_label=wtf +[[ -n ${repo_title+x} ]] || repo_title='MirDebian “WTF” Repository' +typeset -f repo_intro >/dev/null || function repo_intro { + cat <<-'EOF' +

This APT repository contains packages by mirabilos (wtf) + for use with the Debian operating system and its derivates. It is + not affiliated with Debian. Some of the content is merely hosted + for people close to MirBSD or Debian; some is affiliated with The + MirOS Project.
Debian is a registered trademark owned + by Software in the Public Interest, Inc.
“The MirOS Project” + and “MirBSD” are unregistered trademarks owned by mirabilos.

+EOF +} +typeset -f repo_description >/dev/null || function repo_description { typeset suite_nick=$1 print -nr -- "WTF ${suite_nick} Repository" } - - set -A dpkgarchs -- alpha amd64 arm arm64 armel armhf hppa hurd-i386 i386 \ ia64 kfreebsd-amd64 kfreebsd-i386 m68k mips mipsel powerpc powerpcspe \ - ppc64 s390 s390x sh4 sparc sparc64 x32 + ppc64 ppc64el s390 s390x sh4 sparc sparc64 x32 +[[ -n "${normarchs[*]}" ]] || set -A normarchs -- "${dpkgarchs[@]}" + +set +U +export LC_ALL=C +unset LANGUAGE +typeset -Z11 -Uui16 hv function remsign { target=$1; shift @@ -50,6 +80,29 @@ function remsign { return $rv } +function die { + local rv=1 + + if [[ $1 = +([0-9]) ]]; then + rv=$1 + shift + fi + print -ru2 -- "E: $*" + exit $rv +} + +function checkedhash { + if [[ $1 = size ]]; then + REPLY=$(stat -c '%s' "$2") + [[ $REPLY = +([0-9]) ]] || die "Error getting size of '$2'" + else + set -o noglob + set -A REPLY -- $($1 "$2") + set +o noglob + [[ $REPLY = +([0-9a-f]) ]] || die "Error getting $1 of '$2'" + fi +} + function putfile { tee $1 | gzip -n9 >$1.gz } @@ -68,17 +121,6 @@ function sortlist { done | sort $u } -# DJB cdb hash (not via stdio, for speed) -typeset -Z11 -Uui16 Lcdbhash_result -function Lcdbhash_add { - typeset s="$*" - typeset -i i=0 n=${#s} - - while (( i < n )); do - ((# Lcdbhash_result = (Lcdbhash_result * 33) ^ 1#${s:(i++):1} )) - done -} - # escape XHTML characters (three mandatory XML ones plus double quotes, # the latter in an XML safe fashion numerically though) function xhtml_escape { @@ -93,16 +135,12 @@ function xhtml_escape { -e 's"\"g' } -set +U -export LC_ALL=C -unset LANGUAGE -saveIFS=$IFS cd "$(dirname "$0")" rm -f dpkg_578162_workaround -IFS=: +IFS=:; set -o noglob dpkgarchl=:all:"${dpkgarchs[*]}": -IFS=$saveIFS +IFS=$' \t\n'; set +o noglob suites=: for suite in "$@"; do @@ -112,6 +150,7 @@ done allsuites= for suite in dists/*; do allsuites="$allsuites${allsuites:+ }${suite##*/}" + [[ -h $suite ]] && continue [[ $suites = : || $suites = *:"$suite":* ]] || continue archs= distribution= @@ -131,7 +170,9 @@ for suite in dists/*; do archs= [[ -s $dist/distinfo.sh ]] && . $dist/distinfo.sh set -A distarchs -- $(sortlist -u all ${archs:-$suitearchs}) - IFS=:; distarchl=:"${distarchs[*]}":; IFS=$saveIFS + IFS=:; set -o noglob + distarchl=:"${distarchs[*]}": + IFS=$' \t\n'; set +o noglob nmds=0 for arch in $(sortlist -u ${distarchs[*]} ${dpkgarchs[*]}) /; do # put "all" last @@ -139,8 +180,7 @@ for suite in dists/*; do [[ $arch = / ]] && arch=all # create index if [[ $dpkgarchl != *:"$arch":* ]]; then - print -u2 "Invalid arch '$arch' in $dist" - exit 1 + die "Invalid arch '$arch' in $dist" elif [[ $distarchl != *:"$arch":* ]]; then print "\n===> Linking all =>" \ "${dist#dists/}/$arch/Packages" @@ -153,9 +193,12 @@ for suite in dists/*; do print "\n===> Creating" \ "${dist#dists/}/$arch/Packages\n" mkdir -p $dist/binary-$arch - dpkg-scanpackages $oef $om -a $arch \ - $dist $ovf | \ + (dpkg-scanpackages $oef $om -a $arch \ + $dist $ovf || \ + echo $? >$dist/binary-$arch/failed) | \ putfile $dist/binary-$arch/Packages + [[ -e $dist/binary-$arch/failed ]] && \ + exit $(<$dist/binary-$arch/failed) (( nmds++ )) || firstmd=$arch fi done @@ -164,19 +207,102 @@ for suite in dists/*; do [[ -e dpkg_578162_workaround ]] || (dpkg-scansources $oef $osf \ $dist $ovf || touch dpkg_578162_workaround) | \ putfile $dist/source/Sources - [[ -e dpkg_578162_workaround ]] && dpkg-scansources $osf \ - $dist $ovf | \ + [[ -e dpkg_578162_workaround ]] && (dpkg-scansources $osf \ + $dist $ovf || echo $? >$dist/source/failed) | \ putfile $dist/source/Sources + [[ -e $dist/source/failed ]] && exit $(<$dist/source/failed) + print done. + print "\n===> Creating ${dist#dists/}/i18n/Index" + [[ -d $dist/i18n/. ]] || mkdir -p $dist/i18n + [[ -d $dist/i18n/. ]] || die "Cannot create $dist/i18n" + rm -f $dist/i18n/.done + (cd $dist/i18n + tfiles=/ + [[ -h .hashcache || ! -f .hashcache ]] && rm -rf .hashcache + for ent in .* *; do + [[ $ent = . || $ent = .. || $ent = .hashcache ]] && continue + [[ -h $ent || -e $ent ]] || continue + if [[ ! -f $ent || $ent != Translation-* ]]; then + rm -rf "$ent" + continue + fi + ent=${ent#Translation-} + ent=${ent%.bz2} + [[ $tfiles = */"$ent"/* ]] || tfiles+=$ent/ + done + [[ -e .hashcache ]] || :>.hashcache + if [[ $tfiles = / ]]; then + :>Translation-tlh_DE + tfiles=/tlh_DE/ + fi + print SHA1: >Index + IFS=/; set -o noglob + set -A tflist -- ${tfiles#/} + IFS=$' \t\n'; set +o noglob + for ent in "${tflist[@]}"; do + ent=Translation-$ent + if [[ $ent -nt $ent.bz2 ]]; then + if ! bzip2 -9 <"$ent" >"$ent.bz2"; then + rm -f "$ent.bz2" + die "bzip2 '$ent' died" + fi + elif [[ -e $ent ]]; then + rm -f "$ent" + fi + hash=${|checkedhash sha1sum "$ent.bz2";} || exit 1 + hnum=0 + grep "^$hash " .hashcache |& + while read -p hsha1 hsize hmd5 hsha2 usha1 usize umd5 usha2; do + [[ $hsha1 = "$hash" ]] || continue + hnum=1 + while read -p hsha1 x; do + # flush coprocess, look for dupes + [[ $hsha1 = "$hash" ]] && hnum=2 + done + break + done + hsha1=$hash + if (( hnum != 1 )); then + [[ -e $ent ]] || \ + if ! bzip2 -d <"$ent.bz2" >"$ent"; then + rm -f "$ent" + die "bzip2 '$ent.bz2' died" + fi + umd5=${|checkedhash md5sum "$ent";} || exit 1 + hmd5=${|checkedhash md5sum "$ent.bz2";} || exit 1 + usha1=${|checkedhash sha1sum "$ent";} || exit 1 + usha2=${|checkedhash sha256sum "$ent";} || exit 1 + hsha2=${|checkedhash sha256sum "$ent.bz2";} || exit 1 + usize=${|checkedhash size "$ent";} || exit 1 + hsize=${|checkedhash size "$ent.bz2";} || exit 1 + (( hnum )) || print $hsha1 $hsize $hmd5 $hsha2 $usha1 $usize $umd5 $usha2 >>.hashcache + fi + [[ -e $ent ]] && rm -f "$ent" + print -u4 $hsha1 $hsize $hmd5 $hsha2 $usha1 $usize $umd5 $usha2 + print -ru5 " $hsha1 $hsize $ent.bz2" + print -ru6 " $umd5 $usize ${dist##*/}/i18n/$ent" + print -ru6 " $hmd5 $hsize ${dist##*/}/i18n/$ent.bz2" + print -ru7 " $usha1 $usize ${dist##*/}/i18n/$ent" + print -ru7 " $hsha1 $hsize ${dist##*/}/i18n/$ent.bz2" + print -ru8 " $usha2 $usize ${dist##*/}/i18n/$ent" + print -ru8 " $hsha2 $hsize ${dist##*/}/i18n/$ent.bz2" + done 4>.hashcache.new 5>>Index 6>.hashcache.md5 7>.hashcache.sha1 8>.hashcache.sha2 + rm -f .hashcache + mv -f .hashcache.new .hashcache + :>.done) + [[ -e $dist/i18n/.done ]] || die i18n generation unsuccessful + rm -f $dist/i18n/.done print done. done print "\n===> Creating ${suite#dists/}/Release" rm -f $suite/Release-* + xdone=$(realpath $suite/Release-done) (cat <<-EOF Origin: ${repo_origin} Label: ${repo_label} Suite: ${distribution:-${suite##*/}} Codename: ${suite##*/} - Date: $(date -u) + Date: $(date -Ru) Architectures: all ${dpkgarchs[*]} source $components Description: $(repo_description "$nick") @@ -192,76 +318,62 @@ for suite in dists/*; do set -A cache_sha1 set -A cache_sha2 set -A cache_size - for n in Contents-* */{binary-*,source}/{Packag,Sourc}es*; do + for n in Contents-* */{binary-*,i18n,source}/{Index,{Packag,Sourc}es*}; do [[ -f $n ]] || continue # realpath-ise $n and cache the checksum nn=$(realpath "$n") #XXX once mksh can, use associative arrays instead - Lcdbhash_result=5381 - Lcdbhash_add "$nn" + hv=16#${nn@#} # simple hash collision solver by increment - nc=${cache_fn[Lcdbhash_result]} + nc=${cache_fn[hv]} while [[ -n $nc && $nc != "$nn" ]]; do - nc=${cache_fn[++Lcdbhash_result]} + nc=${cache_fn[++hv]} done if [[ $nc = "$nn" ]]; then - nm=${cache_md5[Lcdbhash_result]} - ns=${cache_size[Lcdbhash_result]} - nsha1=${cache_sha1[Lcdbhash_result]} - nsha2=${cache_sha2[Lcdbhash_result]} + nm=${cache_md5[hv]} + ns=${cache_size[hv]} + nsha1=${cache_sha1[hv]} + nsha2=${cache_sha2[hv]} else # GNU *sum tools are horridly inefficient - set -A x -- $(md5sum "$nn") - nm=${x[0]} - set -A x -- $(sha1sum "$nn") - nsha1=${x[0]} - set -A x -- $(sha256sum "$nn") - nsha2=${x[0]} - ns=$(stat -c '%s' "$nn") - cache_md5[Lcdbhash_result]=$nm - cache_size[Lcdbhash_result]=$ns - cache_fn[Lcdbhash_result]=$nn - cache_sha1[Lcdbhash_result]=$nsha1 - cache_sha2[Lcdbhash_result]=$nsha2 + nm=${|checkedhash md5sum "$nn";} || exit 1 + nsha1=${|checkedhash sha1sum "$nn";} || exit 1 + nsha2=${|checkedhash sha256sum "$nn";} || exit 1 + ns=${|checkedhash size "$nn";} || exit 1 + cache_md5[hv]=$nm + cache_size[hv]=$ns + cache_fn[hv]=$nn + cache_sha1[hv]=$nsha1 + cache_sha2[hv]=$nsha2 fi print " $nm $ns $n" print -u4 " $nsha1 $ns $n" print -u5 " $nsha2 $ns $n" - done) >$suite/Release-tmp + if [[ $n = */i18n/Index ]]; then + n=${n%Index} + cat "${n}.hashcache.md5" + cat >&4 "${n}.hashcache.sha1" + cat >&5 "${n}.hashcache.sha2" + rm -f "${n}.hashcache."* + fi + done + :>"$xdone") >$suite/Release-tmp + [[ -e $xdone ]] || die Release generation died cat $suite/Release-sha1 $suite/Release-sha2 >>$suite/Release-tmp - # note: InRelease files can only be safely used by wheezy - # onwards, and oneiric onwards; known to be insecure on - # natty, but usable concurrent to detached files on squeeze - - unset release_sign_detached release_sign_inline - release_sign_detached=x - release_sign_inline=0 + # note: InRelease files can only be safely used by jessie and up. + unset use_inrelease . $suite/distinfo.sh - [[ $release_sign_inline = [01] ]] || release_sign_inline=0 - [[ $release_sign_detached = [01] ]] || \ - (( release_sign_detached = release_sign_inline ? 0 : 1 )) - - # Debian's fix for CVE-2013-1051 consists of disabling - # support for InRelease files altogether (WTF!) - release_sign_detached=1 - release_sign_inline=0 - - (( release_sign_detached )) && $gpg_remote gpg -u $repo_keyid \ - -sab <$suite/Release-tmp >$suite/Release-sig - (( release_sign_inline )) && $gpg_remote gpg -u $repo_keyid \ - --clearsign <$suite/Release-tmp >$suite/Release-inl - - if (( release_sign_inline )); then + rm -f $suite/InRelease $suite/Release $suite/Release.gpg + if [[ $use_inrelease = 1 ]]; then + $gpg_remote $gpg_bin -u $repo_keyid --no-comment --clearsign \ + <$suite/Release-tmp >$suite/Release-inl mv -f $suite/Release-inl $suite/InRelease else - rm -f $suite/InRelease - fi - if (( release_sign_detached )); then + $gpg_remote $gpg_bin -u $repo_keyid --no-comment -sab \ + <$suite/Release-tmp >$suite/Release-sig mv -f $suite/Release-tmp $suite/Release mv -f $suite/Release-sig $suite/Release.gpg - else - rm -f $suite/Release* fi rm -f $suite/Release-* done @@ -280,7 +392,7 @@ if [[ -s mkdebidx.lnk ]]; then [[ $pn = '#'* ]] && continue if [[ $pn != +([a-z0-9_])/+([a-z0-9_-])/+([!/])/@(%|=|+([a-z0-9])) || \ $pd != +([a-z0-9_]) ]]; then - print -u2 "Invalid lnk line '$pn' '$pd'" + print -u2 "W: Invalid lnk line '$pn' '$pd'" continue fi preplsrc[nrpl]=$pn @@ -289,16 +401,17 @@ if [[ -s mkdebidx.lnk ]]; then fi for suite in dists/*; do + [[ -h $suite ]] && continue for dist in $suite/*; do [[ -d $dist/. ]] || continue suitename=${suite##*/} if [[ $suitename != +([a-z0-9_]) ]]; then - print -u2 "Invalid suite name '$suitename'" + print -u2 "W: Invalid suite name '$suitename'" continue 2 fi distname=${dist##*/} if [[ $distname != +([a-z0-9_-]) ]]; then - print -u2 "Invalid dist name '$distname'" + print -u2 "W: Invalid dist name '$distname'" continue fi @@ -373,7 +486,7 @@ for suite in dists/*; do [[ -e $f ]] || continue realpath "$f" done | sort -u) |& - pn=; pv=; pd=; pp=; pN=; pf=; Lf= + pn=; pv=; pd=; pp=; pN=; pf=; pABP=; Lf= while IFS= read -pr line; do case $line { (" "*) @@ -407,17 +520,21 @@ for suite in dists/*; do pf=${line##Filename:*([ ])} Lf=pf ;; + ("Auto-Built-Package: "*) + pABP=${line##Auto-Built-Package:*([ ])} + Lf=pABP + ;; (?*) # anything else Lf= ;; (*) # empty line - if [[ $pf = *:* || $pf = *'%'* ]]; then - print -u2 Illegal character in $dist \ - packages $pp "'Filename: $pf'" - exit 1 - fi + [[ $pf = *:* || $pf = *'%'* ]] && \ + die Illegal character in $dist \ + packages $pp "'Filename: $pf'" [[ -n $pn ]] || pn=$pN - if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then + if [[ $pN = *-dbgsym && $pABP = debug-symbols ]]; then + : skip + elif [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then i=0 while (( i < nbin )); do [[ ${bp_disp[i]} = "$pN" && ${bp_desc[i]} = "$pd" && \ @@ -445,7 +562,7 @@ for suite in dists/*; do eval bp_ver_${suitename}[i]=\$x bp_desc[i]=$pd fi - pn=; pv=; pd=; pp=; pN=; pf=; Lf= + pn=; pv=; pd=; pp=; pN=; pf=; pABP=; Lf= ;; } done @@ -461,8 +578,8 @@ done EOF print -r -- " ${repo_title} Index" +[[ -s NEWS.rss ]] && print '' cat <<'EOF' -