#!/bin/mksh
-rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.61 2014/05/23 09:44:30 tg Exp $'
+rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.74 2016/11/05 16:45:56 tg Exp $'
#-
-# Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014
-# Thorsten Glaser <tg@mirbsd.org>
+# Copyright © 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016
+# mirabilos <m@mirbsd.org>
#
# Provided that these terms and disclaimer and all copyright notices
# are retained or reproduced in an accompanying document, permission
# damage or existence of a defect, except proven that it results out
# of said person's immediate fault when using the work as intended.
-set -A normarchs -- i386
-repo_keyid=0xAA917C6F
+unset normarchs repo_keyid gpg_remote gpg_bin repo_origin repo_label repo_title
+unset -f repo_intro repo_description
+me=$(dirname "$0"); [[ -s $me/mkdebidx.inc ]] && . "$me/mkdebidx.inc"
+unset me
+
+[[ -n ${normarchs+x} ]] || set -A normarchs -- i386
# either '' (locally) or 'remsign user@host.domain.com' (remote ssh)
-gpg_remote=
-repo_origin='The MirOS Project'
-repo_label=wtf
-repo_title='MirDebian “WTF” Repository'
-function repo_description {
+[[ -n ${repo_keyid+x} ]] || repo_keyid=0xAA917C6F
+[[ -n ${gpg_remote+x} ]] || gpg_remote=
+case ${gpg_bin:-x} {
+(gpg|gpg1|gpg2|gnupg|gnupg1|gnupg2) ;;
+(*)
+ if [[ -n $gpg_remote ]]; then
+ gpg_bin=gpg
+ elif ! gpg_bin=$(whence -p gpg1); then
+ gpg_bin=gpg
+ fi
+ ;;
+}
+[[ -n ${repo_origin+x} ]] || repo_origin='The MirOS Project'
+[[ -n ${repo_label+x} ]] || repo_label=wtf
+[[ -n ${repo_title+x} ]] || repo_title='MirDebian “WTF” Repository'
+typeset -f repo_intro >/dev/null || function repo_intro {
+ cat <<-'EOF'
+ <p>This APT repository contains packages by mirabilos (<i>wtf</i>)
+ for use with the Debian operating system and its derivates. It is
+ not affiliated with Debian. Some of the content is merely hosted
+ for people close to MirBSD or Debian; some is affiliated with The
+ MirOS Project.<br /><i>Debian</i> is a registered trademark owned
+ by Software in the Public Interest, Inc.<br />“The MirOS Project”
+ and “MirBSD” are unregistered trademarks owned by mirabilos.</p>
+EOF
+}
+typeset -f repo_description >/dev/null || function repo_description {
typeset suite_nick=$1
print -nr -- "WTF ${suite_nick} Repository"
}
-
-
set -A dpkgarchs -- alpha amd64 arm arm64 armel armhf hppa hurd-i386 i386 \
ia64 kfreebsd-amd64 kfreebsd-i386 m68k mips mipsel powerpc powerpcspe \
- ppc64 s390 s390x sh4 sparc sparc64 x32
+ ppc64 ppc64el s390 s390x sh4 sparc sparc64 x32
+[[ -n "${normarchs[*]}" ]] || set -A normarchs -- "${dpkgarchs[@]}"
+
+set +U
+export LC_ALL=C
+unset LANGUAGE
+typeset -Z11 -Uui16 hv
function remsign {
target=$1; shift
return $rv
}
+function die {
+ local rv=1
+
+ if [[ $1 = +([0-9]) ]]; then
+ rv=$1
+ shift
+ fi
+ print -ru2 -- "E: $*"
+ exit $rv
+}
+
+function checkedhash {
+ if [[ $1 = size ]]; then
+ REPLY=$(stat -c '%s' "$2")
+ [[ $REPLY = +([0-9]) ]] || die "Error getting size of '$2'"
+ else
+ set -o noglob
+ set -A REPLY -- $($1 "$2")
+ set +o noglob
+ [[ $REPLY = +([0-9a-f]) ]] || die "Error getting $1 of '$2'"
+ fi
+}
+
function putfile {
tee $1 | gzip -n9 >$1.gz
}
done | sort $u
}
-# DJB cdb hash (not via stdio, for speed)
-typeset -Z11 -Uui16 Lcdbhash_result
-function Lcdbhash_add {
- typeset s="$*"
- typeset -i i=0 n=${#s}
-
- while (( i < n )); do
- ((# Lcdbhash_result = (Lcdbhash_result * 33) ^ 1#${s:(i++):1} ))
- done
-}
-
# escape XHTML characters (three mandatory XML ones plus double quotes,
# the latter in an XML safe fashion numerically though)
function xhtml_escape {
-e 's\ 1"\ 1\"\ 1g'
}
-set +U
-export LC_ALL=C
-unset LANGUAGE
-saveIFS=$IFS
cd "$(dirname "$0")"
rm -f dpkg_578162_workaround
-IFS=:
+IFS=:; set -o noglob
dpkgarchl=:all:"${dpkgarchs[*]}":
-IFS=$saveIFS
+IFS=$' \t\n'; set +o noglob
suites=:
for suite in "$@"; do
allsuites=
for suite in dists/*; do
allsuites="$allsuites${allsuites:+ }${suite##*/}"
+ [[ -h $suite ]] && continue
[[ $suites = : || $suites = *:"$suite":* ]] || continue
archs=
distribution=
archs=
[[ -s $dist/distinfo.sh ]] && . $dist/distinfo.sh
set -A distarchs -- $(sortlist -u all ${archs:-$suitearchs})
- IFS=:; distarchl=:"${distarchs[*]}":; IFS=$saveIFS
+ IFS=:; set -o noglob
+ distarchl=:"${distarchs[*]}":
+ IFS=$' \t\n'; set +o noglob
nmds=0
for arch in $(sortlist -u ${distarchs[*]} ${dpkgarchs[*]}) /; do
# put "all" last
[[ $arch = / ]] && arch=all
# create index
if [[ $dpkgarchl != *:"$arch":* ]]; then
- print -u2 "Invalid arch '$arch' in $dist"
- exit 1
+ die "Invalid arch '$arch' in $dist"
elif [[ $distarchl != *:"$arch":* ]]; then
print "\n===> Linking all =>" \
"${dist#dists/}/$arch/Packages"
print "\n===> Creating" \
"${dist#dists/}/$arch/Packages\n"
mkdir -p $dist/binary-$arch
- dpkg-scanpackages $oef $om -a $arch \
- $dist $ovf | \
+ (dpkg-scanpackages $oef $om -a $arch \
+ $dist $ovf || \
+ echo $? >$dist/binary-$arch/failed) | \
putfile $dist/binary-$arch/Packages
+ [[ -e $dist/binary-$arch/failed ]] && \
+ exit $(<$dist/binary-$arch/failed)
(( nmds++ )) || firstmd=$arch
fi
done
[[ -e dpkg_578162_workaround ]] || (dpkg-scansources $oef $osf \
$dist $ovf || touch dpkg_578162_workaround) | \
putfile $dist/source/Sources
- [[ -e dpkg_578162_workaround ]] && dpkg-scansources $osf \
- $dist $ovf | \
+ [[ -e dpkg_578162_workaround ]] && (dpkg-scansources $osf \
+ $dist $ovf || echo $? >$dist/source/failed) | \
putfile $dist/source/Sources
+ [[ -e $dist/source/failed ]] && exit $(<$dist/source/failed)
+ print done.
+ print "\n===> Creating ${dist#dists/}/i18n/Index"
+ [[ -d $dist/i18n/. ]] || mkdir -p $dist/i18n
+ [[ -d $dist/i18n/. ]] || die "Cannot create $dist/i18n"
+ rm -f $dist/i18n/.done
+ (cd $dist/i18n
+ tfiles=/
+ [[ -h .hashcache || ! -f .hashcache ]] && rm -rf .hashcache
+ for ent in .* *; do
+ [[ $ent = . || $ent = .. || $ent = .hashcache ]] && continue
+ [[ -h $ent || -e $ent ]] || continue
+ if [[ ! -f $ent || $ent != Translation-* ]]; then
+ rm -rf "$ent"
+ continue
+ fi
+ ent=${ent#Translation-}
+ ent=${ent%.bz2}
+ [[ $tfiles = */"$ent"/* ]] || tfiles+=$ent/
+ done
+ [[ -e .hashcache ]] || :>.hashcache
+ if [[ $tfiles = / ]]; then
+ :>Translation-tlh_DE
+ tfiles=/tlh_DE/
+ fi
+ print SHA1: >Index
+ IFS=/; set -o noglob
+ set -A tflist -- ${tfiles#/}
+ IFS=$' \t\n'; set +o noglob
+ for ent in "${tflist[@]}"; do
+ ent=Translation-$ent
+ if [[ $ent -nt $ent.bz2 ]]; then
+ if ! bzip2 -9 <"$ent" >"$ent.bz2"; then
+ rm -f "$ent.bz2"
+ die "bzip2 '$ent' died"
+ fi
+ elif [[ -e $ent ]]; then
+ rm -f "$ent"
+ fi
+ hash=${|checkedhash sha1sum "$ent.bz2";} || exit 1
+ hnum=0
+ grep "^$hash " .hashcache |&
+ while read -p hsha1 hsize hmd5 hsha2 usha1 usize umd5 usha2; do
+ [[ $hsha1 = "$hash" ]] || continue
+ hnum=1
+ while read -p hsha1 x; do
+ # flush coprocess, look for dupes
+ [[ $hsha1 = "$hash" ]] && hnum=2
+ done
+ break
+ done
+ hsha1=$hash
+ if (( hnum != 1 )); then
+ [[ -e $ent ]] || \
+ if ! bzip2 -d <"$ent.bz2" >"$ent"; then
+ rm -f "$ent"
+ die "bzip2 '$ent.bz2' died"
+ fi
+ umd5=${|checkedhash md5sum "$ent";} || exit 1
+ hmd5=${|checkedhash md5sum "$ent.bz2";} || exit 1
+ usha1=${|checkedhash sha1sum "$ent";} || exit 1
+ usha2=${|checkedhash sha256sum "$ent";} || exit 1
+ hsha2=${|checkedhash sha256sum "$ent.bz2";} || exit 1
+ usize=${|checkedhash size "$ent";} || exit 1
+ hsize=${|checkedhash size "$ent.bz2";} || exit 1
+ (( hnum )) || print $hsha1 $hsize $hmd5 $hsha2 $usha1 $usize $umd5 $usha2 >>.hashcache
+ fi
+ [[ -e $ent ]] && rm -f "$ent"
+ print -u4 $hsha1 $hsize $hmd5 $hsha2 $usha1 $usize $umd5 $usha2
+ print -ru5 " $hsha1 $hsize $ent.bz2"
+ print -ru6 " $umd5 $usize ${dist##*/}/i18n/$ent"
+ print -ru6 " $hmd5 $hsize ${dist##*/}/i18n/$ent.bz2"
+ print -ru7 " $usha1 $usize ${dist##*/}/i18n/$ent"
+ print -ru7 " $hsha1 $hsize ${dist##*/}/i18n/$ent.bz2"
+ print -ru8 " $usha2 $usize ${dist##*/}/i18n/$ent"
+ print -ru8 " $hsha2 $hsize ${dist##*/}/i18n/$ent.bz2"
+ done 4>.hashcache.new 5>>Index 6>.hashcache.md5 7>.hashcache.sha1 8>.hashcache.sha2
+ rm -f .hashcache
+ mv -f .hashcache.new .hashcache
+ :>.done)
+ [[ -e $dist/i18n/.done ]] || die i18n generation unsuccessful
+ rm -f $dist/i18n/.done
print done.
done
print "\n===> Creating ${suite#dists/}/Release"
rm -f $suite/Release-*
+ xdone=$(realpath $suite/Release-done)
(cat <<-EOF
Origin: ${repo_origin}
Label: ${repo_label}
Suite: ${distribution:-${suite##*/}}
Codename: ${suite##*/}
- Date: $(date -u)
+ Date: $(date -Ru)
Architectures: all ${dpkgarchs[*]} source
$components
Description: $(repo_description "$nick")
set -A cache_sha1
set -A cache_sha2
set -A cache_size
- for n in Contents-* */{binary-*,source}/{Packag,Sourc}es*; do
+ for n in Contents-* */{binary-*,i18n,source}/{Index,{Packag,Sourc}es*}; do
[[ -f $n ]] || continue
# realpath-ise $n and cache the checksum
nn=$(realpath "$n")
#XXX once mksh can, use associative arrays instead
- Lcdbhash_result=5381
- Lcdbhash_add "$nn"
+ hv=16#${nn@#}
# simple hash collision solver by increment
- nc=${cache_fn[Lcdbhash_result]}
+ nc=${cache_fn[hv]}
while [[ -n $nc && $nc != "$nn" ]]; do
- nc=${cache_fn[++Lcdbhash_result]}
+ nc=${cache_fn[++hv]}
done
if [[ $nc = "$nn" ]]; then
- nm=${cache_md5[Lcdbhash_result]}
- ns=${cache_size[Lcdbhash_result]}
- nsha1=${cache_sha1[Lcdbhash_result]}
- nsha2=${cache_sha2[Lcdbhash_result]}
+ nm=${cache_md5[hv]}
+ ns=${cache_size[hv]}
+ nsha1=${cache_sha1[hv]}
+ nsha2=${cache_sha2[hv]}
else
# GNU *sum tools are horridly inefficient
- set -A x -- $(md5sum "$nn")
- nm=${x[0]}
- set -A x -- $(sha1sum "$nn")
- nsha1=${x[0]}
- set -A x -- $(sha256sum "$nn")
- nsha2=${x[0]}
- ns=$(stat -c '%s' "$nn")
- cache_md5[Lcdbhash_result]=$nm
- cache_size[Lcdbhash_result]=$ns
- cache_fn[Lcdbhash_result]=$nn
- cache_sha1[Lcdbhash_result]=$nsha1
- cache_sha2[Lcdbhash_result]=$nsha2
+ nm=${|checkedhash md5sum "$nn";} || exit 1
+ nsha1=${|checkedhash sha1sum "$nn";} || exit 1
+ nsha2=${|checkedhash sha256sum "$nn";} || exit 1
+ ns=${|checkedhash size "$nn";} || exit 1
+ cache_md5[hv]=$nm
+ cache_size[hv]=$ns
+ cache_fn[hv]=$nn
+ cache_sha1[hv]=$nsha1
+ cache_sha2[hv]=$nsha2
fi
print " $nm $ns $n"
print -u4 " $nsha1 $ns $n"
print -u5 " $nsha2 $ns $n"
- done) >$suite/Release-tmp
+ if [[ $n = */i18n/Index ]]; then
+ n=${n%Index}
+ cat "${n}.hashcache.md5"
+ cat >&4 "${n}.hashcache.sha1"
+ cat >&5 "${n}.hashcache.sha2"
+ rm -f "${n}.hashcache."*
+ fi
+ done
+ :>"$xdone") >$suite/Release-tmp
+ [[ -e $xdone ]] || die Release generation died
cat $suite/Release-sha1 $suite/Release-sha2 >>$suite/Release-tmp
- # note: InRelease files can only be safely used by wheezy
- # onwards, and oneiric onwards; known to be insecure on
- # natty, but usable concurrent to detached files on squeeze
-
- unset release_sign_detached release_sign_inline
- release_sign_detached=x
- release_sign_inline=0
+ # note: InRelease files can only be safely used by jessie and up.
+ unset use_inrelease
. $suite/distinfo.sh
- [[ $release_sign_inline = [01] ]] || release_sign_inline=0
- [[ $release_sign_detached = [01] ]] || \
- (( release_sign_detached = release_sign_inline ? 0 : 1 ))
-
- # Debian's fix for CVE-2013-1051 consists of disabling
- # support for InRelease files altogether (WTF!)
- release_sign_detached=1
- release_sign_inline=0
-
- (( release_sign_detached )) && $gpg_remote gpg -u $repo_keyid \
- -sab <$suite/Release-tmp >$suite/Release-sig
- (( release_sign_inline )) && $gpg_remote gpg -u $repo_keyid \
- --clearsign <$suite/Release-tmp >$suite/Release-inl
-
- if (( release_sign_inline )); then
+ rm -f $suite/InRelease $suite/Release $suite/Release.gpg
+ if [[ $use_inrelease = 1 ]]; then
+ $gpg_remote $gpg_bin -u $repo_keyid --no-comment --clearsign \
+ <$suite/Release-tmp >$suite/Release-inl
mv -f $suite/Release-inl $suite/InRelease
else
- rm -f $suite/InRelease
- fi
- if (( release_sign_detached )); then
+ $gpg_remote $gpg_bin -u $repo_keyid --no-comment -sab \
+ <$suite/Release-tmp >$suite/Release-sig
mv -f $suite/Release-tmp $suite/Release
mv -f $suite/Release-sig $suite/Release.gpg
- else
- rm -f $suite/Release*
fi
rm -f $suite/Release-*
done
[[ $pn = '#'* ]] && continue
if [[ $pn != +([a-z0-9_])/+([a-z0-9_-])/+([!/])/@(%|=|+([a-z0-9])) || \
$pd != +([a-z0-9_]) ]]; then
- print -u2 "Invalid lnk line '$pn' '$pd'"
+ print -u2 "W: Invalid lnk line '$pn' '$pd'"
continue
fi
preplsrc[nrpl]=$pn
fi
for suite in dists/*; do
+ [[ -h $suite ]] && continue
for dist in $suite/*; do
[[ -d $dist/. ]] || continue
suitename=${suite##*/}
if [[ $suitename != +([a-z0-9_]) ]]; then
- print -u2 "Invalid suite name '$suitename'"
+ print -u2 "W: Invalid suite name '$suitename'"
continue 2
fi
distname=${dist##*/}
if [[ $distname != +([a-z0-9_-]) ]]; then
- print -u2 "Invalid dist name '$distname'"
+ print -u2 "W: Invalid dist name '$distname'"
continue
fi
[[ -e $f ]] || continue
realpath "$f"
done | sort -u) |&
- pn=; pv=; pd=; pp=; pN=; pf=; Lf=
+ pn=; pv=; pd=; pp=; pN=; pf=; pABP=; Lf=
while IFS= read -pr line; do
case $line {
(" "*)
pf=${line##Filename:*([ ])}
Lf=pf
;;
+ ("Auto-Built-Package: "*)
+ pABP=${line##Auto-Built-Package:*([ ])}
+ Lf=pABP
+ ;;
(?*) # anything else
Lf=
;;
(*) # empty line
- if [[ $pf = *:* || $pf = *'%'* ]]; then
- print -u2 Illegal character in $dist \
- packages $pp "'Filename: $pf'"
- exit 1
- fi
+ [[ $pf = *:* || $pf = *'%'* ]] && \
+ die Illegal character in $dist \
+ packages $pp "'Filename: $pf'"
[[ -n $pn ]] || pn=$pN
- if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
+ if [[ $pN = *-dbgsym && $pABP = debug-symbols ]]; then
+ : skip
+ elif [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
i=0
while (( i < nbin )); do
[[ ${bp_disp[i]} = "$pN" && ${bp_desc[i]} = "$pd" && \
eval bp_ver_${suitename}[i]=\$x
bp_desc[i]=$pd
fi
- pn=; pv=; pd=; pp=; pN=; pf=; Lf=
+ pn=; pv=; pd=; pp=; pN=; pf=; pABP=; Lf=
;;
}
done
<meta name="MSSmartTagsPreventParsing" content="TRUE" />
EOF
print -r -- " <title>${repo_title} Index</title>"
+[[ -s NEWS.rss ]] && print '<link rel="alternate" type="application/rss+xml" title="RSS" href="NEWS.rss" />'
cat <<'EOF'
- <meta name="generator" content="$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.61 2014/05/23 09:44:30 tg Exp $" />
<style type="text/css"><!--/*--><![CDATA[/*><!--*/
table {
border: 1px solid black;
</head><body>
EOF
print -r -- "<h1>${repo_title}</h1>"
+repo_intro
cat <<'EOF'
<p><a href="dists/">Browse</a> the repository or read about how to amend <a
href="sources.txt">/etc/apt/sources.list</a> in order to use it.
EOF
[[ -s 0-NOTE.txt ]] && print ' Also read my <a href="0-NOTE.txt">notes</a>.'
+[[ -s NEWS.rss ]] && print ' There is an <a href="NEWS.rss">RSS newsfeed</a>.'
cat <<EOF
This repository uses <a
href="http://pgp.uni-mainz.de:11371/pks/lookup?search=${repo_keyid}&op=vindex">${repo_keyid}</a>
for suitename in $allsuites; do
suite=dists/$suitename
+ if [[ -h $suite ]]; then
+ ent=$(realpath "$suite")
+ n=$(realpath dists)
+ [[ $ent = "$n"/+([!/]) ]] || continue
+ ent=${ent#"$n"/}
+ for n in $allsuites; do
+ [[ $n = "$ent" ]] && break
+ done
+ [[ $n = "$ent" ]] && print -r \
+ " <li>$suitename: symbolic link to $ent</li>"
+ continue
+ fi
. $suite/distinfo.sh
print -n " <li>${suite##*/}: <a href=\"$suite/\">$desc</a> (dists:"
for dist in $suite/*; do
print " <th class=\"tableheadcell\">dist</th>"
print " <th class=\"tableheadcell\" rowspan=\"2\">Binary / Description</th>"
for suitename in $allsuites; do
+ [[ -h dists/$suitename ]] && continue
print " <th class=\"tableheadcell\" rowspan=\"2\">$suitename</th>"
done
print "</tr><tr class=\"tablehead\">"
done
print " <td rowspan=\"2\" class=\"srcpkgdesc\">${pd#, }</td>"
for suitename in $allsuites; do
+ [[ -h dists/$suitename ]] && continue
eval pvo=\${sp_ver_${suitename}[num]}
eval ppo=\${sp_dir_${suitename}[num]}
- IFS=,
+ IFS=,; set -o noglob
set -A pva -- $pvo
set -A ppa -- $ppo
- IFS=$saveIFS
+ IFS=$' \t\n'; set +o noglob
(( ${#pva[*]} )) || pva[0]=
y=
i=0
print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
print " <td class=\"binpkgdesc\">$(xhtml_escape "${bp_desc[i]}")</td>"
for suitename in $allsuites; do
+ [[ -h dists/$suitename ]] && continue
eval pv=\${bp_ver_${suitename}[i]}
if [[ -z $pv ]]; then
pv=-
print " <td class=\"srcpkgdesc\">binary" \
"packages without a matching source package</td>"
for suitename in $allsuites; do
+ [[ -h dists/$suitename ]] && continue
print " <td class=\"srcpkgitem\">-</td>"
done
print "</tr>"
print " <td class=\"binpkgdist\">${bp_dist[i]}</td>"
print " <td rowspan=\"2\" class=\"binpkgdesc\">$(xhtml_escape "${bp_desc[i]}")</td>"
for suitename in $allsuites; do
+ [[ -h dists/$suitename ]] && continue
eval pv=\${bp_ver_${suitename}[i]}
if [[ -z $pv ]]; then
pv=-
projects / shellsnippets / shellsnippets.git / blobdiff