Automatic APT Repository build infrastructure
[shellsnippets/shellsnippets.git] / mksh / sysadmin / mvndebri.sh
1 #!/bin/mksh
2 rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.51 2011/05/13 20:53:29 tg Exp $'
3 rcsid='$Id: mvndebri.sh 2080 2011-06-09 13:21:50Z tglase $'
4 #-
5 # Copyright (c) 2008, 2009, 2010, 2011
6 #       Thorsten Glaser <tg@mirbsd.org>
7 # Copyright (c) 2011
8 #       Thorsten Glaser <t.glaser@tarent.de>
9 #
10 # Provided that these terms and disclaimer and all copyright notices
11 # are retained or reproduced in an accompanying document, permission
12 # is granted to deal in this work without restriction, including un-
13 # limited rights to use, publicly perform, distribute, sell, modify,
14 # merge, give away, or sublicence.
15 #
16 # This work is provided "AS IS" and WITHOUT WARRANTY of any kind, to
17 # the utmost extent permitted by applicable law, neither express nor
18 # implied; without malicious intent or gross negligence. In no event
19 # may a licensor, author or contributor be held liable for indirect,
20 # direct, other damage, loss, or other issues arising in any way out
21 # of dealing in the work, even if advised of the possibility of such
22 # damage or existence of a defect, except proven that it results out
23 # of said person's immediate fault when using the work as intended.
24
25 if [[ $1 != /* || -z $2 || ! -d $1/$2/. ]]; then
26         print -u2 Syntax error.
27         exit 1
28 fi
29 jobname=$2
30
31 hn=$(hostname)
32 case $hn {
33 (dev-hudson?(.*))
34         hn=dev-hudson
35         repo_keyid=0x199D2F3B
36         ;;
37 (test-hudson?(.*))
38         hn=test-hudson
39         repo_keyid=0xDB58AF73
40         ;;
41 # […]
42 (tglase.bonn.tarent.de)
43         hn=devel-testsystem
44         repo_keyid=0x5EB8D3B3
45         ;;
46 (*)
47         print -u2 "Cannot determine hostname from '$hn'"
48         exit 1
49         ;;
50 }
51 repo_origin='tarent solutions GmbH'
52 repo_label=tarent_auto_${hn}_$jobname
53 repo_title="tarent $hn $jobname automatic DEB Repository"
54 function repo_description {
55         typeset suite_nick=$1
56
57         print -nr -- "tarent-$hn $jobname/$suite_nick autobuild repository"
58 }
59
60
61 #set -A dpkgarchs -- alpha amd64 arm armeb armel armhf avr32 hppa \
62 #    i386 ia64 kfreebsd-amd64 kfreebsd-i386 lpia m32r m68k mips mipsel \
63 #    powerpc powerpcspe ppc64 s390 s390x sh3 sh3eb sh4 sh4eb sparc
64 set -A dpkgarchs -- amd64 i386
65
66 function putfile {
67         tee $1 | gzip -n9 >$1.gz
68 }
69
70 function sortlist {
71         typeset x u=$1
72
73         if [[ $u = -u ]]; then
74                 shift
75         else
76                 u=
77         fi
78
79         for x in "$@"; do
80                 print -r -- "$x"
81         done | sort $u
82 }
83
84 set +U
85 export LC_ALL=C
86 unset LANGUAGE
87 saveIFS=$IFS
88 cd "$1/$2"
89 shift
90 shift
91 if [[ ! -d dists/. ]]; then
92         print -u2 Not an APT repository.
93         exit 1
94 fi
95
96 suites=:
97 for suite in "$@"; do
98         suites=:dists/$suite$suites
99 done
100
101 allsuites=
102 for suite in dists/*; do
103         allsuites="$allsuites${allsuites:+ }${suite##*/}"
104         [[ $suites = : || $suites = *:"$suite":* ]] || continue
105         components=Components:
106         for dist in $suite/*; do
107                 [[ -d $dist/. ]] || continue
108                 rm -rf $dist/binary-* $dist/source
109                 ovf= oef= osf=
110                 [[ -s $dist/override.file ]] && ovf=$dist/override.file
111                 [[ -s $dist/override.extra ]] && oef="-e $dist/override.extra"
112                 [[ -s $dist/override.src ]] && osf="-s $dist/override.src"
113                 components="$components ${dist##*/}"
114                 for arch in ${dpkgarchs[*]} all; do
115                                 print "\n===> Creating" \
116                                     "${dist#dists/}/$arch/Packages\n"
117                                 mkdir -p $dist/binary-$arch
118                                 dpkg-scanpackages $oef -m -a $arch \
119                                     $dist $ovf | \
120                                     putfile $dist/binary-$arch/Packages
121                 done
122                 print "\n===> Creating ${dist#dists/}/Sources"
123                 mkdir -p $dist/source
124                 dpkg-scansources $osf \
125                     $dist $ovf | \
126                     putfile $dist/source/Sources
127                 print done.
128         done
129         print "\n===> Creating ${suite#dists/}/Release.gpg"
130         rm -f $suite/Release*
131         (cat <<-EOF
132                 Origin: ${repo_origin}
133                 Label: ${repo_label}
134                 Suite: ${suite##*/}
135                 Codename: ${suite##*/}
136                 Date: $(date -u)
137                 Architectures: all ${dpkgarchs[*]} source
138                 $components
139                 Description: $(repo_description "${suite##*/}")
140                 MD5Sum:
141         EOF
142         exec 4>$suite/Release-sha1
143         exec 5>$suite/Release-sha2
144         print -u4 SHA1:
145         print -u5 SHA256:
146         cd $suite
147         for n in Contents-* */{binary-*,source}/{Packag,Sourc}es*; do
148                 [[ -f $n ]] || continue
149                 # GNU *sum tools are horridly inefficient
150                 set -A x -- $(md5sum "$n")
151                 nm=${x[0]}
152                 set -A x -- $(sha1sum "$n")
153                 nsha1=${x[0]}
154                 set -A x -- $(sha256sum "$n")
155                 nsha2=${x[0]}
156                 ns=$(stat -c '%s' "$n")
157                 print " $nm $ns $n"
158                 print -u4 " $nsha1 $ns $n"
159                 print -u5 " $nsha2 $ns $n"
160         done) >$suite/Release
161         cat $suite/Release-sha1 $suite/Release-sha2 >>$suite/Release
162         rm $suite/Release-sha1 $suite/Release-sha2
163         gpg -u $repo_keyid -sb --batch \
164             --passphrase-file /etc/tarent/maven.kpw \
165             <$suite/Release >$suite/Release.gpg
166 done
167
168 print "\n===> Creating debidx.htm\n"
169
170 integer nsrc=0 nbin=0
171 br='<br />'
172
173 for suite in dists/*; do
174         for dist in $suite/*; do
175                 [[ -d $dist/. ]] || continue
176                 suitename=${suite##*/}
177                 if [[ $suitename != +([a-z0-9_]) ]]; then
178                         print -u2 "Invalid suite name '$suitename'"
179                         continue 2
180                 fi
181                 distname=${dist##*/}
182                 if [[ $distname != +([a-z0-9_-]) ]]; then
183                         print -u2 "Invalid dist name '$distname'"
184                         continue
185                 fi
186
187                 gzip -dc $dist/source/Sources.gz |&
188                 pn=; pv=; pd=; pp=; Lf=
189                 while IFS= read -pr line; do
190                         case $line {
191                         (" "*)
192                                 if [[ -n $Lf ]]; then
193                                         eval x=\$$Lf
194                                         x=$x$line
195                                         eval $Lf=\$x
196                                 fi
197                                 ;;
198                         ("Package: "*)
199                                 pn=${line##Package:*([   ])}
200                                 Lf=pn
201                                 ;;
202                         ("Version: "*)
203                                 pv=${line##Version:*([   ])}
204                                 Lf=pv
205                                 ;;
206                         ("Binary: "*)
207                                 pd=${line##Binary:*([    ])}
208                                 Lf=pd
209                                 ;;
210                         ("Directory: "*)
211                                 pp=${line##Directory:*([         ])}
212                                 Lf=pp
213                                 ;;
214                         (?*)    # anything else
215                                 Lf=
216                                 ;;
217                         (*)     # empty line
218                                 if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
219                                         i=0
220                                         while (( i < nsrc )); do
221                                                 [[ ${sp_name[i]} = "$pn" && \
222                                                     ${sp_dist[i]} = "$distname" ]] && break
223                                                 let i++
224                                         done
225                                         if (( i == nsrc )); then
226                                                 let nsrc++
227                                                 pvo=
228                                                 ppo=
229                                         else
230                                                 eval pvo=\$\{sp_ver_${suitename}[i]\}
231                                                 eval ppo=\$\{sp_dir_${suitename}[i]\}
232                                         fi
233                                         sp_name[i]=$pn
234                                         sp_dist[i]=$distname
235                                         #sp_suites[i]="${sp_suites[i]} $suitename"
236                                         eval sp_ver_${suitename}[i]='${pvo:+$pvo,}$pv'
237                                         eval sp_dir_${suitename}[i]='${ppo:+$ppo,}$pp/'
238                                         sp_desc[i]=${sp_desc[i]},$pd
239                                 fi
240                                 pn=; pv=; pd=; pp=; Lf=
241                                 ;;
242                         }
243                 done
244
245                 gzip -dc $(for f in $dist/binary-*/Packages.gz; do
246                         [[ -e $f ]] || continue
247                         realpath "$f"
248                 done | sort -u) |&
249                 pn=; pv=; pd=; pp=; pN=; pf=; Lf=
250                 while IFS= read -pr line; do
251                         case $line {
252                         (" "*)
253                                 if [[ -n $Lf ]]; then
254                                         eval x=\$$Lf
255                                         x=$x$line
256                                         eval $Lf=\$x
257                                 fi
258                                 ;;
259                         ("Package: "*)
260                                 pN=${line##Package:*([   ])}
261                                 Lf=pN
262                                 ;;
263                         ("Source: "*)
264                                 pn=${line##Source:*([    ])}
265                                 pn=${pn%% *}
266                                 Lf=pn
267                                 ;;
268                         ("Version: "*)
269                                 pv=${line##Version:*([   ])}
270                                 Lf=pv
271                                 ;;
272                         ("Description: "*)
273                                 pd=${line##Description:*([       ])}
274                                 ;;
275                         ("Architecture: "*)
276                                 pp=${line##Architecture:*([      ])}
277                                 Lf=pp
278                                 ;;
279                         ("Filename: "*)
280                                 pf=${line##Filename:*([  ])}
281                                 Lf=pf
282                                 ;;
283                         (?*)    # anything else
284                                 Lf=
285                                 ;;
286                         (*)     # empty line
287                                 if [[ $pf = *:* || $pf = *'%'* ]]; then
288                                         print -u2 Illegal character in $dist \
289                                             packages $pp "'Filename: $pf'"
290                                         exit 1
291                                 fi
292                                 [[ -n $pn ]] || pn=$pN
293                                 if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
294                                         i=0
295                                         while (( i < nbin )); do
296                                                 [[ ${bp_disp[i]} = "$pN" && ${bp_desc[i]} = "$pd" && \
297                                                     ${bp_dist[i]} = "$distname" ]] && break
298                                                 let i++
299                                         done
300                                         (( i == nbin )) && let nbin++
301                                         bp_name[i]=$pn
302                                         bp_disp[i]=$pN
303                                         bp_dist[i]=$distname
304                                         #bp_suites[i]="${bp_suites[i]} $suitename"
305                                         [[ -n $pf ]] && pv="<a href=\"$pf\">$pv</a>"
306                                         pv="$pp: $pv"
307                                         eval x=\${bp_ver_${suitename}[i]}
308                                         [[ $br$x$br = *"$br$pv$br"* ]] || x=$x${x:+$br}$pv
309                                         eval bp_ver_${suitename}[i]=\$x
310                                         bp_desc[i]=$pd
311                                 fi
312                                 pn=; pv=; pd=; pp=; pN=; pf=; Lf=
313                                 ;;
314                         }
315                 done
316         done
317 done
318
319 (cat <<'EOF'
320 <?xml version="1.0"?>\r
321 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
322  "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
323 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head>
324  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
325  <meta name="MSSmartTagsPreventParsing" content="TRUE" />
326 EOF
327 print -r -- " <title>${repo_title} Index</title>"
328 cat <<'EOF'
329  <meta name="generator" content="$Id: mvndebri.sh 2080 2011-06-09 13:21:50Z tglase $ based on $MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.51 2011/05/13 20:53:29 tg Exp $" />
330  <style type="text/css">
331   table {
332    border: 1px solid black;
333    border-collapse: collapse;
334    text-align: left;
335    vertical-align: top;
336   }
337   tr {
338    border: 1px solid black;
339    text-align: left;
340    vertical-align: top;
341   }
342   td {
343    border: 1px solid black;
344    text-align: left;
345    vertical-align: top;
346   }
347   th {
348    background-color: #000000;
349    color: #FFFFFF;
350   }
351   .tableheadcell {
352    border: 1px solid #999999;
353    padding: 3px;
354    white-space: nowrap;
355   }
356   .srcpkgline {
357    background-color: #CCCCCC;
358   }
359   .srcpkgdist {
360    background-color: #666666;
361    color: #FFFFFF;
362    font-weight: bold;
363   }
364   .binpkgdist {
365    background-color: #999999;
366    color: #FFFFFF;
367    font-weight: bold;
368   }
369  </style>
370 </head><body>
371 EOF
372 print -r -- "<h1>${repo_title}</h1>"
373 cat <<EOF
374 <p><a href="dists/">Browse</a> the repository.
375  This repository uses <a
376   href="http://pgp.uni-mainz.de:11371/pks/lookup?search=${repo_keyid}&amp;op=vindex">${repo_keyid}</a>
377  as signing key.
378 </p>
379 <h2>Suites</h2>
380 <ul>
381 EOF
382
383 allsuites=$(for suitename in $allsuites; do
384         print $suitename
385 done | sort -u)
386
387 for suitename in $allsuites; do
388         suite=dists/$suitename
389         print -n " <li>${suite##*/}: <a href=\"$suite/\">$jobname for $suitename</a> (dists:"
390         vdists=
391         for dist in $suite/*; do
392                 [[ -d $dist/. ]] || continue
393                 distname=${dist##*/}
394                 vdists="$vdists $distname"
395                 print -n " <a href=\"$suite/$distname/\">$distname</a>"
396         done
397         print ")<br />"
398         print " <tt>deb http://${hn}-debs.bonn.tarent.de/$jobname $suitename$vdists</tt>"
399         print "</li>"
400 done
401 print "</ul>"
402 print "<h2>Packages</h2>"
403 print "<table width=\"100%\"><thead>"
404 print "<tr class=\"tablehead\">"
405 print " <th class=\"tableheadcell\">dist</th>"
406 print " <th class=\"tableheadcell\" rowspan=\"2\">Binary / Description</th>"
407 for suitename in $allsuites; do
408         print " <th class=\"tableheadcell\" rowspan=\"2\">$suitename</th>"
409 done
410 print "</tr><tr class=\"tablehead\">"
411 print " <th class=\"tableheadcell\">package name</th>"
412 print "</tr></thead><tbody>"
413
414 set -A bp_sort
415 i=0
416 while (( i < nbin )); do
417         print $i ${bp_disp[i++]} #${bp_suites[i]}
418 done | sort -k2 |&
419 while read -p num rest; do
420         bp_sort[${#bp_sort[*]}]=$num
421 done
422
423 i=0
424 while (( i < nsrc )); do
425         print $i ${sp_name[i++]}
426 done | sort -k2 |&
427 while read -p num rest; do
428         print "\n<!-- sp #$num = ${sp_name[num]} -->"
429         print "<tr class=\"srcpkgline\">"
430         print " <td class=\"srcpkgdist\">${sp_dist[num]}</td>"
431         pd=
432         for x in $(tr ', ' '\n' <<<"${sp_desc[num]}" | sort -u); do
433                 [[ -n $x ]] && pd="$pd, $x"
434         done
435         print " <td rowspan=\"2\" class=\"srcpkgdesc\">${pd#, }</td>"
436         for suitename in $allsuites; do
437                 eval pvo=\${sp_ver_${suitename}[num]}
438                 eval ppo=\${sp_dir_${suitename}[num]}
439                 IFS=,
440                 set -A pva -- $pvo
441                 set -A ppa -- $ppo
442                 IFS=$saveIFS
443                 (( ${#pva[*]} )) || pva[0]=
444                 y=
445                 i=0
446                 while (( i < ${#pva[*]} )); do
447                         pv=${pva[i]}
448                         pp=${ppa[i]}
449                         if [[ $pv = *"\1a"* ]]; then
450                                 pvdsc=${pv%%"\1a"*}
451                                 pv=${pv##*"\1a"}
452                         else
453                                 pvdsc=$pv
454                         fi
455                         if [[ -z $pv ]]; then
456                                 pv=-
457                         elif [[ $pp != ?(/) ]]; then
458                                 pv="<a href=\"$pp${sp_name[num]}_${pvdsc##+([0-9]):}.dsc\">$pv</a>"
459                         fi
460                         [[ $pp != ?(/) ]] && pv="<a href=\"$pp\">[dir]</a> $pv"
461                         y=${y:+"$y<br />"}$pv
462                         let i++
463                 done
464                 print " <td rowspan=\"2\" class=\"srcpkgitem\">$y</td>"
465         done
466         print "</tr><tr class=\"srcpkgline\">"
467         print " <td class=\"srcpkgname\">${sp_name[num]}</td>"
468         print "</tr>"
469         k=0
470         while (( k < nbin )); do
471                 (( (i = bp_sort[k++]) < 0 )) && continue
472                 [[ ${bp_name[i]} = "${sp_name[num]}" && \
473                     ${bp_dist[i]} = "${sp_dist[num]}" ]] || continue
474                 bp_sort[k - 1]=-1
475                 #print "<!-- bp #$i for${bp_suites[i]} -->"
476                 print "<!-- bp #$i -->"
477                 print "<tr class=\"binpkgline\">"
478                 print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
479                 print " <td class=\"binpkgdesc\">${bp_desc[i]}</td>"
480                 for suitename in $allsuites; do
481                         eval pv=\${bp_ver_${suitename}[i]}
482                         if [[ -z $pv ]]; then
483                                 pv=-
484                         fi
485                         print " <td class=\"binpkgitem\">$pv</td>"
486                 done
487                 print "</tr>"
488         done
489 done
490
491 num=0
492 for i in ${bp_sort[*]}; do
493         (( i < 0 )) && continue
494         if (( !num )); then
495                 print "\n<!-- sp ENOENT -->"
496                 print "<tr class=\"srcpkgline\">"
497                 print " <td class=\"srcpkgname\">~ENOENT~</td>"
498                 print " <td class=\"srcpkgdesc\">binary" \
499                     "packages without a matching source package</td>"
500                 for suitename in $allsuites; do
501                         print " <td class=\"srcpkgitem\">-</td>"
502                 done
503                 print "</tr>"
504                 num=1
505         fi
506         #print "<!-- bp #$i for${bp_suites[i]} -->"
507         print "<!-- bp #$i -->"
508         print "<tr class=\"binpkgline\">"
509         print " <td class=\"binpkgdist\">${bp_dist[i]}</td>"
510         print " <td rowspan=\"2\" class=\"binpkgdesc\">${bp_desc[i]}</td>"
511         for suitename in $allsuites; do
512                 eval pv=\${bp_ver_${suitename}[i]}
513                 if [[ -z $pv ]]; then
514                         pv=-
515                 fi
516                 print " <td rowspan=\"2\" class=\"binpkgitem\">$pv</td>"
517         done
518         print "</tr><tr class=\"binpkgline\">"
519         print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
520         print "</tr>"
521 done
522
523 cat <<EOF
524
525 </tbody></table>
526
527 <p>• <a href="http://validator.w3.org/check/referer">Valid XHTML/1.1!</a>
528  • <small>Generated on $(date -u +'%F %T') by <tt
529  style="white-space:pre;">$rcsid</tt></small> •</p>
530 </body></html>
531 EOF
532
533 :) >debidx.htm
534 print done.