sync with MirBSD and tarent: proper CDATA escapes for JS and CSS
[shellsnippets/shellsnippets.git] / mksh / sysadmin / mvndebri.sh
1 #!/bin/mksh
2 rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.61 2014/05/23 09:44:30 tg Exp $'
3 rcsid='$Id: mvndebri.sh 4044 2014-05-23 09:48:47Z tglase $'
4 #-
5 # Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014
6 #       Thorsten Glaser <tg@mirbsd.org>
7 # Copyright (c) 2011, 2014
8 #       Thorsten Glaser <t.glaser@tarent.de>
9 #
10 # Provided that these terms and disclaimer and all copyright notices
11 # are retained or reproduced in an accompanying document, permission
12 # is granted to deal in this work without restriction, including un-
13 # limited rights to use, publicly perform, distribute, sell, modify,
14 # merge, give away, or sublicence.
15 #
16 # This work is provided "AS IS" and WITHOUT WARRANTY of any kind, to
17 # the utmost extent permitted by applicable law, neither express nor
18 # implied; without malicious intent or gross negligence. In no event
19 # may a licensor, author or contributor be held liable for indirect,
20 # direct, other damage, loss, or other issues arising in any way out
21 # of dealing in the work, even if advised of the possibility of such
22 # damage or existence of a defect, except proven that it results out
23 # of said person's immediate fault when using the work as intended.
24
25 if [[ $1 != /* || -z $2 || ! -d $1/$2/. ]]; then
26         print -u2 Syntax error.
27         exit 1
28 fi
29 jobname=$2
30
31 hn=$(hostname)
32 case $hn {
33 (ci-evolvis?(.*))
34         hn=ci-evolvis
35         repo_keyid=0x16B5D1B2
36         ;;
37 #[…]
38 #(test-hudson?(.*))
39 #       hn=test-hudson
40 #       repo_keyid=0xDB58AF73
41 #       ;;
42 (tglase.bonn.tarent.de)
43         hn=devel-testsystem
44         repo_keyid=0x5EB8D3B3
45         ;;
46 (*)
47         print -u2 "Cannot determine hostname from '$hn'"
48         exit 1
49         ;;
50 }
51 case $hn {
52 (test-hudson|devel-testsystem)
53         print -u2 moved
54         exit 1
55         dhn=jenkins-debs.evolvis.org
56         ;;
57 (*)
58         dhn=${hn}-debs.lan.tarent.de
59         ;;
60 }
61 repo_origin='tarent solutions GmbH'
62 repo_label=tarent_auto_${hn}_$jobname
63 repo_title="tarent $hn $jobname automatic DEB Repository"
64 function repo_description {
65         typeset suite_nick=$1
66
67         print -nr -- "tarent $hn $jobname/$suite_nick autobuild repository"
68 }
69
70
71 #set -A dpkgarchs -- alpha amd64 arm arm64 armel armhf hppa hurd-i386 i386 \
72 #    ia64 kfreebsd-amd64 kfreebsd-i386 m68k mips mipsel powerpc powerpcspe \
73 #    ppc64 s390 s390x sh4 sparc sparc64 x32
74 set -A dpkgarchs -- amd64 i386
75
76 function putfile {
77         tee $1 | gzip -n9 >$1.gz
78 }
79
80 function sortlist {
81         typeset x u=$1
82
83         if [[ $u = -u ]]; then
84                 shift
85         else
86                 u=
87         fi
88
89         for x in "$@"; do
90                 print -r -- "$x"
91         done | sort $u
92 }
93
94 # escape XHTML characters (three mandatory XML ones plus double quotes,
95 # the latter in an XML safe fashion numerically though)
96 function xhtml_escape {
97         if (( $# )); then
98                 print -nr -- "$@"
99         else
100                 cat
101         fi | sed \
102             -e 's\ 1&\ 1\&amp;\ 1g' \
103             -e 's\ 1<\ 1\&lt;\ 1g' \
104             -e 's\ 1>\ 1\&gt;\ 1g' \
105             -e 's\ 1"\ 1\&#34;\ 1g'
106 }
107
108 set +U
109 export LC_ALL=C
110 unset LANGUAGE
111 saveIFS=$IFS
112 cd "$1/$2"
113 shift
114 shift
115 if [[ ! -d dists/. ]]; then
116         print -u2 Not an APT repository.
117         exit 1
118 fi
119
120 suites=:
121 for suite in "$@"; do
122         suites=:dists/$suite$suites
123 done
124
125 allsuites=
126 for suite in dists/*; do
127         allsuites="$allsuites${allsuites:+ }${suite##*/}"
128         [[ $suites = : || $suites = *:"$suite":* ]] || continue
129         components=Components:
130         for dist in $suite/*; do
131                 [[ -d $dist/. ]] || continue
132                 rm -rf $dist/binary-* $dist/source
133                 ovf= oef= osf=
134                 [[ -s $dist/override.file ]] && ovf=$dist/override.file
135                 [[ -s $dist/override.extra ]] && oef="-e $dist/override.extra"
136                 [[ -s $dist/override.src ]] && osf="-s $dist/override.src"
137                 components="$components ${dist##*/}"
138                 for arch in ${dpkgarchs[*]} all; do
139                                 print "\n===> Creating" \
140                                     "${dist#dists/}/$arch/Packages\n"
141                                 mkdir -p $dist/binary-$arch
142                                 dpkg-scanpackages $oef -m -a $arch \
143                                     $dist $ovf | \
144                                     putfile $dist/binary-$arch/Packages
145                 done
146                 print "\n===> Creating ${dist#dists/}/Sources"
147                 mkdir -p $dist/source
148                 dpkg-scansources $osf \
149                     $dist $ovf | \
150                     putfile $dist/source/Sources
151                 print done.
152         done
153         print "\n===> Creating ${suite#dists/}/Release"
154         rm -f $suite/Release-*
155         (cat <<-EOF
156                 Origin: ${repo_origin}
157                 Label: ${repo_label}
158                 Suite: ${suite##*/}
159                 Codename: ${suite##*/}
160                 Date: $(date -u)
161                 Architectures: all ${dpkgarchs[*]} source
162                 $components
163                 Description: $(repo_description "${suite##*/}")
164                 MD5Sum:
165         EOF
166         exec 4>$suite/Release-sha1
167         exec 5>$suite/Release-sha2
168         print -u4 SHA1:
169         print -u5 SHA256:
170         cd $suite
171         for n in Contents-* */{binary-*,source}/{Packag,Sourc}es*; do
172                 [[ -f $n ]] || continue
173                 # GNU *sum tools are horridly inefficient
174                 set -A x -- $(md5sum "$n")
175                 nm=${x[0]}
176                 set -A x -- $(sha1sum "$n")
177                 nsha1=${x[0]}
178                 set -A x -- $(sha256sum "$n")
179                 nsha2=${x[0]}
180                 ns=$(stat -c '%s' "$n")
181                 print " $nm $ns $n"
182                 print -u4 " $nsha1 $ns $n"
183                 print -u5 " $nsha2 $ns $n"
184         done) >$suite/Release-tmp
185         cat $suite/Release-sha1 $suite/Release-sha2 >>$suite/Release-tmp
186
187         gpg -u $repo_keyid --batch --passphrase-file /etc/tarent/maven.kpw \
188             -sab <$suite/Release-tmp >$suite/Release-sig
189         # Debian's fix for CVE-2013-1051 consists of disabling
190         # support for InRelease files altogether (WTF!)
191         #gpg -u $repo_keyid --batch --passphrase-file /etc/tarent/maven.kpw \
192         #    --clearsign <$suite/Release-tmp >$suite/Release-inl
193
194         rm -f $suite/InRelease
195         #mv -f $suite/Release-inl $suite/InRelease
196         mv -f $suite/Release-tmp $suite/Release
197         mv -f $suite/Release-sig $suite/Release.gpg
198         rm -f $suite/Release-*
199 done
200
201 print "\n===> Creating debidx.htm\n"
202
203 integer nsrc=0 nbin=0
204 br='<br />'
205
206 for suite in dists/*; do
207         for dist in $suite/*; do
208                 [[ -d $dist/. ]] || continue
209                 suitename=${suite##*/}
210                 if [[ $suitename != +([a-z0-9_]) ]]; then
211                         print -u2 "Invalid suite name '$suitename'"
212                         continue 2
213                 fi
214                 distname=${dist##*/}
215                 if [[ $distname != +([a-z0-9_-]) ]]; then
216                         print -u2 "Invalid dist name '$distname'"
217                         continue
218                 fi
219
220                 gzip -dc $dist/source/Sources.gz |&
221                 pn=; pv=; pd=; pp=; Lf=
222                 while IFS= read -pr line; do
223                         case $line {
224                         (" "*)
225                                 if [[ -n $Lf ]]; then
226                                         eval x=\$$Lf
227                                         x=$x$line
228                                         eval $Lf=\$x
229                                 fi
230                                 ;;
231                         ("Package: "*)
232                                 pn=${line##Package:*([   ])}
233                                 Lf=pn
234                                 ;;
235                         ("Version: "*)
236                                 pv=${line##Version:*([   ])}
237                                 Lf=pv
238                                 ;;
239                         ("Binary: "*)
240                                 pd=${line##Binary:*([    ])}
241                                 Lf=pd
242                                 ;;
243                         ("Directory: "*)
244                                 pp=${line##Directory:*([         ])}
245                                 Lf=pp
246                                 ;;
247                         (?*)    # anything else
248                                 Lf=
249                                 ;;
250                         (*)     # empty line
251                                 if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
252                                         i=0
253                                         while (( i < nsrc )); do
254                                                 [[ ${sp_name[i]} = "$pn" && \
255                                                     ${sp_dist[i]} = "$distname" ]] && break
256                                                 let i++
257                                         done
258                                         if (( i == nsrc )); then
259                                                 let nsrc++
260                                                 pvo=
261                                                 ppo=
262                                         else
263                                                 eval pvo=\$\{sp_ver_${suitename}[i]\}
264                                                 eval ppo=\$\{sp_dir_${suitename}[i]\}
265                                         fi
266                                         sp_name[i]=$pn
267                                         sp_dist[i]=$distname
268                                         #sp_suites[i]="${sp_suites[i]} $suitename"
269                                         eval sp_ver_${suitename}[i]='${pvo:+$pvo,}$pv'
270                                         eval sp_dir_${suitename}[i]='${ppo:+$ppo,}$pp/'
271                                         sp_desc[i]=${sp_desc[i]},$pd
272                                 fi
273                                 pn=; pv=; pd=; pp=; Lf=
274                                 ;;
275                         }
276                 done
277
278                 gzip -dc $(for f in $dist/binary-*/Packages.gz; do
279                         [[ -e $f ]] || continue
280                         realpath "$f"
281                 done | sort -u) |&
282                 pn=; pv=; pd=; pp=; pN=; pf=; Lf=
283                 while IFS= read -pr line; do
284                         case $line {
285                         (" "*)
286                                 if [[ -n $Lf ]]; then
287                                         eval x=\$$Lf
288                                         x=$x$line
289                                         eval $Lf=\$x
290                                 fi
291                                 ;;
292                         ("Package: "*)
293                                 pN=${line##Package:*([   ])}
294                                 Lf=pN
295                                 ;;
296                         ("Source: "*)
297                                 pn=${line##Source:*([    ])}
298                                 pn=${pn%% *}
299                                 Lf=pn
300                                 ;;
301                         ("Version: "*)
302                                 pv=${line##Version:*([   ])}
303                                 Lf=pv
304                                 ;;
305                         ("Description: "*)
306                                 pd=${line##Description:*([       ])}
307                                 ;;
308                         ("Architecture: "*)
309                                 pp=${line##Architecture:*([      ])}
310                                 Lf=pp
311                                 ;;
312                         ("Filename: "*)
313                                 pf=${line##Filename:*([  ])}
314                                 Lf=pf
315                                 ;;
316                         (?*)    # anything else
317                                 Lf=
318                                 ;;
319                         (*)     # empty line
320                                 if [[ $pf = *:* || $pf = *'%'* ]]; then
321                                         print -u2 Illegal character in $dist \
322                                             packages $pp "'Filename: $pf'"
323                                         exit 1
324                                 fi
325                                 [[ -n $pn ]] || pn=$pN
326                                 if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
327                                         i=0
328                                         while (( i < nbin )); do
329                                                 [[ ${bp_disp[i]} = "$pN" && ${bp_desc[i]} = "$pd" && \
330                                                     ${bp_dist[i]} = "$distname" ]] && break
331                                                 let i++
332                                         done
333                                         (( i == nbin )) && let nbin++
334                                         bp_name[i]=$pn
335                                         bp_disp[i]=$pN
336                                         bp_dist[i]=$distname
337                                         #bp_suites[i]="${bp_suites[i]} $suitename"
338                                         [[ -n $pf ]] && pv="<a href=\"$pf\">$pv</a>"
339                                         pv="$pp: $pv"
340                                         eval x=\${bp_ver_${suitename}[i]}
341                                         [[ $br$x$br = *"$br$pv$br"* ]] || x=$x${x:+$br}$pv
342                                         eval bp_ver_${suitename}[i]=\$x
343                                         bp_desc[i]=$pd
344                                 fi
345                                 pn=; pv=; pd=; pp=; pN=; pf=; Lf=
346                                 ;;
347                         }
348                 done
349         done
350 done
351
352 (cat <<'EOF'
353 <?xml version="1.0"?>\r
354 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
355  "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
356 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head>
357  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
358  <meta name="MSSmartTagsPreventParsing" content="TRUE" />
359 EOF
360 print -r -- " <title>${repo_title} Index</title>"
361 cat <<'EOF'
362  <meta name="generator" content="Evolvis shellsnippets git based on $$Id: mvndebri.sh 4044 2014-05-23 09:48:47Z tglase $ based on $MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.61 2014/05/23 09:44:30 tg Exp $" />
363  <style type="text/css"><!--/*--><![CDATA[/*><!--*/
364   table {
365    border: 1px solid black;
366    border-collapse: collapse;
367    text-align: left;
368    vertical-align: top;
369   }
370   tr {
371    border: 1px solid black;
372    text-align: left;
373    vertical-align: top;
374   }
375   td {
376    border: 1px solid black;
377    text-align: left;
378    vertical-align: top;
379   }
380   th {
381    background-color: #000000;
382    color: #FFFFFF;
383   }
384   .tableheadcell {
385    border: 1px solid #999999;
386    padding: 3px;
387    white-space: nowrap;
388   }
389   .srcpkgline {
390    background-color: #CCCCCC;
391   }
392   .srcpkgdist {
393    background-color: #666666;
394    color: #FFFFFF;
395    font-weight: bold;
396   }
397   .binpkgdist {
398    background-color: #999999;
399    color: #FFFFFF;
400    font-weight: bold;
401   }
402  /*]]>*/--></style>
403 </head><body>
404 EOF
405 print -r -- "<h1>${repo_title}</h1>"
406 cat <<EOF
407 <p><a href="dists/">Browse</a> the repository.
408  This repository uses <a
409   href="http://pgp.uni-mainz.de:11371/pks/lookup?search=${repo_keyid}&amp;op=vindex">${repo_keyid}</a>
410  as signing key.
411 </p>
412 <h2>Suites</h2>
413 <ul>
414 EOF
415
416 allsuites=$(for suitename in $allsuites; do
417         print $suitename
418 done | sort -u)
419
420 for suitename in $allsuites; do
421         suite=dists/$suitename
422         print -n " <li>${suite##*/}: <a href=\"$suite/\">$jobname for $suitename</a> (dists:"
423         vdists=
424         for dist in $suite/*; do
425                 [[ -d $dist/. ]] || continue
426                 distname=${dist##*/}
427                 vdists="$vdists $distname"
428                 print -n " <a href=\"$suite/$distname/\">$distname</a>"
429         done
430         print ")<br />"
431         print " <tt>deb https://${dhn}/$jobname $suitename$vdists</tt>"
432         print "</li>"
433 done
434 print "</ul>"
435 print "<h2>Packages</h2>"
436 print "<table width=\"100%\"><thead>"
437 print "<tr class=\"tablehead\">"
438 print " <th class=\"tableheadcell\">dist</th>"
439 print " <th class=\"tableheadcell\" rowspan=\"2\">Binary / Description</th>"
440 for suitename in $allsuites; do
441         print " <th class=\"tableheadcell\" rowspan=\"2\">$suitename</th>"
442 done
443 print "</tr><tr class=\"tablehead\">"
444 print " <th class=\"tableheadcell\">package name</th>"
445 print "</tr></thead><tbody>"
446
447 set -A bp_sort
448 i=0
449 while (( i < nbin )); do
450         print $i ${bp_disp[i++]} #${bp_suites[i]}
451 done | sort -k2 |&
452 while read -p num rest; do
453         bp_sort[${#bp_sort[*]}]=$num
454 done
455
456 i=0
457 while (( i < nsrc )); do
458         print $i ${sp_name[i++]}
459 done | sort -k2 |&
460 while read -p num rest; do
461         print "\n<!-- sp #$num = ${sp_name[num]} -->"
462         print "<tr class=\"srcpkgline\">"
463         print " <td class=\"srcpkgdist\">${sp_dist[num]}</td>"
464         pd=
465         for x in $(tr ', ' '\n' <<<"${sp_desc[num]}" | sort -u); do
466                 [[ -n $x ]] && pd="$pd, $x"
467         done
468         print " <td rowspan=\"2\" class=\"srcpkgdesc\">${pd#, }</td>"
469         for suitename in $allsuites; do
470                 eval pvo=\${sp_ver_${suitename}[num]}
471                 eval ppo=\${sp_dir_${suitename}[num]}
472                 IFS=,
473                 set -A pva -- $pvo
474                 set -A ppa -- $ppo
475                 IFS=$saveIFS
476                 (( ${#pva[*]} )) || pva[0]=
477                 y=
478                 i=0
479                 while (( i < ${#pva[*]} )); do
480                         pv=${pva[i]}
481                         pp=${ppa[i]}
482                         if [[ $pv = *"\1a"* ]]; then
483                                 pvdsc=${pv%%"\1a"*}
484                                 pv=${pv##*"\1a"}
485                         else
486                                 pvdsc=$pv
487                         fi
488                         if [[ -z $pv ]]; then
489                                 pv=-
490                         elif [[ $pp != ?(/) ]]; then
491                                 pv="<a href=\"$pp${sp_name[num]}_${pvdsc##+([0-9]):}.dsc\">$pv</a>"
492                         fi
493                         [[ $pp != ?(/) ]] && pv="<a href=\"$pp\">[dir]</a> $pv"
494                         y=${y:+"$y<br />"}$pv
495                         let i++
496                 done
497                 print " <td rowspan=\"2\" class=\"srcpkgitem\">$y</td>"
498         done
499         print "</tr><tr class=\"srcpkgline\">"
500         print " <td class=\"srcpkgname\">${sp_name[num]}</td>"
501         print "</tr>"
502         k=0
503         while (( k < nbin )); do
504                 (( (i = bp_sort[k++]) < 0 )) && continue
505                 [[ ${bp_name[i]} = "${sp_name[num]}" && \
506                     ${bp_dist[i]} = "${sp_dist[num]}" ]] || continue
507                 bp_sort[k - 1]=-1
508                 #print "<!-- bp #$i for${bp_suites[i]} -->"
509                 print "<!-- bp #$i -->"
510                 print "<tr class=\"binpkgline\">"
511                 print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
512                 print " <td class=\"binpkgdesc\">$(xhtml_escape "${bp_desc[i]}")</td>"
513                 for suitename in $allsuites; do
514                         eval pv=\${bp_ver_${suitename}[i]}
515                         if [[ -z $pv ]]; then
516                                 pv=-
517                         fi
518                         print " <td class=\"binpkgitem\">$pv</td>"
519                 done
520                 print "</tr>"
521         done
522 done
523
524 num=0
525 for i in ${bp_sort[*]}; do
526         (( i < 0 )) && continue
527         if (( !num )); then
528                 print "\n<!-- sp ENOENT -->"
529                 print "<tr class=\"srcpkgline\">"
530                 print " <td class=\"srcpkgname\">~ENOENT~</td>"
531                 print " <td class=\"srcpkgdesc\">binary" \
532                     "packages without a matching source package</td>"
533                 for suitename in $allsuites; do
534                         print " <td class=\"srcpkgitem\">-</td>"
535                 done
536                 print "</tr>"
537                 num=1
538         fi
539         #print "<!-- bp #$i for${bp_suites[i]} -->"
540         print "<!-- bp #$i -->"
541         print "<tr class=\"binpkgline\">"
542         print " <td class=\"binpkgdist\">${bp_dist[i]}</td>"
543         print " <td rowspan=\"2\" class=\"binpkgdesc\">$(xhtml_escape "${bp_desc[i]}")</td>"
544         for suitename in $allsuites; do
545                 eval pv=\${bp_ver_${suitename}[i]}
546                 if [[ -z $pv ]]; then
547                         pv=-
548                 fi
549                 print " <td rowspan=\"2\" class=\"binpkgitem\">$pv</td>"
550         done
551         print "</tr><tr class=\"binpkgline\">"
552         print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
553         print "</tr>"
554 done
555
556 cat <<EOF
557
558 </tbody></table>
559
560 <p>• <a href="http://validator.w3.org/check/referer">Valid XHTML/1.1!</a>
561  • <small>Generated on $(date -u +'%F %T') by <tt
562  style="white-space:pre;">$rcsid</tt></small> •</p>
563 </body></html>
564 EOF
565
566 :) >debidx.htm
567 print done.