ba3466ad4b17d84df4177f20b91e170dbaabbf90
[shellsnippets/shellsnippets.git] / mksh / debian-dev / mkdebidx.sh
1 #!/bin/mksh
2 rcsid='$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.46 2011/03/04 14:24:32 tg Exp $'
3 #-
4 # Copyright (c) 2008, 2009, 2010, 2011
5 #       Thorsten Glaser <tg@mirbsd.org>
6 #
7 # Provided that these terms and disclaimer and all copyright notices
8 # are retained or reproduced in an accompanying document, permission
9 # is granted to deal in this work without restriction, including un-
10 # limited rights to use, publicly perform, distribute, sell, modify,
11 # merge, give away, or sublicence.
12 #
13 # This work is provided "AS IS" and WITHOUT WARRANTY of any kind, to
14 # the utmost extent permitted by applicable law, neither express nor
15 # implied; without malicious intent or gross negligence. In no event
16 # may a licensor, author or contributor be held liable for indirect,
17 # direct, other damage, loss, or other issues arising in any way out
18 # of dealing in the work, even if advised of the possibility of such
19 # damage or existence of a defect, except proven that it results out
20 # of said person's immediate fault when using the work as intended.
21
22 set -A normarchs -- i386
23 repo_keyid=0x405422DD
24 # either '' (locally) or 'remsign user@host.domain.com' (remote ssh)
25 gpg_remote=
26 repo_origin='The MirOS Project'
27 repo_label=wtf
28 repo_title='MirDebian “WTF” Repository'
29 function repo_description {
30         typeset suite_nick=$1
31
32         print -nr -- "WTF ${suite_nick} Repository"
33 }
34
35
36 set -A dpkgarchs -- alpha amd64 arm armeb armel armhf avr32 hppa \
37     i386 ia64 kfreebsd-amd64 kfreebsd-i386 lpia m32r m68k mips mipsel \
38     powerpc powerpcspe ppc64 s390 s390x sh3 sh3eb sh4 sh4eb sparc
39
40 function remsign {
41         target=$1; shift
42         master=remsign.ctl$$
43         tmpfnm=remsign.tmp$$
44         ssh -fNM -o ControlPath=$tmpfnm "$target"
45         ssh -o ControlPath=$tmpfnm "$target" cat \>$tmpfnm
46         ssh -o ControlPath=$tmpfnm -t "$target" "$* $tmpfnm" 0<&2 1>&2
47         rv=$?
48         ssh -o ControlPath=$tmpfnm "$target" "cat $tmpfnm.sig; rm -f $tmpfnm $tmpfnm.sig"
49         ssh -o ControlPath=$tmpfnm "$target" -O exit
50         return $rv
51 }
52
53 function putfile {
54         tee $1 | gzip -n9 >$1.gz
55 }
56
57 function sortlist {
58         typeset x u=$1
59
60         if [[ $u = -u ]]; then
61                 shift
62         else
63                 u=
64         fi
65
66         for x in "$@"; do
67                 print -r -- "$x"
68         done | sort $u
69 }
70
71 # DJB cdb hash (not via stdio, for speed)
72 typeset -Z11 -Uui16 Lcdbhash_result
73 function Lcdbhash_add {
74         typeset s="$*"
75         typeset -i i=0 n=${#s}
76
77         while (( i < n )); do
78                 ((# Lcdbhash_result = (Lcdbhash_result * 33) ^ 1#${s:(i++):1} ))
79         done
80 }
81
82 set +U
83 export LC_ALL=C
84 unset LANGUAGE
85 saveIFS=$IFS
86 cd "$(dirname "$0")"
87 rm -f dpkg_578162_workaround
88
89 IFS=:
90 dpkgarchl=:all:"${dpkgarchs[*]}":
91 IFS=$saveIFS
92
93 suites=:
94 for suite in "$@"; do
95         suites=:dists/$suite$suites
96 done
97
98 allsuites=
99 for suite in dists/*; do
100         allsuites="$allsuites${allsuites:+ }${suite##*/}"
101         [[ $suites = : || $suites = *:"$suite":* ]] || continue
102         archs=
103         . $suite/distinfo.sh
104         suitearchs=${archs:-${normarchs[*]}}
105         components=Components:
106         for dist in $suite/*; do
107                 [[ -d $dist/. ]] || continue
108                 rm -rf $dist/binary-* $dist/source
109                 ovf= oef= osf=
110                 [[ -s $dist/override.file ]] && ovf=$dist/override.file
111                 [[ -s $dist/override.extra ]] && oef="-e $dist/override.extra"
112                 [[ -s $dist/override.src ]] && osf="-s $dist/override.src"
113                 components="$components ${dist##*/}"
114                 archs=
115                 [[ -s $dist/distinfo.sh ]] && . $dist/distinfo.sh
116                 set -A distarchs -- $(sortlist -u all ${archs:-$suitearchs})
117                 IFS=:; distarchl=:"${distarchs[*]}":; IFS=$saveIFS
118                 for arch in $(sortlist -u ${distarchs[*]} ${dpkgarchs[*]}); do
119                         if [[ $dpkgarchl != *:"$arch":* ]]; then
120                                 print -u2 "Invalid arch '$arch' in $dist"
121                                 exit 1
122                         elif [[ $distarchl != *:"$arch":* ]]; then
123                                 print "\n===> Linking all =>" \
124                                     "${dist#dists/}/$arch/Packages"
125                                 ln -s binary-all $dist/binary-$arch
126                         else
127                                 print "\n===> Creating" \
128                                     "${dist#dists/}/$arch/Packages\n"
129                                 mkdir -p $dist/binary-$arch
130                                 dpkg-scanpackages $oef -m -a $arch \
131                                     $dist $ovf | \
132                                     putfile $dist/binary-$arch/Packages
133                         fi
134                 done
135                 print "\n===> Creating ${dist#dists/}/Sources"
136                 mkdir -p $dist/source
137                 [[ -e dpkg_578162_workaround ]] || (dpkg-scansources $oef $osf \
138                     $dist $ovf || touch dpkg_578162_workaround) | \
139                     putfile $dist/source/Sources
140                 [[ -e dpkg_578162_workaround ]] && dpkg-scansources $osf \
141                     $dist $ovf | \
142                     putfile $dist/source/Sources
143                 print done.
144         done
145         print "\n===> Creating ${suite#dists/}/Release.gpg"
146         rm -f $suite/Release*
147         (cat <<-EOF
148                 Origin: ${repo_origin}
149                 Label: ${repo_label}
150                 Suite: ${suite##*/}
151                 Codename: ${suite##*/}
152                 Date: $(date -u)
153                 Architectures: all ${dpkgarchs[*]} source
154                 $components
155                 Description: $(repo_description "$nick")
156                 MD5Sum:
157         EOF
158         cd $suite
159         set -A cache_fn
160         set -A cache_md5
161         set -A cache_size
162         for n in Contents-* */{binary-*,source}/{Packag,Sourc}es*; do
163                 [[ -f $n ]] || continue
164                 # realpath-ise $n and cache the checksum
165                 nn=$(realpath "$n")
166                 #XXX once mksh can, use associative arrays instead
167                 Lcdbhash_result=5381
168                 Lcdbhash_add "$nn"
169                 # simple hash collision solver by increment
170                 nc=${cache_fn[Lcdbhash_result]}
171                 while [[ -n $nc && $nc != "$nn" ]]; do
172                         nc=${cache_fn[++Lcdbhash_result]}
173                 done
174                 if [[ $nc = "$nn" ]]; then
175                         nm=${cache_md5[Lcdbhash_result]}
176                         ns=${cache_size[Lcdbhash_result]}
177                 else
178                         set -A x -- $(md5sum "$nn")
179                         nm=${x[0]}
180                         ns=$(stat -c '%s' "$nn")
181                         cache_md5[Lcdbhash_result]=$nm
182                         cache_size[Lcdbhash_result]=$ns
183                         cache_fn[Lcdbhash_result]=$nn
184                 fi
185                 print " $nm $ns $n"
186         done) >$suite/Release
187         $gpg_remote gpg -u $repo_keyid -sb <$suite/Release >$suite/Release.gpg
188 done
189
190 print "\n===> Creating debidx.htm\n"
191
192 set -A preplsrc
193 set -A prepldst
194 integer nsrc=0 nbin=0 nrpl=0
195 br='<br />'
196
197 # syntax:       ${suitename}/${distname}/${pN}/${pp} <suite>
198 # example:      sid/wtf/openntpd/i386 lenny
199 if [[ -s mkdebidx.lnk ]]; then
200         while read pn pd; do
201                 [[ $pn = '#'* ]] && continue
202                 if [[ $pn != +([a-z0-9_])/+([a-z0-9_-])/+([!/])/@(%|=|+([a-z0-9])) || \
203                     $pd != +([a-z0-9_]) ]]; then
204                         print -u2 "Invalid lnk line '$pn' '$pd'"
205                         continue
206                 fi
207                 preplsrc[nrpl]=$pn
208                 prepldst[nrpl++]=$pd
209         done <mkdebidx.lnk
210 fi
211
212 for suite in dists/*; do
213         for dist in $suite/*; do
214                 [[ -d $dist/. ]] || continue
215                 suitename=${suite##*/}
216                 if [[ $suitename != +([a-z0-9_]) ]]; then
217                         print -u2 "Invalid suite name '$suitename'"
218                         continue 2
219                 fi
220                 distname=${dist##*/}
221                 if [[ $distname != +([a-z0-9_-]) ]]; then
222                         print -u2 "Invalid dist name '$distname'"
223                         continue
224                 fi
225
226                 pn=; pv=; pd=; pp=; pN=; pf=
227
228                 gzip -dc $dist/source/Sources.gz |&
229                 while IFS= read -pr line; do
230                         case $line {
231                         ("Package: "*)
232                                 pn=${line##Package:*([   ])}
233                                 ;;
234                         ("Version: "*)
235                                 pv=${line##Version:*([   ])}
236                                 ;;
237                         ("Binary: "*)
238                                 pd=${line##Binary:*([    ])}
239                                 ;;
240                         ("Directory: "*)
241                                 pp=${line##Directory:*([         ])}
242                                 ;;
243                         (?*)    # anything else
244                                 ;;
245                         (*)     # empty line
246                                 if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
247                                         i=0
248                                         while (( i < nsrc )); do
249                                                 [[ ${sp_name[i]} = "$pn" && \
250                                                     ${sp_dist[i]} = "$distname" ]] && break
251                                                 let i++
252                                         done
253                                         if (( i == nsrc )); then
254                                                 let nsrc++
255                                                 pvo=
256                                                 ppo=
257                                         else
258                                                 eval pvo=\$\{sp_ver_${suitename}[i]\}
259                                                 eval ppo=\$\{sp_dir_${suitename}[i]\}
260                                         fi
261                                         sp_name[i]=$pn
262                                         sp_dist[i]=$distname
263                                         #sp_suites[i]="${sp_suites[i]} $suitename"
264                                         if (( nrpl )); then
265                                                 x=${suitename}/${distname}/${pn}/source
266                                                 j=0
267                                                 while (( j < nrpl )); do
268                                                         [[ ${preplsrc[j]} = "$x" ]] && break
269                                                         let j++
270                                                 done
271                                                 (( j < nrpl )) && pv="${pv}\1afrom ${prepldst[j]}"
272                                         fi
273                                         eval sp_ver_${suitename}[i]='${pvo:+$pvo,}$pv'
274                                         eval sp_dir_${suitename}[i]='${ppo:+$ppo,}$pp/'
275                                         sp_desc[i]=${sp_desc[i]},$pd
276                                 fi
277                                 pn=; pv=; pd=; pp=
278                                 ;;
279                         }
280                 done
281
282                 gzip -dc $(for f in $dist/binary-*/Packages.gz; do
283                         [[ -e $f ]] || continue
284                         realpath "$f"
285                 done | sort -u) |&
286                 while IFS= read -pr line; do
287                         case $line {
288                         ("Package: "*)
289                                 pN=${line##Package:*([   ])}
290                                 ;;
291                         ("Source: "*)
292                                 pn=${line##Source:*([    ])}
293                                 pn=${pn%% *}
294                                 ;;
295                         ("Version: "*)
296                                 pv=${line##Version:*([   ])}
297                                 ;;
298                         ("Description: "*)
299                                 pd=${line##Description:*([       ])}
300                                 ;;
301                         ("Architecture: "*)
302                                 pp=${line##Architecture:*([      ])}
303                                 ;;
304                         ("Filename: "*)
305                                 pf=${line##Filename:*([  ])}
306                                 ;;
307                         (?*)    # anything else
308                                 ;;
309                         (*)     # empty line
310                                 if [[ $pf = *:* || $pf = *'%'* ]]; then
311                                         print -u2 Illegal character in $dist \
312                                             packages $pp "'Filename: $pf'"
313                                         exit 1
314                                 fi
315                                 [[ -n $pn ]] || pn=$pN
316                                 if [[ -n $pn && -n $pv && -n $pd && -n $pp ]]; then
317                                         i=0
318                                         while (( i < nbin )); do
319                                                 [[ ${bp_disp[i]} = "$pN" && ${bp_desc[i]} = "$pd" && \
320                                                     ${bp_dist[i]} = "$distname" ]] && break
321                                                 let i++
322                                         done
323                                         (( i == nbin )) && let nbin++
324                                         bp_name[i]=$pn
325                                         bp_disp[i]=$pN
326                                         bp_dist[i]=$distname
327                                         #bp_suites[i]="${bp_suites[i]} $suitename"
328                                         if (( nrpl )); then
329                                                 x=${suitename}/${distname}/${pN}/${pp}
330                                                 j=0
331                                                 while (( j < nrpl )); do
332                                                         [[ ${preplsrc[j]} = "$x" ]] && break
333                                                         let j++
334                                                 done
335                                                 (( j < nrpl )) && pv="from ${prepldst[j]}"
336                                         fi
337                                         [[ -n $pf ]] && pv="<a href=\"$pf\">$pv</a>"
338                                         pv="$pp: $pv"
339                                         eval x=\${bp_ver_${suitename}[i]}
340                                         [[ $br$x$br = *"$br$pv$br"* ]] || x=$x${x:+$br}$pv
341                                         eval bp_ver_${suitename}[i]=\$x
342                                         bp_desc[i]=$pd
343                                 fi
344                                 pn=; pv=; pd=; pp=; pN=
345                                 ;;
346                         }
347                 done
348         done
349 done
350
351 (cat <<'EOF'
352 <?xml version="1.0"?>\r
353 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
354  "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
355 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head>
356  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
357  <meta name="MSSmartTagsPreventParsing" content="TRUE" />
358 EOF
359 print -r -- " <title>${repo_title} Index</title>"
360 cat <<'EOF'
361  <meta name="generator" content="$MirOS: contrib/hosted/tg/deb/mkdebidx.sh,v 1.46 2011/03/04 14:24:32 tg Exp $" />
362  <style type="text/css">
363   table {
364    border: 1px solid black;
365    border-collapse: collapse;
366    text-align: left;
367    vertical-align: top;
368   }
369   tr {
370    border: 1px solid black;
371    text-align: left;
372    vertical-align: top;
373   }
374   td {
375    border: 1px solid black;
376    text-align: left;
377    vertical-align: top;
378   }
379   th {
380    background-color: #000000;
381    color: #FFFFFF;
382   }
383   .tableheadcell {
384    border: 1px solid #999999;
385    padding: 3px;
386    white-space: nowrap;
387   }
388   .srcpkgline {
389    background-color: #CCCCCC;
390   }
391   .srcpkgdist {
392    background-color: #666666;
393    color: #FFFFFF;
394    font-weight: bold;
395   }
396   .binpkgdist {
397    background-color: #999999;
398    color: #FFFFFF;
399    font-weight: bold;
400   }
401  </style>
402 </head><body>
403 EOF
404 print -r -- "<h1>${repo_title}</h1>"
405 cat <<'EOF'
406 <p><a href="dists/">Browse</a> the repository or read about how to amend <a
407  href="sources.txt">/etc/apt/sources.list</a> in order to use it.
408 EOF
409 [[ -s 0-NOTE.txt ]] && print ' Also read my <a href="0-NOTE.txt">notes</a>.'
410 cat <<EOF
411  This repository uses <a
412   href="http://pgpkeys.pca.dfn.de/pks/lookup?search=${repo_keyid}&amp;op=vindex">${repo_keyid}</a>
413  as signing key.
414 </p>
415 <h2>Suites</h2>
416 <ul>
417 EOF
418
419 allsuites=$(for suitename in $allsuites; do
420         print $suitename
421 done | sort -u)
422
423 for suitename in $allsuites; do
424         suite=dists/$suitename
425         . $suite/distinfo.sh
426         print -n " <li>${suite##*/}: <a href=\"$suite/\">$desc</a> (dists:"
427         for dist in $suite/*; do
428                 [[ -d $dist/. ]] || continue
429                 distname=${dist##*/}
430                 print -n " <a href=\"$suite/$distname/\">$distname</a>"
431         done
432         print ")</li>"
433 done
434 print "</ul>"
435 print "<h2>Packages</h2>"
436 print "<table width=\"100%\"><thead>"
437 print "<tr class=\"tablehead\">"
438 print " <th class=\"tableheadcell\">dist</th>"
439 print " <th class=\"tableheadcell\" rowspan=\"2\">Binary / Description</th>"
440 for suitename in $allsuites; do
441         print " <th class=\"tableheadcell\" rowspan=\"2\">$suitename</th>"
442 done
443 print "</tr><tr class=\"tablehead\">"
444 print " <th class=\"tableheadcell\">package name</th>"
445 print "</tr></thead><tbody>"
446
447 set -A bp_sort
448 i=0
449 while (( i < nbin )); do
450         print $i ${bp_disp[i++]} #${bp_suites[i]}
451 done | sort -k2 |&
452 while read -p num rest; do
453         bp_sort[${#bp_sort[*]}]=$num
454 done
455
456 i=0
457 while (( i < nsrc )); do
458         print $i ${sp_name[i++]}
459 done | sort -k2 |&
460 while read -p num rest; do
461         print "\n<!-- sp #$num = ${sp_name[num]} -->"
462         print "<tr class=\"srcpkgline\">"
463         print " <td class=\"srcpkgdist\">${sp_dist[num]}</td>"
464         pd=
465         for x in $(tr ', ' '\n' <<<"${sp_desc[num]}" | sort -u); do
466                 [[ -n $x ]] && pd="$pd, $x"
467         done
468         print " <td rowspan=\"2\" class=\"srcpkgdesc\">${pd#, }</td>"
469         for suitename in $allsuites; do
470                 eval pvo=\${sp_ver_${suitename}[num]}
471                 eval ppo=\${sp_dir_${suitename}[num]}
472                 IFS=,
473                 set -A pva -- $pvo
474                 set -A ppa -- $ppo
475                 IFS=$saveIFS
476                 (( ${#pva[*]} )) || pva[0]=
477                 y=
478                 i=0
479                 while (( i < ${#pva[*]} )); do
480                         pv=${pva[i]}
481                         pp=${ppa[i]}
482                         if [[ $pv = *"\1a"* ]]; then
483                                 pvdsc=${pv%%"\1a"*}
484                                 pv=${pv##*"\1a"}
485                         else
486                                 pvdsc=$pv
487                         fi
488                         if [[ -z $pv ]]; then
489                                 pv=-
490                                 if (( nrpl )); then
491                                         x=${suitename}/${sp_dist[num]}/${sp_name[num]}/%
492                                         j=0
493                                         while (( j < nrpl )); do
494                                                 [[ ${preplsrc[j]} = "$x" ]] && break
495                                                 let j++
496                                         done
497                                         (( j < nrpl )) && pv=${prepldst[j]}
498                                 fi
499                         elif [[ $pp != ?(/) ]]; then
500                                 pv="<a href=\"$pp${sp_name[num]}_${pvdsc##+([0-9]):}.dsc\">$pv</a>"
501                         fi
502                         [[ $pp != ?(/) ]] && pv="<a href=\"$pp\">[dir]</a> $pv"
503                         y=${y:+"$y<br />"}$pv
504                         let i++
505                 done
506                 print " <td rowspan=\"2\" class=\"srcpkgitem\">$y</td>"
507         done
508         print "</tr><tr class=\"srcpkgline\">"
509         print " <td class=\"srcpkgname\">${sp_name[num]}</td>"
510         print "</tr>"
511         k=0
512         while (( k < nbin )); do
513                 (( (i = bp_sort[k++]) < 0 )) && continue
514                 [[ ${bp_name[i]} = "${sp_name[num]}" && \
515                     ${bp_dist[i]} = "${sp_dist[num]}" ]] || continue
516                 bp_sort[k - 1]=-1
517                 #print "<!-- bp #$i for${bp_suites[i]} -->"
518                 print "<!-- bp #$i -->"
519                 print "<tr class=\"binpkgline\">"
520                 print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
521                 print " <td class=\"binpkgdesc\">${bp_desc[i]}</td>"
522                 for suitename in $allsuites; do
523                         eval pv=\${bp_ver_${suitename}[i]}
524                         if [[ -z $pv ]]; then
525                                 pv=-
526                                 if (( nrpl )); then
527                                         x=${suitename}/${sp_dist[num]}/${sp_name[num]}/%
528                                         j=0
529                                         while (( j < nrpl )); do
530                                                 [[ ${preplsrc[j]} = "$x" ]] && break
531                                                 let j++
532                                         done
533                                         (( j < nrpl )) && pv=${prepldst[j]}
534                                 fi
535                         fi
536                         print " <td class=\"binpkgitem\">$pv</td>"
537                 done
538                 print "</tr>"
539         done
540 done
541
542 num=0
543 for i in ${bp_sort[*]}; do
544         (( i < 0 )) && continue
545         if (( !num )); then
546                 print "\n<!-- sp ENOENT -->"
547                 print "<tr class=\"srcpkgline\">"
548                 print " <td class=\"srcpkgname\">~ENOENT~</td>"
549                 print " <td class=\"srcpkgdesc\">binary" \
550                     "packages without a matching source package</td>"
551                 for suitename in $allsuites; do
552                         print " <td class=\"srcpkgitem\">-</td>"
553                 done
554                 print "</tr>"
555                 num=1
556         fi
557         #print "<!-- bp #$i for${bp_suites[i]} -->"
558         print "<!-- bp #$i -->"
559         print "<tr class=\"binpkgline\">"
560         print " <td class=\"binpkgdist\">${bp_dist[i]}</td>"
561         print " <td rowspan=\"2\" class=\"binpkgdesc\">${bp_desc[i]}</td>"
562         for suitename in $allsuites; do
563                 eval pv=\${bp_ver_${suitename}[i]}
564                 if [[ -z $pv ]]; then
565                         pv=-
566                         if (( nrpl )); then
567                                 x=${suitename}/${bp_dist[num]}/${bp_disp[num]}/=
568                                 j=0
569                                 while (( j < nrpl )); do
570                                         [[ ${preplsrc[j]} = "$x" ]] && break
571                                         let j++
572                                 done
573                                 (( j < nrpl )) && pv=${prepldst[j]}
574                         fi
575                 fi
576                 print " <td rowspan=\"2\" class=\"binpkgitem\">$pv</td>"
577         done
578         print "</tr><tr class=\"binpkgline\">"
579         print " <td class=\"binpkgname\">${bp_disp[i]}</td>"
580         print "</tr>"
581 done
582
583 cat <<EOF
584
585 </tbody></table>
586
587 <p>• <a href="http://validator.w3.org/check/referer">Valid XHTML/1.1!</a>
588  • <small>Generated on $(date -u +'%F %T') by <tt
589  style="white-space:pre;">$rcsid</tt></small> •</p>
590 </body></html>
591 EOF
592
593 :) >debidx.htm
594 print done.