OILS / build / doc.sh View on Github | oilshell.org

687 lines, 365 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=. doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97
98 # Data language
99 qsn
100 qtt
101 j8-notation
102 pretty-printing
103
104 doc-toolchain
105 doc-plugins
106 idioms
107 shell-idioms
108 ysh-faq
109
110 language-influences
111 ysh-vs-python
112 ysh-vs-shell
113
114 syntactic-concepts
115 syntax-feelings
116 command-vs-expression-mode
117
118 # needs polish
119 # Note: docs about the YSH are prefixed 'ysh-'.
120 # data-model and command-vs-expression-mode span both OSH and YSH
121
122 index
123 faq-doc
124
125 options
126
127 old/index
128 old/project-tour
129 old/legacy-array
130 old/ysh-keywords
131 old/modules
132 old/expression-language
133 old/word-language
134 old/errors
135 old/ysh-builtins
136
137 io-builtins
138 unicode
139 framing
140 xtrace
141 headless
142 completion
143 strings
144 variables
145
146 # Internal stuff
147 interpreter-state
148 process-model
149 architecture-notes
150 parser-architecture
151)
152
153# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
154# locale), which is loaded by Python into say u'\u5e74'. But the default
155# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
156# fail.
157#
158# I believe --rfc-e-mail should never output a Unicode character.
159#
160# A better fix would be to implement json_utf8.load(f), which doesn't decode
161# into unicode instances. This would remove useless conversions.
162
163readonly TIMESTAMP=$(date --rfc-email)
164
165split-and-render() {
166 local src=${1:-doc/known-differences.md}
167
168 local rel_path=${src%'.md'} # doc/known-differences
169 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
170
171 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
172 local web_url=${3:-'../web'}
173
174 mkdir -v -p $(dirname $out) $tmp_prefix
175
176 # Also add could add css_files. The one in the file takes precedence always?
177
178 # css_files: a space-separated list
179 # all_docs_url: so we link from doc/foo.html -> doc/
180
181 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
182
183 doctools/split_doc.py \
184 -v build_timestamp="$TIMESTAMP" \
185 -v oil_version="$OIL_VERSION" \
186 -v css_files="$css_files" \
187 -v all_docs_url='.' \
188 -v repo_url="$src" \
189 $src $tmp_prefix
190
191 #ls -l _tmp/doc
192 #head _tmp/doc/*
193 #return
194
195 # for ysh-tour code blocks
196 local code_out=_tmp/code-blocks/$rel_path.txt
197 mkdir -v -p $(dirname $code_out)
198
199 cmark \
200 --code-block-output $code_out \
201 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
202
203 log "$tmp_prefix -> (doctools/cmark) -> $out"
204}
205
206render-from-kate() {
207 ### Make it easier to configure Kate editor
208
209 # It want to pass an absolute path
210 # TODO: I can't figure out how to run this from Kate?
211
212 local full_path=$1
213
214 case $full_path in
215 $REPO_ROOT/*)
216 rel_path=${full_path#"$REPO_ROOT/"}
217 echo "relative path = $rel_path"
218 ;;
219 *)
220 die "$full_path should start with repo root $REPO_ROOT"
221 ;;
222 esac
223
224 split-and-render $rel_path
225}
226
227# Special case for README
228# Do NOT split because we don't want front matter in the markdown source.
229render-only() {
230 local src=${1:-README.md}
231
232 local name
233 case $src in
234 *.md)
235 name=$(basename $src .md)
236 ;;
237 *.txt)
238 name=$(basename $src .txt)
239 ;;
240 *)
241 name=$(basename $src)
242 ;;
243 esac
244
245 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
246 local css_files=${3:-'../web/manual.css ../web/toc.css'}
247 local title=${4:-'Oils Source Code'}
248
249 local prefix=_tmp/doc/$name
250
251 local meta=${prefix}_meta.json
252 cat >$meta <<EOF
253{ "title": "$title",
254 "repo_url": "$src",
255 "css_files": "$css_files",
256 "all_docs_url": ".",
257
258 "build_timestamp": "$TIMESTAMP",
259 "oil_version": "$OIL_VERSION"
260}
261EOF
262
263 cmark $meta $src > $out
264 log "Wrote $out"
265}
266
267special() {
268 # TODO: do all READMEs
269 split-and-render mycpp/README.md \
270 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
271 ../../../web
272
273 local web_dir='../../web'
274 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
275 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
276
277 local web_dir='../web'
278 render-only INSTALL.txt '' \
279 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
280
281 render-only INSTALL-old.txt '' \
282 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
283
284 # These pages aren't in doc/
285 split-and-render doc/release-index.md _tmp/release-index.html
286 split-and-render doc/release-quality.md _tmp/release-quality.html
287}
288
289all-markdown() {
290 make-dirs
291
292 # TODO: We can set repo_url here! Then we don't need it for most docs.
293 # split_doc.py can return {} if the doc doesn't start with ---
294
295 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
296 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
297 for d in "${MARKDOWN_DOCS[@]}"; do
298 split-and-render doc/$d.md
299 done
300
301 special
302}
303
304redir-body() {
305 local to_url=$1 # WARNING: no escaping
306 cat <<EOF
307<head>
308 <meta http-equiv="Refresh" content="0; URL=$to_url" />
309</head>
310EOF
311}
312
313redirect-pairs() {
314 # we want want /release/latest/ URLs to still work
315 cat <<EOF
316oil-language-tour ysh-tour
317oil-language-faq ysh-faq
318oil-help ysh-help
319oil-help-topics ysh-help-topics
320ysh-help ref/toc-ysh
321ysh-help-topics ref/toc-ysh
322EOF
323}
324
325all-redirects() {
326 redirect-pairs | while read -r from_page to_page; do
327 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
328 done
329}
330
331# TODO: This could use some CSS.
332man-page() {
333 local root_dir=${1:-_release/VERSION}
334 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
335 ls -l $root_dir
336}
337
338# I want to ship the INSTALL file literally, so just mutate things
339_sed-ext() {
340 sed --regexp-extended -i "$@"
341}
342
343update-src-versions() {
344 # Update tarball names, etc.
345 _sed-ext \
346 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
347 doc/release-*.md INSTALL.txt INSTALL-old.txt
348
349 # Update /release/0.8.4/ URL, etc.
350 _sed-ext \
351 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
352 doc/osh.1
353}
354
355#
356# Test Tools
357#
358
359split-doc-demo() {
360 cat > _tmp/testdoc.md <<EOF
361---
362title: foo
363---
364
365Title
366=====
367
368hello
369
370EOF
371
372 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
373
374 head _tmp/testdoc*
375}
376
377#
378# Help is both markdown and text
379#
380
381readonly TMP_DIR=_tmp/doc
382readonly CODE_BLOCK_DIR=_tmp/code-blocks
383readonly TEXT_DIR=_devbuild/help
384readonly HTML_DIR=_release/VERSION
385readonly CODE_DIR=_devbuild/gen
386
387cards-from-indices() {
388 ### Make help cards
389
390 for lang in osh ysh data; do
391 help-gen cards-from-index $lang $TEXT_DIR \
392 < $HTML_DIR/doc/ref/toc-$lang.html
393 done
394}
395
396cards-from-chapters() {
397 ### Turn h3 topics into cards
398
399 local py_out=$CODE_DIR/help_meta.py
400
401 mkdir -p _gen/frontend
402 local cc_prefix=_gen/frontend/help_meta
403
404 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
405 $HTML_DIR/doc/ref/chap-*.html
406}
407
408ref-check() {
409 help-gen ref-check \
410 doc/ref/toc-*.md \
411 _release/VERSION/doc/ref/chap-*.html
412}
413
414fmt-check() {
415 PYTHONPATH=. doctools/fmt_check.py _release/VERSION/doc/ref/*.html
416}
417
418
419write-metrics() {
420 ### Check indexes and chapters against each other
421
422 local out=_release/VERSION/doc/metrics.txt
423
424 # send stderr to the log file too
425 ref-check > $out 2>&1
426
427 echo "Wrote $out"
428}
429
430tour() {
431 ### Build the Tour of YSH, and execute code as validation
432 local name=${1:-ysh-tour}
433
434 split-and-render doc/$name.md
435
436 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
437
438 mkdir -p $work_dir/lib
439
440 # Files used by module example
441 touch $work_dir/{build,test}.sh
442
443 cat >$work_dir/lib/util.ysh <<EOF
444log() { echo "$@" 1>&2; }
445EOF
446
447 pushd $work_dir
448 $REPO_ROOT/bin/ysh $name.txt
449 popd
450
451 # My own dev tools
452 # if test -d ~/vm-shared; then
453 if false; then
454 local path=_release/VERSION/doc/$name.html
455 cp -v $path ~/vm-shared/$path
456 fi
457}
458
459one() {
460 ### Iterate on one doc quickly
461
462 local name=${1:-options}
463
464 split-and-render doc/$name.md
465
466 # Make sure the doc has valid YSH code?
467 # TODO: Maybe need an attribute for OSH or YSH
468 pushd _tmp/code-blocks/doc
469 $REPO_ROOT/bin/ysh $name.txt
470 popd
471
472 if test -d ~/vm-shared; then
473 local out="${name%.md}.html"
474 local path=_release/VERSION/$out
475 cp -v $path ~/vm-shared/$path
476 fi
477}
478
479make-dirs() {
480 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
481}
482
483one-ref() {
484 local md=${1:-doc/ref/index.md}
485 split-and-render $md '' '../../web'
486}
487
488all-ref() {
489 ### Build doc/ref in text and HTML. Depends on libcmark.so
490
491 log "Removing $TEXT_DIR/*"
492 rm -f $TEXT_DIR/*
493 make-dirs
494
495 # Make the indexes and chapters
496 for d in doc/ref/*.md; do
497 split-and-render $d '' '../../web'
498 done
499
500 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
501 # extract topics from all chapters first, and then make help_meta.json, like
502 # we have _devbuild/gen/help_meta.py.
503
504 # Text cards
505 cards-from-indices
506 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
507 cards-from-chapters
508
509 if command -v pysum; then
510 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
511 # compress it.
512 echo 'Size of embedded help:'
513 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
514 fi
515
516 # Better sorting
517 #LANG=C ls -l $TEXT_DIR
518}
519
520_copy-path() {
521 local src=$1 dest=$2
522 mkdir -p $(dirname $dest)
523 cp -v $src $dest
524}
525
526copy-web() {
527 find web \
528 \( -name _tmp -a -prune \) -o \
529 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
530 xargs -n 2 -- $0 _copy-path
531}
532
533pretty-size() {
534 local path=$1
535 stat --format '%s' "$path" | python -c '
536import sys
537num_bytes = int(sys.stdin.read())
538print "{:,}".format(num_bytes)
539'
540}
541
542# NOTE: It might be better to link to files like this in the /release/ tree.
543# Although I am not signing them.
544
545# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
546
547tarball-links-row-html() {
548 local version=$1
549
550 cat <<EOF
551<tr class="file-table-heading">
552 <td></td>
553 <td>File / SHA256 checksum</td>
554 <td class="size">Size</td>
555 <td></td>
556</tr>
557EOF
558
559 # we switched to .gz for oils-for-unix
560 # note: legacy names for old releases
561 for name in \
562 oils-for-unix-$version.tar.{gz,xz} \
563 oil-$version.tar.{gz,xz} \
564 oil-native-$version.tar.xz; do
565
566 local url="/download/$name" # The server URL
567 local path="../oilshell.org__deploy/download/$name"
568
569 # Don't show tarballs that don't exist
570 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
571 continue
572 fi
573 if [[ $name == oil-native-* && ! -f $path ]]; then
574 continue
575 fi
576
577 local checksum
578 checksum=$(sha256sum $path | awk '{print $1}')
579 local size
580 size=$(pretty-size $path)
581
582 # TODO: Port this to oil with "commas" extension.
583
584 # Three columns: date, version, and links
585 cat <<EOF
586 <tr>
587 <td></td>
588 <td class="filename"><a href="$url">$name</a></td>
589 <td class="size">$size</td>
590 </tr>
591 <tr>
592 <td></td>
593 <td colspan=2 class="checksum">$checksum</td>
594 </tr>
595EOF
596 done
597}
598
599this-release-links() {
600 echo '<div class="file-table">'
601 echo '<table>'
602 tarball-links-row-html "$OIL_VERSION"
603 echo '</table>'
604 echo '</div>'
605}
606
607# Turn HTML comment into a download link
608add-date-and-links() {
609 local snippet
610 snippet=$(this-release-links)
611
612 awk -v date=$1 -v snippet="$snippet" '
613 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
614 print(snippet)
615 next
616 }
617
618 /<!-- REPLACE_WITH_DATE -->/ {
619 print(date)
620 next
621 }
622
623 # Everything else
624 { print }
625 '
626}
627
628patch-release-pages() {
629 local release_date
630 release_date=$(cat _build/release-date.txt)
631
632 local root=_release/VERSION
633
634 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
635 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
636}
637
638copy-release-pages() {
639 ### For testing without releasing
640
641 cat < _tmp/release-index.html > $root/index.html
642 cat < _tmp/release-quality.html > $root/quality.html
643}
644
645run-for-release() {
646 ### Build a tree. Requires _build/release-date.txt to exist
647
648 local root=_release/VERSION
649 mkdir -p $root/{doc,test,pub}
650
651 tour
652
653 # Metadata
654 cp -v _build/release-date.txt oil-version.txt $root
655
656 # Docs
657 # Writes _release/VERSION and _tmp/release-index.html
658 all-markdown
659 all-ref
660 all-redirects # backward compat
661
662 patch-release-pages
663
664 write-metrics
665
666 # Problem: You can't preview it without .wwz!
667 # Maybe have local redirects VERSION/test/wild/ to
668 #
669 # Instead of linking, I should compress them all here.
670
671 copy-web
672
673 if command -v tree >/dev/null; then
674 tree $root
675 else
676 find $root
677 fi
678}
679
680soil-run() {
681 build/stamp.sh write-release-date
682
683 run-for-release
684}
685
686"$@"
687