OILS / build / doc.sh View on Github | oilshell.org

690 lines, 367 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=. doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97
98 # Data language
99 qsn
100 qtt
101 j8-notation
102 pretty-printing
103 stream-table-process
104
105 doc-toolchain
106 doc-plugins
107 idioms
108 shell-idioms
109 ysh-faq
110
111 language-influences
112 ysh-vs-python
113 ysh-vs-shell
114
115 syntactic-concepts
116 syntax-feelings
117 command-vs-expression-mode
118
119 # needs polish
120 # Note: docs about the YSH are prefixed 'ysh-'.
121 # data-model and command-vs-expression-mode span both OSH and YSH
122
123 index
124 faq-doc
125
126 options
127
128 old/index
129 old/project-tour
130 old/legacy-array
131 old/ysh-keywords
132 old/modules
133 old/expression-language
134 old/word-language
135 old/errors
136 old/ysh-builtins
137
138 io-builtins
139 unicode
140 framing
141 xtrace
142 headless
143 completion
144 strings
145 variables
146
147 # Internal stuff
148 interpreter-state
149 process-model
150 architecture-notes
151 parser-architecture
152)
153
154# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
155# locale), which is loaded by Python into say u'\u5e74'. But the default
156# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
157# fail.
158#
159# I believe --rfc-e-mail should never output a Unicode character.
160#
161# A better fix would be to implement json_utf8.load(f), which doesn't decode
162# into unicode instances. This would remove useless conversions.
163
164readonly TIMESTAMP=$(date --rfc-email)
165
166split-and-render() {
167 local src=${1:-doc/known-differences.md}
168
169 local rel_path=${src%'.md'} # doc/known-differences
170 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
171
172 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
173 local web_url=${3:-'../web'}
174
175 mkdir -v -p $(dirname $out) $tmp_prefix
176
177 # Also add could add css_files. The one in the file takes precedence always?
178
179 # css_files: a space-separated list
180 # all_docs_url: so we link from doc/foo.html -> doc/
181
182 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
183
184 doctools/split_doc.py \
185 -v build_timestamp="$TIMESTAMP" \
186 -v oil_version="$OIL_VERSION" \
187 -v css_files="$css_files" \
188 -v all_docs_url='.' \
189 -v repo_url="$src" \
190 $src $tmp_prefix
191
192 #ls -l _tmp/doc
193 #head _tmp/doc/*
194 #return
195
196 # for ysh-tour code blocks
197 local code_out=_tmp/code-blocks/$rel_path.txt
198 mkdir -v -p $(dirname $code_out)
199
200 cmark \
201 --code-block-output $code_out \
202 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
203
204 log "$tmp_prefix -> (doctools/cmark) -> $out"
205}
206
207render-from-kate() {
208 ### Make it easier to configure Kate editor
209
210 # It want to pass an absolute path
211 # TODO: I can't figure out how to run this from Kate?
212
213 local full_path=$1
214
215 case $full_path in
216 $REPO_ROOT/*)
217 rel_path=${full_path#"$REPO_ROOT/"}
218 echo "relative path = $rel_path"
219 ;;
220 *)
221 die "$full_path should start with repo root $REPO_ROOT"
222 ;;
223 esac
224
225 split-and-render $rel_path
226}
227
228# Special case for README
229# Do NOT split because we don't want front matter in the markdown source.
230render-only() {
231 local src=${1:-README.md}
232
233 local name
234 case $src in
235 *.md)
236 name=$(basename $src .md)
237 ;;
238 *.txt)
239 name=$(basename $src .txt)
240 ;;
241 *)
242 name=$(basename $src)
243 ;;
244 esac
245
246 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
247 local css_files=${3:-'../web/manual.css ../web/toc.css'}
248 local title=${4:-'Oils Source Code'}
249
250 local prefix=_tmp/doc/$name
251
252 local meta=${prefix}_meta.json
253 cat >$meta <<EOF
254{ "title": "$title",
255 "repo_url": "$src",
256 "css_files": "$css_files",
257 "all_docs_url": ".",
258
259 "build_timestamp": "$TIMESTAMP",
260 "oil_version": "$OIL_VERSION"
261}
262EOF
263
264 cmark $meta $src > $out
265 log "Wrote $out"
266}
267
268special() {
269 # TODO: do all READMEs
270 split-and-render mycpp/README.md \
271 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
272 ../../../web
273
274 local web_dir='../../web'
275 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
276 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
277
278 local web_dir='../web'
279 render-only INSTALL.txt '' \
280 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
281
282 render-only INSTALL-old.txt '' \
283 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
284
285 # These pages aren't in doc/
286 split-and-render doc/release-index.md _tmp/release-index.html
287 split-and-render doc/release-quality.md _tmp/release-quality.html
288}
289
290all-markdown() {
291 make-dirs
292
293 # TODO: We can set repo_url here! Then we don't need it for most docs.
294 # split_doc.py can return {} if the doc doesn't start with ---
295
296 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
297 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
298 for d in "${MARKDOWN_DOCS[@]}"; do
299 split-and-render doc/$d.md
300 done
301
302 special
303}
304
305redir-body() {
306 local to_url=$1 # WARNING: no escaping
307 cat <<EOF
308<head>
309 <meta http-equiv="Refresh" content="0; URL=$to_url" />
310</head>
311EOF
312}
313
314redirect-pairs() {
315 # we want want /release/latest/ URLs to still work
316 cat <<EOF
317oil-language-tour ysh-tour
318oil-language-faq ysh-faq
319oil-help ysh-help
320oil-help-topics ysh-help-topics
321ysh-help ref/toc-ysh
322ysh-help-topics ref/toc-ysh
323EOF
324}
325
326all-redirects() {
327 redirect-pairs | while read -r from_page to_page; do
328 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
329 done
330}
331
332# TODO: This could use some CSS.
333man-page() {
334 local root_dir=${1:-_release/VERSION}
335 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
336 ls -l $root_dir
337}
338
339# I want to ship the INSTALL file literally, so just mutate things
340_sed-ext() {
341 sed --regexp-extended -i "$@"
342}
343
344update-src-versions() {
345 # Update tarball names, etc.
346 _sed-ext \
347 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
348 doc/release-*.md INSTALL.txt INSTALL-old.txt
349
350 # Update /release/0.8.4/ URL, etc.
351 _sed-ext \
352 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
353 doc/osh.1
354}
355
356#
357# Test Tools
358#
359
360split-doc-demo() {
361 cat > _tmp/testdoc.md <<EOF
362---
363title: foo
364---
365
366Title
367=====
368
369hello
370
371EOF
372
373 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
374
375 head _tmp/testdoc*
376}
377
378#
379# Help is both markdown and text
380#
381
382readonly TMP_DIR=_tmp/doc
383readonly CODE_BLOCK_DIR=_tmp/code-blocks
384readonly TEXT_DIR=_devbuild/help
385readonly HTML_DIR=_release/VERSION
386readonly CODE_DIR=_devbuild/gen
387
388cards-from-indices() {
389 ### Make help cards
390
391 for lang in osh ysh data; do
392 help-gen cards-from-index $lang $TEXT_DIR \
393 < $HTML_DIR/doc/ref/toc-$lang.html
394 done
395}
396
397cards-from-chapters() {
398 ### Turn h3 topics into cards
399
400 local py_out=$CODE_DIR/help_meta.py
401
402 mkdir -p _gen/frontend
403 local cc_prefix=_gen/frontend/help_meta
404
405 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
406 $HTML_DIR/doc/ref/chap-*.html
407}
408
409ref-check() {
410 help-gen ref-check \
411 doc/ref/toc-*.md \
412 _release/VERSION/doc/ref/chap-*.html
413}
414
415fmt-check() {
416 PYTHONPATH=. doctools/fmt_check.py _release/VERSION/doc/ref/*.html
417}
418
419
420write-metrics() {
421 ### Check indexes and chapters against each other
422
423 local out=_release/VERSION/doc/metrics.txt
424
425 # send stderr to the log file too
426 ref-check > $out 2>&1
427
428 echo "Wrote $out"
429}
430
431tour() {
432 ### Build the Tour of YSH, and execute code as validation
433 local name=${1:-ysh-tour}
434
435 split-and-render doc/$name.md
436
437 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
438
439 mkdir -p $work_dir/lib
440
441 # Files used by module example
442 touch $work_dir/{build,test}.sh
443
444 cat >$work_dir/lib/util.ysh <<EOF
445log() { echo "$@" 1>&2; }
446EOF
447
448 pushd $work_dir
449 $REPO_ROOT/bin/ysh $name.txt
450 popd
451
452 # My own dev tools
453 # if test -d ~/vm-shared; then
454 if false; then
455 local path=_release/VERSION/doc/$name.html
456 cp -v $path ~/vm-shared/$path
457 fi
458}
459
460one() {
461 ### Iterate on one doc quickly
462
463 local name=${1:-options}
464
465 split-and-render doc/$name.md
466
467 # Make sure the doc has valid YSH code?
468 # TODO: Maybe need an attribute for OSH or YSH
469 pushd _tmp/code-blocks/doc
470 $REPO_ROOT/bin/ysh $name.txt
471 popd
472
473 if test -d ~/vm-shared; then
474 local out="${name%.md}.html"
475 local path=_release/VERSION/$out
476 cp -v $path ~/vm-shared/$path
477 fi
478}
479
480make-dirs() {
481 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
482}
483
484one-ref() {
485 local md=${1:-doc/ref/index.md}
486 split-and-render $md '' '../../web'
487}
488
489all-ref() {
490 ### Build doc/ref in text and HTML. Depends on libcmark.so
491
492 log "Removing $TEXT_DIR/*"
493 rm -f $TEXT_DIR/*
494 make-dirs
495
496 # Make the indexes and chapters
497 for d in doc/ref/*.md; do
498 split-and-render $d '' '../../web'
499 done
500
501 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
502 # extract topics from all chapters first, and then make help_meta.json, like
503 # we have _devbuild/gen/help_meta.py.
504
505 # Text cards
506 cards-from-indices
507 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
508 cards-from-chapters
509
510 if command -v pysum; then
511 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
512 # compress it.
513 echo 'Size of embedded help:'
514 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
515 fi
516
517 # Better sorting
518 #LANG=C ls -l $TEXT_DIR
519}
520
521_copy-path() {
522 local src=$1 dest=$2
523 mkdir -p $(dirname $dest)
524 cp -v $src $dest
525}
526
527copy-web() {
528 find web \
529 \( -name _tmp -a -prune \) -o \
530 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
531 xargs -n 2 -- $0 _copy-path
532}
533
534pretty-size() {
535 local path=$1
536 stat --format '%s' "$path" | python -c '
537import sys
538num_bytes = int(sys.stdin.read())
539print "{:,}".format(num_bytes)
540'
541}
542
543# NOTE: It might be better to link to files like this in the /release/ tree.
544# Although I am not signing them.
545
546# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
547
548tarball-links-row-html() {
549 local version=$1
550
551 cat <<EOF
552<tr class="file-table-heading">
553 <td></td>
554 <td>File / SHA256 checksum</td>
555 <td class="size">Size</td>
556 <td></td>
557</tr>
558EOF
559
560 # we switched to .gz for oils-for-unix
561 # note: legacy names for old releases
562 for name in \
563 oils-for-unix-$version.tar.{gz,xz} \
564 oil-$version.tar.{gz,xz} \
565 oil-native-$version.tar.xz; do
566
567 local url="/download/$name" # The server URL
568 local path="../oilshell.org__deploy/download/$name"
569
570 # Don't show tarballs that don't exist
571 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
572 continue
573 fi
574 if [[ $name == oil-native-* && ! -f $path ]]; then
575 continue
576 fi
577
578 local checksum
579 checksum=$(sha256sum $path | awk '{print $1}')
580 local size
581 size=$(pretty-size $path)
582
583 # TODO: Port this to oil with "commas" extension.
584
585 # Three columns: date, version, and links
586 cat <<EOF
587 <tr>
588 <td></td>
589 <td class="filename"><a href="$url">$name</a></td>
590 <td class="size">$size</td>
591 </tr>
592 <tr>
593 <td></td>
594 <td colspan=2 class="checksum">$checksum</td>
595 </tr>
596EOF
597 done
598}
599
600this-release-links() {
601 echo '<div class="file-table">'
602 echo '<table>'
603 tarball-links-row-html "$OIL_VERSION"
604 echo '</table>'
605 echo '</div>'
606}
607
608# Turn HTML comment into a download link
609add-date-and-links() {
610 local snippet
611 snippet=$(this-release-links)
612
613 awk -v date=$1 -v snippet="$snippet" '
614 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
615 print(snippet)
616 next
617 }
618
619 /<!-- REPLACE_WITH_DATE -->/ {
620 print(date)
621 next
622 }
623
624 # Everything else
625 { print }
626 '
627}
628
629patch-release-pages() {
630 local release_date
631 release_date=$(cat _build/release-date.txt)
632
633 local root=_release/VERSION
634
635 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
636 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
637}
638
639copy-release-pages() {
640 ### For testing without releasing
641
642 cat < _tmp/release-index.html > $root/index.html
643 cat < _tmp/release-quality.html > $root/quality.html
644}
645
646run-for-release() {
647 ### Build a tree. Requires _build/release-date.txt to exist
648
649 local root=_release/VERSION
650 mkdir -p $root/{doc,test,pub}
651
652 tour
653
654 # Metadata
655 cp -v _build/release-date.txt oil-version.txt $root
656
657 # Docs
658 # Writes _release/VERSION and _tmp/release-index.html
659 all-markdown
660 all-ref
661 all-redirects # backward compat
662
663 fmt-check # Needs to run *after* we build the HTML
664
665 patch-release-pages
666
667 write-metrics
668
669 # Problem: You can't preview it without .wwz!
670 # Maybe have local redirects VERSION/test/wild/ to
671 #
672 # Instead of linking, I should compress them all here.
673
674 copy-web
675
676 if command -v tree >/dev/null; then
677 tree $root
678 else
679 find $root
680 fi
681}
682
683soil-run() {
684 build/stamp.sh write-release-date
685
686 run-for-release
687}
688
689"$@"
690