1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Wait for artifacts on a web server.
|
4 | #
|
5 | # Usage:
|
6 | # soil/wait.sh
|
7 |
|
8 | set -o nounset
|
9 | set -o pipefail
|
10 | set -o errexit
|
11 |
|
12 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
13 | source soil/common.sh
|
14 |
|
15 | fast-curl-until-200() {
|
16 | ### Retry fetch until HTTP 200, REUSING curl process AND connection
|
17 |
|
18 | # Similar to
|
19 | # https://stackoverflow.com/questions/42873285/curl-retry-mechanism
|
20 | # --retry-all-errors is 7.71 !
|
21 |
|
22 | # --retry-all-errors not present in curl 7.58.0 on Ubuntu 18.04
|
23 | # Do we have to upgrade? We're using Debian buster-slim
|
24 | #
|
25 | # curl 7.64 ! Gah.
|
26 |
|
27 | # Curl versions
|
28 | #
|
29 | # 7.58 - Ubuntu 18.04, --retry-all-errors not present
|
30 | # 7.64 - Debian Buster slim, our container base
|
31 | # https://packages.debian.org/buster/curl
|
32 | # 7.71 - --retry-all-errors
|
33 | # 7.88 - Debian Bookworm
|
34 |
|
35 | local url=$1
|
36 | local out_path=$2
|
37 | local num_retries=${3:-10} # number of times through the loop
|
38 | local interval=${4:-10} # retry every n seconds
|
39 |
|
40 | mkdir -p "$(dirname $out_path)"
|
41 |
|
42 | # --retry-all-errors and --fail are used to make curl try on a 404
|
43 |
|
44 | # Note: might need --retry-conn-refused as well, in case the server
|
45 | # disconnects
|
46 |
|
47 | curl \
|
48 | --output $out_path \
|
49 | --max-time 10 \
|
50 | --retry $num_retries \
|
51 | --retry-all-errors \
|
52 | --fail \
|
53 | --retry-delay $interval \
|
54 | $url
|
55 | }
|
56 |
|
57 | curl-until-200() {
|
58 | ### bash version of the function above
|
59 |
|
60 | local url=$1
|
61 | local out_path=$2
|
62 | local num_retries=${3:-10} # number of times through the loop
|
63 | local interval=${4:-10} # retry every n seconds
|
64 |
|
65 | mkdir -p "$(dirname $out_path)"
|
66 |
|
67 | local i=0
|
68 | while true; do
|
69 | local http_code
|
70 | http_code=$(curl --output $out_path --write-out '%{http_code}' $url)
|
71 |
|
72 | if test "$http_code" = 200; then
|
73 | log "Curl wrote $out_path"
|
74 | ls -l $out_path
|
75 | break;
|
76 | fi
|
77 |
|
78 | log "HTTP status $http_code; retrying in $interval seconds"
|
79 | sleep $interval
|
80 |
|
81 | i=$(( i + 1 ))
|
82 | if test $i -eq $num_retries; then
|
83 | log "Giving up after $num_retries tries"
|
84 | return 1 # fail
|
85 | fi
|
86 | done
|
87 | }
|
88 |
|
89 | # Users
|
90 | # - test/wild.sh soil-run
|
91 | # - benchmarks/perf.sh in the raw-vm task
|
92 | # - test/ble.sh, in app-tests task
|
93 |
|
94 | for-cpp-tarball() {
|
95 | local prefix=${1:-github-}
|
96 |
|
97 | # There are now 3 tasks waiting for the cpp-tarball
|
98 | # wild - might want to wait 80 seconds
|
99 | # app-tests - run clone, build, bash first
|
100 | # raw-vm - install perf first
|
101 | #
|
102 | # So just wait 1 second
|
103 | local sleep_secs=${2:-1}
|
104 |
|
105 | # Retry for 18 times, every 10 seconds = 3 minutes.
|
106 |
|
107 | # If we have 10 clients, then we're hitting it once a second, which is not
|
108 | # unreasonable. We're also keeping 10 connections
|
109 |
|
110 | local num_retries=${3:-18}
|
111 | local interval=${4:-10}
|
112 |
|
113 | local git_commit_url
|
114 | git_commit_url=$(git-commit-url $prefix)
|
115 |
|
116 | local url="$git_commit_url/oils-for-unix.tar"
|
117 |
|
118 | set -x
|
119 | sleep $sleep_secs
|
120 |
|
121 | curl-until-200 $url _release/oils-for-unix.tar $num_retries $interval
|
122 | }
|
123 |
|
124 | readonly TEST_FILE='oilshell.org/tmp/curl-test'
|
125 |
|
126 | for-test-file() {
|
127 | curl-until-200 "http://www.$TEST_FILE" _tmp/$(basename $TEST_FILE) 5 10
|
128 | }
|
129 |
|
130 | touch-remote() {
|
131 | ssh oilshell.org "echo hi > $TEST_FILE"
|
132 | }
|
133 |
|
134 | rm-remote() {
|
135 | ssh oilshell.org "rm -v $TEST_FILE"
|
136 | }
|
137 |
|
138 | test-for-cpp-tarball() {
|
139 | for-cpp-tarball
|
140 | }
|
141 |
|
142 | "$@"
|