1 | #!/usr/bin/env bash
|
2 |
|
3 | # Copyright 2016 The Kubernetes Authors.
|
4 | #
|
5 | # Licensed under the Apache License, Version 2.0 (the "License");
|
6 | # you may not use this file except in compliance with the License.
|
7 | # You may obtain a copy of the License at
|
8 | #
|
9 | # http://www.apache.org/licenses/LICENSE-2.0
|
10 | #
|
11 | # Unless required by applicable law or agreed to in writing, software
|
12 | # distributed under the License is distributed on an "AS IS" BASIS,
|
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14 | # See the License for the specific language governing permissions and
|
15 | # limitations under the License.
|
16 |
|
17 | # This script is for configuring kubernetes master and node instances. It is
|
18 | # uploaded in the manifests tar ball.
|
19 |
|
20 | # TODO: this script duplicates templating logic from cluster/saltbase/salt
|
21 | # using sed. It should use an actual template parser on the manifest
|
22 | # files.
|
23 |
|
24 | set -o errexit
|
25 | set -o nounset
|
26 | set -o pipefail
|
27 |
|
28 | function setup-os-params {
|
29 | # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to
|
30 | # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for
|
31 | # now, set a generic core_pattern that users can work with.
|
32 | echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern
|
33 | }
|
34 |
|
35 | function config-ip-firewall {
|
36 | echo "Configuring IP firewall rules"
|
37 | # The GCI image has host firewall which drop most inbound/forwarded packets.
|
38 | # We need to add rules to accept all TCP/UDP/ICMP packets.
|
39 | if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
|
40 | echo "Add rules to accept all inbound TCP/UDP/ICMP packets"
|
41 | iptables -A INPUT -w -p TCP -j ACCEPT
|
42 | iptables -A INPUT -w -p UDP -j ACCEPT
|
43 | iptables -A INPUT -w -p ICMP -j ACCEPT
|
44 | fi
|
45 | if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
|
46 | echo "Add rules to accept all forwarded TCP/UDP/ICMP packets"
|
47 | iptables -A FORWARD -w -p TCP -j ACCEPT
|
48 | iptables -A FORWARD -w -p UDP -j ACCEPT
|
49 | iptables -A FORWARD -w -p ICMP -j ACCEPT
|
50 | fi
|
51 |
|
52 | iptables -N KUBE-METADATA-SERVER
|
53 | iptables -I FORWARD -p tcp -d 169.254.169.254 --dport 80 -j KUBE-METADATA-SERVER
|
54 |
|
55 | if [[ -n "${KUBE_FIREWALL_METADATA_SERVER:-}" ]]; then
|
56 | iptables -A KUBE-METADATA-SERVER -j DROP
|
57 | fi
|
58 | }
|
59 |
|
60 | function create-dirs {
|
61 | echo "Creating required directories"
|
62 | mkdir -p /var/lib/kubelet
|
63 | mkdir -p /etc/kubernetes/manifests
|
64 | if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then
|
65 | mkdir -p /var/lib/kube-proxy
|
66 | fi
|
67 | }
|
68 |
|
69 | # Formats the given device ($1) if needed and mounts it at given mount point
|
70 | # ($2).
|
71 | function safe-format-and-mount() {
|
72 | device=$1
|
73 | mountpoint=$2
|
74 |
|
75 | # Format only if the disk is not already formatted.
|
76 | if ! tune2fs -l "${device}" ; then
|
77 | echo "Formatting '${device}'"
|
78 | mkfs.ext4 -F "${device}"
|
79 | fi
|
80 |
|
81 | mkdir -p "${mountpoint}"
|
82 | echo "Mounting '${device}' at '${mountpoint}'"
|
83 | mount -o discard,defaults "${device}" "${mountpoint}"
|
84 | }
|
85 |
|
86 | # Local ssds, if present, are mounted at /mnt/disks/ssdN.
|
87 | function ensure-local-ssds() {
|
88 | for ssd in /dev/disk/by-id/google-local-ssd-*; do
|
89 | if [ -e "${ssd}" ]; then
|
90 | ssdnum=`echo ${ssd} | sed -e 's/\/dev\/disk\/by-id\/google-local-ssd-\([0-9]*\)/\1/'`
|
91 | ssdmount="/mnt/disks/ssd${ssdnum}/"
|
92 | mkdir -p ${ssdmount}
|
93 | safe-format-and-mount "${ssd}" ${ssdmount}
|
94 | echo "Mounted local SSD $ssd at ${ssdmount}"
|
95 | chmod a+w ${ssdmount}
|
96 | else
|
97 | echo "No local SSD disks found."
|
98 | fi
|
99 | done
|
100 | }
|
101 |
|
102 | # Installs logrotate configuration files
|
103 | function setup-logrotate() {
|
104 | mkdir -p /etc/logrotate.d/
|
105 | # Configure log rotation for all logs in /var/log, which is where k8s services
|
106 | # are configured to write their log files. Whenever logrotate is ran, this
|
107 | # config will:
|
108 | # * rotate the log file if its size is > 100Mb OR if one day has elapsed
|
109 | # * save rotated logs into a gzipped timestamped backup
|
110 | # * log file timestamp (controlled by 'dateformat') includes seconds too. This
|
111 | # ensures that logrotate can generate unique logfiles during each rotation
|
112 | # (otherwise it skips rotation if 'maxsize' is reached multiple times in a
|
113 | # day).
|
114 | # * keep only 5 old (rotated) logs, and will discard older logs.
|
115 | cat > /etc/logrotate.d/allvarlogs <<EOF
|
116 | /var/log/*.log {
|
117 | rotate ${LOGROTATE_FILES_MAX_COUNT:-5}
|
118 | copytruncate
|
119 | missingok
|
120 | notifempty
|
121 | compress
|
122 | maxsize ${LOGROTATE_MAX_SIZE:-100M}
|
123 | daily
|
124 | dateext
|
125 | dateformat -%Y%m%d-%s
|
126 | create 0644 root root
|
127 | }
|
128 | EOF
|
129 |
|
130 | }
|
131 |
|
132 | # Finds the master PD device; returns it in MASTER_PD_DEVICE
|
133 | function find-master-pd {
|
134 | MASTER_PD_DEVICE=""
|
135 | if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then
|
136 | return
|
137 | fi
|
138 | device_info=$(ls -l /dev/disk/by-id/google-master-pd)
|
139 | relative_path=${device_info##* }
|
140 | MASTER_PD_DEVICE="/dev/disk/by-id/${relative_path}"
|
141 | }
|
142 |
|
143 | # Mounts a persistent disk (formatting if needed) to store the persistent data
|
144 | # on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
145 | # safe-format-and-mount only formats an unformatted disk, and mkdir -p will
|
146 | # leave a directory be if it already exists.
|
147 | function mount-master-pd {
|
148 | find-master-pd
|
149 | if [[ -z "${MASTER_PD_DEVICE:-}" ]]; then
|
150 | return
|
151 | fi
|
152 |
|
153 | echo "Mounting master-pd"
|
154 | local -r pd_path="/dev/disk/by-id/google-master-pd"
|
155 | local -r mount_point="/mnt/disks/master-pd"
|
156 | # Format and mount the disk, create directories on it for all of the master's
|
157 | # persistent data, and link them to where they're used.
|
158 | mkdir -p "${mount_point}"
|
159 | safe-format-and-mount "${pd_path}" "${mount_point}"
|
160 | echo "Mounted master-pd '${pd_path}' at '${mount_point}'"
|
161 |
|
162 | # NOTE: These locations on the PD store persistent data, so to maintain
|
163 | # upgradeability, these locations should not change. If they do, take care
|
164 | # to maintain a migration path from these locations to whatever new
|
165 | # locations.
|
166 |
|
167 | # Contains all the data stored in etcd.
|
168 | mkdir -m 700 -p "${mount_point}/var/etcd"
|
169 | ln -s -f "${mount_point}/var/etcd" /var/etcd
|
170 | mkdir -p /etc/srv
|
171 | # Contains the dynamically generated apiserver auth certs and keys.
|
172 | mkdir -p "${mount_point}/srv/kubernetes"
|
173 | ln -s -f "${mount_point}/srv/kubernetes" /etc/srv/kubernetes
|
174 | # Directory for kube-apiserver to store SSH key (if necessary).
|
175 | mkdir -p "${mount_point}/srv/sshproxy"
|
176 | ln -s -f "${mount_point}/srv/sshproxy" /etc/srv/sshproxy
|
177 |
|
178 | if ! id etcd &>/dev/null; then
|
179 | useradd -s /sbin/nologin -d /var/etcd etcd
|
180 | fi
|
181 | chown -R etcd "${mount_point}/var/etcd"
|
182 | chgrp -R etcd "${mount_point}/var/etcd"
|
183 | }
|
184 |
|
185 | # append_or_replace_prefixed_line ensures:
|
186 | # 1. the specified file exists
|
187 | # 2. existing lines with the specified ${prefix} are removed
|
188 | # 3. a new line with the specified ${prefix}${suffix} is appended
|
189 | function append_or_replace_prefixed_line {
|
190 | local -r file="${1:-}"
|
191 | local -r prefix="${2:-}"
|
192 | local -r suffix="${3:-}"
|
193 | local -r dirname="$(dirname ${file})"
|
194 | local -r tmpfile="$(mktemp -t filtered.XXXX --tmpdir=${dirname})"
|
195 |
|
196 | touch "${file}"
|
197 | awk "substr(\$0,0,length(\"${prefix}\")) != \"${prefix}\" { print }" "${file}" > "${tmpfile}"
|
198 | echo "${prefix}${suffix}" >> "${tmpfile}"
|
199 | mv "${tmpfile}" "${file}"
|
200 | }
|
201 |
|
202 | function write-pki-data {
|
203 | local data="${1}"
|
204 | local path="${2}"
|
205 | (umask 077; echo "${data}" | base64 --decode > "${path}")
|
206 | }
|
207 |
|
208 | function create-node-pki {
|
209 | echo "Creating node pki files"
|
210 |
|
211 | local -r pki_dir="/etc/srv/kubernetes/pki"
|
212 | mkdir -p "${pki_dir}"
|
213 |
|
214 | if [[ -z "${CA_CERT_BUNDLE:-}" ]]; then
|
215 | CA_CERT_BUNDLE="${CA_CERT}"
|
216 | fi
|
217 |
|
218 | CA_CERT_BUNDLE_PATH="${pki_dir}/ca-certificates.crt"
|
219 | write-pki-data "${CA_CERT_BUNDLE}" "${CA_CERT_BUNDLE_PATH}"
|
220 |
|
221 | if [[ ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then
|
222 | KUBELET_CERT_PATH="${pki_dir}/kubelet.crt"
|
223 | write-pki-data "${KUBELET_CERT}" "${KUBELET_CERT_PATH}"
|
224 |
|
225 | KUBELET_KEY_PATH="${pki_dir}/kubelet.key"
|
226 | write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}"
|
227 | fi
|
228 |
|
229 | # TODO(mikedanese): remove this when we don't support downgrading to versions
|
230 | # < 1.6.
|
231 | ln -sf "${CA_CERT_BUNDLE_PATH}" /etc/srv/kubernetes/ca.crt
|
232 | }
|
233 |
|
234 | function create-master-pki {
|
235 | echo "Creating master pki files"
|
236 |
|
237 | local -r pki_dir="/etc/srv/kubernetes/pki"
|
238 | mkdir -p "${pki_dir}"
|
239 |
|
240 | CA_CERT_PATH="${pki_dir}/ca.crt"
|
241 | write-pki-data "${CA_CERT}" "${CA_CERT_PATH}"
|
242 |
|
243 | # this is not true on GKE
|
244 | if [[ ! -z "${CA_KEY:-}" ]]; then
|
245 | CA_KEY_PATH="${pki_dir}/ca.key"
|
246 | write-pki-data "${CA_KEY}" "${CA_KEY_PATH}"
|
247 | fi
|
248 |
|
249 | if [[ -z "${APISERVER_SERVER_CERT:-}" || -z "${APISERVER_SERVER_KEY:-}" ]]; then
|
250 | APISERVER_SERVER_CERT="${MASTER_CERT}"
|
251 | APISERVER_SERVER_KEY="${MASTER_KEY}"
|
252 | fi
|
253 |
|
254 | APISERVER_SERVER_CERT_PATH="${pki_dir}/apiserver.crt"
|
255 | write-pki-data "${APISERVER_SERVER_CERT}" "${APISERVER_SERVER_CERT_PATH}"
|
256 |
|
257 | APISERVER_SERVER_KEY_PATH="${pki_dir}/apiserver.key"
|
258 | write-pki-data "${APISERVER_SERVER_KEY}" "${APISERVER_SERVER_KEY_PATH}"
|
259 |
|
260 | if [[ -z "${APISERVER_CLIENT_CERT:-}" || -z "${APISERVER_CLIENT_KEY:-}" ]]; then
|
261 | APISERVER_CLIENT_CERT="${KUBEAPISERVER_CERT}"
|
262 | APISERVER_CLIENT_KEY="${KUBEAPISERVER_KEY}"
|
263 | fi
|
264 |
|
265 | APISERVER_CLIENT_CERT_PATH="${pki_dir}/apiserver-client.crt"
|
266 | write-pki-data "${APISERVER_CLIENT_CERT}" "${APISERVER_CLIENT_CERT_PATH}"
|
267 |
|
268 | APISERVER_CLIENT_KEY_PATH="${pki_dir}/apiserver-client.key"
|
269 | write-pki-data "${APISERVER_CLIENT_KEY}" "${APISERVER_CLIENT_KEY_PATH}"
|
270 |
|
271 | if [[ -z "${SERVICEACCOUNT_CERT:-}" || -z "${SERVICEACCOUNT_KEY:-}" ]]; then
|
272 | SERVICEACCOUNT_CERT="${MASTER_CERT}"
|
273 | SERVICEACCOUNT_KEY="${MASTER_KEY}"
|
274 | fi
|
275 |
|
276 | SERVICEACCOUNT_CERT_PATH="${pki_dir}/serviceaccount.crt"
|
277 | write-pki-data "${SERVICEACCOUNT_CERT}" "${SERVICEACCOUNT_CERT_PATH}"
|
278 |
|
279 | SERVICEACCOUNT_KEY_PATH="${pki_dir}/serviceaccount.key"
|
280 | write-pki-data "${SERVICEACCOUNT_KEY}" "${SERVICEACCOUNT_KEY_PATH}"
|
281 |
|
282 | # TODO(mikedanese): remove this when we don't support downgrading to versions
|
283 | # < 1.6.
|
284 | ln -sf "${APISERVER_SERVER_KEY_PATH}" /etc/srv/kubernetes/server.key
|
285 | ln -sf "${APISERVER_SERVER_CERT_PATH}" /etc/srv/kubernetes/server.cert
|
286 |
|
287 | if [[ ! -z "${REQUESTHEADER_CA_CERT:-}" ]]; then
|
288 | AGGREGATOR_CA_KEY_PATH="${pki_dir}/aggr_ca.key"
|
289 | write-pki-data "${AGGREGATOR_CA_KEY}" "${AGGREGATOR_CA_KEY_PATH}"
|
290 |
|
291 | REQUESTHEADER_CA_CERT_PATH="${pki_dir}/aggr_ca.crt"
|
292 | write-pki-data "${REQUESTHEADER_CA_CERT}" "${REQUESTHEADER_CA_CERT_PATH}"
|
293 |
|
294 | PROXY_CLIENT_KEY_PATH="${pki_dir}/proxy_client.key"
|
295 | write-pki-data "${PROXY_CLIENT_KEY}" "${PROXY_CLIENT_KEY_PATH}"
|
296 |
|
297 | PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt"
|
298 | write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}"
|
299 | fi
|
300 | }
|
301 |
|
302 | # After the first boot and on upgrade, these files exist on the master-pd
|
303 | # and should never be touched again (except perhaps an additional service
|
304 | # account, see NB below.) One exception is if METADATA_CLOBBERS_CONFIG is
|
305 | # enabled. In that case the basic_auth.csv file will be rewritten to make
|
306 | # sure it matches the metadata source of truth.
|
307 | function create-master-auth {
|
308 | echo "Creating master auth files"
|
309 | local -r auth_dir="/etc/srv/kubernetes"
|
310 | local -r basic_auth_csv="${auth_dir}/basic_auth.csv"
|
311 | if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then
|
312 | if [[ -e "${basic_auth_csv}" && "${METADATA_CLOBBERS_CONFIG:-false}" == "true" ]]; then
|
313 | # If METADATA_CLOBBERS_CONFIG is true, we want to rewrite the file
|
314 | # completely, because if we're changing KUBE_USER and KUBE_PASSWORD, we
|
315 | # have nothing to match on. The file is replaced just below with
|
316 | # append_or_replace_prefixed_line.
|
317 | rm "${basic_auth_csv}"
|
318 | fi
|
319 | append_or_replace_prefixed_line "${basic_auth_csv}" "${KUBE_PASSWORD},${KUBE_USER}," "admin,system:masters"
|
320 | fi
|
321 |
|
322 | local -r known_tokens_csv="${auth_dir}/known_tokens.csv"
|
323 | if [[ -e "${known_tokens_csv}" && "${METADATA_CLOBBERS_CONFIG:-false}" == "true" ]]; then
|
324 | rm "${known_tokens_csv}"
|
325 | fi
|
326 | if [[ -n "${KUBE_BEARER_TOKEN:-}" ]]; then
|
327 | append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_BEARER_TOKEN}," "admin,admin,system:masters"
|
328 | fi
|
329 | if [[ -n "${KUBE_CONTROLLER_MANAGER_TOKEN:-}" ]]; then
|
330 | append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_CONTROLLER_MANAGER_TOKEN}," "system:kube-controller-manager,uid:system:kube-controller-manager"
|
331 | fi
|
332 | if [[ -n "${KUBE_SCHEDULER_TOKEN:-}" ]]; then
|
333 | append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_SCHEDULER_TOKEN}," "system:kube-scheduler,uid:system:kube-scheduler"
|
334 | fi
|
335 | if [[ -n "${KUBE_PROXY_TOKEN:-}" ]]; then
|
336 | append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_PROXY_TOKEN}," "system:kube-proxy,uid:kube_proxy"
|
337 | fi
|
338 | if [[ -n "${NODE_PROBLEM_DETECTOR_TOKEN:-}" ]]; then
|
339 | append_or_replace_prefixed_line "${known_tokens_csv}" "${NODE_PROBLEM_DETECTOR_TOKEN}," "system:node-problem-detector,uid:node-problem-detector"
|
340 | fi
|
341 | local use_cloud_config="false"
|
342 | cat <<EOF >/etc/gce.conf
|
343 | [global]
|
344 | EOF
|
345 | if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then
|
346 | cat <<EOF >>/etc/gce.conf
|
347 | api-endpoint = ${GCE_API_ENDPOINT}
|
348 | EOF
|
349 | fi
|
350 | if [[ -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" ]]; then
|
351 | use_cloud_config="true"
|
352 | cat <<EOF >>/etc/gce.conf
|
353 | token-url = ${TOKEN_URL}
|
354 | token-body = ${TOKEN_BODY}
|
355 | EOF
|
356 | fi
|
357 | if [[ -n "${PROJECT_ID:-}" ]]; then
|
358 | use_cloud_config="true"
|
359 | cat <<EOF >>/etc/gce.conf
|
360 | project-id = ${PROJECT_ID}
|
361 | EOF
|
362 | fi
|
363 | if [[ -n "${NETWORK_PROJECT_ID:-}" ]]; then
|
364 | use_cloud_config="true"
|
365 | cat <<EOF >>/etc/gce.conf
|
366 | network-project-id = ${NETWORK_PROJECT_ID}
|
367 | EOF
|
368 | fi
|
369 | if [[ -n "${NODE_NETWORK:-}" ]]; then
|
370 | use_cloud_config="true"
|
371 | cat <<EOF >>/etc/gce.conf
|
372 | network-name = ${NODE_NETWORK}
|
373 | EOF
|
374 | fi
|
375 | if [[ -n "${NODE_SUBNETWORK:-}" ]]; then
|
376 | use_cloud_config="true"
|
377 | cat <<EOF >>/etc/gce.conf
|
378 | subnetwork-name = ${NODE_SUBNETWORK}
|
379 | EOF
|
380 | fi
|
381 | if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then
|
382 | use_cloud_config="true"
|
383 | if [[ -n "${NODE_TAGS:-}" ]]; then
|
384 | local -r node_tags="${NODE_TAGS}"
|
385 | else
|
386 | local -r node_tags="${NODE_INSTANCE_PREFIX}"
|
387 | fi
|
388 | cat <<EOF >>/etc/gce.conf
|
389 | node-tags = ${node_tags}
|
390 | node-instance-prefix = ${NODE_INSTANCE_PREFIX}
|
391 | EOF
|
392 | fi
|
393 | if [[ -n "${MULTIZONE:-}" ]]; then
|
394 | use_cloud_config="true"
|
395 | cat <<EOF >>/etc/gce.conf
|
396 | multizone = ${MULTIZONE}
|
397 | EOF
|
398 | fi
|
399 | if [[ -n "${GCE_ALPHA_FEATURES:-}" ]]; then
|
400 | use_cloud_config="true"
|
401 | cat <<EOF >>/etc/gce.conf
|
402 | alpha-features = ${GCE_ALPHA_FEATURES}
|
403 | EOF
|
404 | fi
|
405 | if [[ -n "${SECONDARY_RANGE_NAME:-}" ]]; then
|
406 | use_cloud_config="true"
|
407 | cat <<EOF >> /etc/gce.conf
|
408 | secondary-range-name = ${SECONDARY-RANGE-NAME}
|
409 | EOF
|
410 | fi
|
411 | if [[ "${use_cloud_config}" != "true" ]]; then
|
412 | rm -f /etc/gce.conf
|
413 | fi
|
414 |
|
415 | if [[ -n "${GCP_AUTHN_URL:-}" ]]; then
|
416 | cat <<EOF >/etc/gcp_authn.config
|
417 | clusters:
|
418 | - name: gcp-authentication-server
|
419 | cluster:
|
420 | server: ${GCP_AUTHN_URL}
|
421 | users:
|
422 | - name: kube-apiserver
|
423 | user:
|
424 | auth-provider:
|
425 | name: gcp
|
426 | current-context: webhook
|
427 | contexts:
|
428 | - context:
|
429 | cluster: gcp-authentication-server
|
430 | user: kube-apiserver
|
431 | name: webhook
|
432 | EOF
|
433 | fi
|
434 |
|
435 | if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then
|
436 | cat <<EOF >/etc/gcp_authz.config
|
437 | clusters:
|
438 | - name: gcp-authorization-server
|
439 | cluster:
|
440 | server: ${GCP_AUTHZ_URL}
|
441 | users:
|
442 | - name: kube-apiserver
|
443 | user:
|
444 | auth-provider:
|
445 | name: gcp
|
446 | current-context: webhook
|
447 | contexts:
|
448 | - context:
|
449 | cluster: gcp-authorization-server
|
450 | user: kube-apiserver
|
451 | name: webhook
|
452 | EOF
|
453 | fi
|
454 |
|
455 | if [[ -n "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then
|
456 | # This is the config file for the image review webhook.
|
457 | cat <<EOF >/etc/gcp_image_review.config
|
458 | clusters:
|
459 | - name: gcp-image-review-server
|
460 | cluster:
|
461 | server: ${GCP_IMAGE_VERIFICATION_URL}
|
462 | users:
|
463 | - name: kube-apiserver
|
464 | user:
|
465 | auth-provider:
|
466 | name: gcp
|
467 | current-context: webhook
|
468 | contexts:
|
469 | - context:
|
470 | cluster: gcp-image-review-server
|
471 | user: kube-apiserver
|
472 | name: webhook
|
473 | EOF
|
474 | # This is the config for the image review admission controller.
|
475 | cat <<EOF >/etc/admission_controller.config
|
476 | imagePolicy:
|
477 | kubeConfigFile: /etc/gcp_image_review.config
|
478 | allowTTL: 30
|
479 | denyTTL: 30
|
480 | retryBackoff: 500
|
481 | defaultAllow: true
|
482 | EOF
|
483 | fi
|
484 | }
|
485 |
|
486 | # Write the config for the audit policy.
|
487 | function create-master-audit-policy {
|
488 | local -r path="${1}"
|
489 | local -r policy="${2:-}"
|
490 |
|
491 | if [[ -n "${policy}" ]]; then
|
492 | echo "${policy}" > "${path}"
|
493 | return
|
494 | fi
|
495 |
|
496 | # Known api groups
|
497 | local -r known_apis='
|
498 | - group: "" # core
|
499 | - group: "admissionregistration.k8s.io"
|
500 | - group: "apiextensions.k8s.io"
|
501 | - group: "apiregistration.k8s.io"
|
502 | - group: "apps"
|
503 | - group: "authentication.k8s.io"
|
504 | - group: "authorization.k8s.io"
|
505 | - group: "autoscaling"
|
506 | - group: "batch"
|
507 | - group: "certificates.k8s.io"
|
508 | - group: "extensions"
|
509 | - group: "metrics.k8s.io"
|
510 | - group: "networking.k8s.io"
|
511 | - group: "policy"
|
512 | - group: "rbac.authorization.k8s.io"
|
513 | - group: "settings.k8s.io"
|
514 | - group: "storage.k8s.io"'
|
515 |
|
516 | cat <<EOF >"${path}"
|
517 | apiVersion: audit.k8s.io/v1beta1
|
518 | kind: Policy
|
519 | rules:
|
520 | # The following requests were manually identified as high-volume and low-risk,
|
521 | # so drop them.
|
522 | - level: None
|
523 | users: ["system:kube-proxy"]
|
524 | verbs: ["watch"]
|
525 | resources:
|
526 | - group: "" # core
|
527 | resources: ["endpoints", "services", "services/status"]
|
528 | - level: None
|
529 | # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
|
530 | # TODO(#46983): Change this to the ingress controller service account.
|
531 | users: ["system:unsecured"]
|
532 | namespaces: ["kube-system"]
|
533 | verbs: ["get"]
|
534 | resources:
|
535 | - group: "" # core
|
536 | resources: ["configmaps"]
|
537 | - level: None
|
538 | users: ["kubelet"] # legacy kubelet identity
|
539 | verbs: ["get"]
|
540 | resources:
|
541 | - group: "" # core
|
542 | resources: ["nodes", "nodes/status"]
|
543 | - level: None
|
544 | userGroups: ["system:nodes"]
|
545 | verbs: ["get"]
|
546 | resources:
|
547 | - group: "" # core
|
548 | resources: ["nodes", "nodes/status"]
|
549 | - level: None
|
550 | users:
|
551 | - system:kube-controller-manager
|
552 | - system:kube-scheduler
|
553 | - system:serviceaccount:kube-system:endpoint-controller
|
554 | verbs: ["get", "update"]
|
555 | namespaces: ["kube-system"]
|
556 | resources:
|
557 | - group: "" # core
|
558 | resources: ["endpoints"]
|
559 | - level: None
|
560 | users: ["system:apiserver"]
|
561 | verbs: ["get"]
|
562 | resources:
|
563 | - group: "" # core
|
564 | resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
|
565 | # Don't log HPA fetching metrics.
|
566 | - level: None
|
567 | users:
|
568 | - system:kube-controller-manager
|
569 | verbs: ["get", "list"]
|
570 | resources:
|
571 | - group: "metrics.k8s.io"
|
572 |
|
573 | # Don't log these read-only URLs.
|
574 | - level: None
|
575 | nonResourceURLs:
|
576 | - /healthz*
|
577 | - /version
|
578 | - /swagger*
|
579 |
|
580 | # Don't log events requests.
|
581 | - level: None
|
582 | resources:
|
583 | - group: "" # core
|
584 | resources: ["events"]
|
585 |
|
586 | # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
|
587 | - level: Request
|
588 | users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
|
589 | verbs: ["update","patch"]
|
590 | resources:
|
591 | - group: "" # core
|
592 | resources: ["nodes/status", "pods/status"]
|
593 | omitStages:
|
594 | - "RequestReceived"
|
595 | - level: Request
|
596 | userGroups: ["system:nodes"]
|
597 | verbs: ["update","patch"]
|
598 | resources:
|
599 | - group: "" # core
|
600 | resources: ["nodes/status", "pods/status"]
|
601 | omitStages:
|
602 | - "RequestReceived"
|
603 |
|
604 | # deletecollection calls can be large, don't log responses for expected namespace deletions
|
605 | - level: Request
|
606 | users: ["system:serviceaccount:kube-system:namespace-controller"]
|
607 | verbs: ["deletecollection"]
|
608 | omitStages:
|
609 | - "RequestReceived"
|
610 |
|
611 | # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
|
612 | # so only log at the Metadata level.
|
613 | - level: Metadata
|
614 | resources:
|
615 | - group: "" # core
|
616 | resources: ["secrets", "configmaps"]
|
617 | - group: authentication.k8s.io
|
618 | resources: ["tokenreviews"]
|
619 | omitStages:
|
620 | - "RequestReceived"
|
621 | # Get repsonses can be large; skip them.
|
622 | - level: Request
|
623 | verbs: ["get", "list", "watch"]
|
624 | resources: ${known_apis}
|
625 | omitStages:
|
626 | - "RequestReceived"
|
627 | # Default level for known APIs
|
628 | - level: RequestResponse
|
629 | resources: ${known_apis}
|
630 | omitStages:
|
631 | - "RequestReceived"
|
632 | # Default level for all other requests.
|
633 | - level: Metadata
|
634 | omitStages:
|
635 | - "RequestReceived"
|
636 | EOF
|
637 | }
|
638 |
|
639 | # Writes the configuration file used by the webhook advanced auditing backend.
|
640 | function create-master-audit-webhook-config {
|
641 | local -r path="${1}"
|
642 |
|
643 | if [[ -n "${GCP_AUDIT_URL:-}" ]]; then
|
644 | # The webhook config file is a kubeconfig file describing the webhook endpoint.
|
645 | cat <<EOF >"${path}"
|
646 | clusters:
|
647 | - name: gcp-audit-server
|
648 | cluster:
|
649 | server: ${GCP_AUDIT_URL}
|
650 | users:
|
651 | - name: kube-apiserver
|
652 | user:
|
653 | auth-provider:
|
654 | name: gcp
|
655 | current-context: webhook
|
656 | contexts:
|
657 | - context:
|
658 | cluster: gcp-audit-server
|
659 | user: kube-apiserver
|
660 | name: webhook
|
661 | EOF
|
662 | fi
|
663 | }
|
664 |
|
665 | # Arg 1: the IP address of the API server
|
666 | function create-kubelet-kubeconfig() {
|
667 | local apiserver_address="${1}"
|
668 | if [[ -z "${apiserver_address}" ]]; then
|
669 | echo "Must provide API server address to create Kubelet kubeconfig file!"
|
670 | exit 1
|
671 | fi
|
672 | echo "Creating kubelet kubeconfig file"
|
673 | cat <<EOF >/var/lib/kubelet/bootstrap-kubeconfig
|
674 | apiVersion: v1
|
675 | kind: Config
|
676 | users:
|
677 | - name: kubelet
|
678 | user:
|
679 | client-certificate: ${KUBELET_CERT_PATH}
|
680 | client-key: ${KUBELET_KEY_PATH}
|
681 | clusters:
|
682 | - name: local
|
683 | cluster:
|
684 | server: https://${apiserver_address}
|
685 | certificate-authority: ${CA_CERT_BUNDLE_PATH}
|
686 | contexts:
|
687 | - context:
|
688 | cluster: local
|
689 | user: kubelet
|
690 | name: service-account-context
|
691 | current-context: service-account-context
|
692 | EOF
|
693 | }
|
694 |
|
695 | # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
|
696 | # to generate a kubeconfig file for the kubelet to securely connect to the apiserver.
|
697 | # Set REGISTER_MASTER_KUBELET to true if kubelet on the master node
|
698 | # should register to the apiserver.
|
699 | function create-master-kubelet-auth {
|
700 | # Only configure the kubelet on the master if the required variables are
|
701 | # set in the environment.
|
702 | if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then
|
703 | REGISTER_MASTER_KUBELET="true"
|
704 | create-kubelet-kubeconfig ${KUBELET_APISERVER}
|
705 | fi
|
706 | }
|
707 |
|
708 | function create-kubeproxy-user-kubeconfig {
|
709 | echo "Creating kube-proxy user kubeconfig file"
|
710 | cat <<EOF >/var/lib/kube-proxy/kubeconfig
|
711 | apiVersion: v1
|
712 | kind: Config
|
713 | users:
|
714 | - name: kube-proxy
|
715 | user:
|
716 | token: ${KUBE_PROXY_TOKEN}
|
717 | clusters:
|
718 | - name: local
|
719 | cluster:
|
720 | certificate-authority-data: ${CA_CERT_BUNDLE}
|
721 | contexts:
|
722 | - context:
|
723 | cluster: local
|
724 | user: kube-proxy
|
725 | name: service-account-context
|
726 | current-context: service-account-context
|
727 | EOF
|
728 | }
|
729 |
|
730 | function create-kubecontrollermanager-kubeconfig {
|
731 | echo "Creating kube-controller-manager kubeconfig file"
|
732 | mkdir -p /etc/srv/kubernetes/kube-controller-manager
|
733 | cat <<EOF >/etc/srv/kubernetes/kube-controller-manager/kubeconfig
|
734 | apiVersion: v1
|
735 | kind: Config
|
736 | users:
|
737 | - name: kube-controller-manager
|
738 | user:
|
739 | token: ${KUBE_CONTROLLER_MANAGER_TOKEN}
|
740 | clusters:
|
741 | - name: local
|
742 | cluster:
|
743 | insecure-skip-tls-verify: true
|
744 | server: https://localhost:443
|
745 | contexts:
|
746 | - context:
|
747 | cluster: local
|
748 | user: kube-controller-manager
|
749 | name: service-account-context
|
750 | current-context: service-account-context
|
751 | EOF
|
752 | }
|
753 |
|
754 | function create-kubescheduler-kubeconfig {
|
755 | echo "Creating kube-scheduler kubeconfig file"
|
756 | mkdir -p /etc/srv/kubernetes/kube-scheduler
|
757 | cat <<EOF >/etc/srv/kubernetes/kube-scheduler/kubeconfig
|
758 | apiVersion: v1
|
759 | kind: Config
|
760 | users:
|
761 | - name: kube-scheduler
|
762 | user:
|
763 | token: ${KUBE_SCHEDULER_TOKEN}
|
764 | clusters:
|
765 | - name: local
|
766 | cluster:
|
767 | insecure-skip-tls-verify: true
|
768 | server: https://localhost:443
|
769 | contexts:
|
770 | - context:
|
771 | cluster: local
|
772 | user: kube-scheduler
|
773 | name: kube-scheduler
|
774 | current-context: kube-scheduler
|
775 | EOF
|
776 | }
|
777 |
|
778 | function create-node-problem-detector-kubeconfig {
|
779 | echo "Creating node-problem-detector kubeconfig file"
|
780 | mkdir -p /var/lib/node-problem-detector
|
781 | cat <<EOF >/var/lib/node-problem-detector/kubeconfig
|
782 | apiVersion: v1
|
783 | kind: Config
|
784 | users:
|
785 | - name: node-problem-detector
|
786 | user:
|
787 | token: ${NODE_PROBLEM_DETECTOR_TOKEN}
|
788 | clusters:
|
789 | - name: local
|
790 | cluster:
|
791 | certificate-authority-data: ${CA_CERT}
|
792 | contexts:
|
793 | - context:
|
794 | cluster: local
|
795 | user: node-problem-detector
|
796 | name: service-account-context
|
797 | current-context: service-account-context
|
798 | EOF
|
799 | }
|
800 |
|
801 | function create-master-etcd-auth {
|
802 | if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
|
803 | local -r auth_dir="/etc/srv/kubernetes"
|
804 | echo "${ETCD_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-ca.crt"
|
805 | echo "${ETCD_PEER_KEY}" | base64 --decode > "${auth_dir}/etcd-peer.key"
|
806 | echo "${ETCD_PEER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-peer.crt"
|
807 | fi
|
808 | }
|
809 |
|
810 | function assemble-docker-flags {
|
811 | echo "Assemble docker command line flags"
|
812 | local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
|
813 | if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
|
814 | docker_opts+=" --log-level=debug"
|
815 | else
|
816 | docker_opts+=" --log-level=warn"
|
817 | fi
|
818 | local use_net_plugin="true"
|
819 | if [[ "${NETWORK_PROVIDER:-}" == "kubenet" || "${NETWORK_PROVIDER:-}" == "cni" ]]; then
|
820 | # set docker0 cidr to private ip address range to avoid conflict with cbr0 cidr range
|
821 | docker_opts+=" --bip=169.254.123.1/24"
|
822 | else
|
823 | use_net_plugin="false"
|
824 | docker_opts+=" --bridge=cbr0"
|
825 | fi
|
826 |
|
827 | # Decide whether to enable a docker registry mirror. This is taken from
|
828 | # the "kube-env" metadata value.
|
829 | if [[ -n "${DOCKER_REGISTRY_MIRROR_URL:-}" ]]; then
|
830 | echo "Enable docker registry mirror at: ${DOCKER_REGISTRY_MIRROR_URL}"
|
831 | docker_opts+=" --registry-mirror=${DOCKER_REGISTRY_MIRROR_URL}"
|
832 | fi
|
833 |
|
834 | # Configure docker logging
|
835 | docker_opts+=" --log-driver=${DOCKER_LOG_DRIVER:-json-file}"
|
836 | docker_opts+=" --log-opt=max-size=${DOCKER_LOG_MAX_SIZE:-10m}"
|
837 | docker_opts+=" --log-opt=max-file=${DOCKER_LOG_MAX_FILE:-5}"
|
838 |
|
839 | echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
|
840 |
|
841 | if [[ "${use_net_plugin}" == "true" ]]; then
|
842 | # If using a network plugin, extend the docker configuration to always remove
|
843 | # the network checkpoint to avoid corrupt checkpoints.
|
844 | # (https://github.com/docker/docker/issues/18283).
|
845 | echo "Extend the docker.service configuration to remove the network checkpiont"
|
846 | mkdir -p /etc/systemd/system/docker.service.d
|
847 | cat <<EOF >/etc/systemd/system/docker.service.d/01network.conf
|
848 | [Service]
|
849 | ExecStartPre=/bin/sh -x -c "rm -rf /var/lib/docker/network"
|
850 | EOF
|
851 | fi
|
852 |
|
853 | # Ensure TasksMax is sufficient for docker.
|
854 | # (https://github.com/kubernetes/kubernetes/issues/51977)
|
855 | echo "Extend the docker.service configuration to set a higher pids limit"
|
856 | mkdir -p /etc/systemd/system/docker.service.d
|
857 | cat <<EOF >/etc/systemd/system/docker.service.d/02tasksmax.conf
|
858 | [Service]
|
859 | TasksMax=infinity
|
860 | EOF
|
861 |
|
862 | systemctl daemon-reload
|
863 | echo "Docker command line is updated. Restart docker to pick it up"
|
864 | systemctl restart docker
|
865 | }
|
866 |
|
867 | # This function assembles the kubelet systemd service file and starts it
|
868 | # using systemctl.
|
869 | function start-kubelet {
|
870 | echo "Start kubelet"
|
871 |
|
872 | local -r kubelet_cert_dir="/var/lib/kubelet/pki/"
|
873 | mkdir -p "${kubelet_cert_dir}"
|
874 |
|
875 | local kubelet_bin="${KUBE_HOME}/bin/kubelet"
|
876 | local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")"
|
877 | local -r builtin_kubelet="/usr/bin/kubelet"
|
878 | if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
|
879 | # Determine which binary to use on test clusters. We use the built-in
|
880 | # version only if the downloaded version is the same as the built-in
|
881 | # version. This allows GCI to run some of the e2e tests to qualify the
|
882 | # built-in kubelet.
|
883 | if [[ -x "${builtin_kubelet}" ]]; then
|
884 | local -r builtin_version="$("${builtin_kubelet}" --version=true | cut -f2 -d " ")"
|
885 | if [[ "${builtin_version}" == "${version}" ]]; then
|
886 | kubelet_bin="${builtin_kubelet}"
|
887 | fi
|
888 | fi
|
889 | fi
|
890 | echo "Using kubelet binary at ${kubelet_bin}"
|
891 | local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
|
892 | flags+=" --allow-privileged=true"
|
893 | flags+=" --cgroup-root=/"
|
894 | flags+=" --cloud-provider=gce"
|
895 | flags+=" --cluster-dns=${DNS_SERVER_IP}"
|
896 | flags+=" --cluster-domain=${DNS_DOMAIN}"
|
897 | flags+=" --pod-manifest-path=/etc/kubernetes/manifests"
|
898 | flags+=" --experimental-mounter-path=${CONTAINERIZED_MOUNTER_HOME}/mounter"
|
899 | flags+=" --experimental-check-node-capabilities-before-mount=true"
|
900 | flags+=" --cert-dir=${kubelet_cert_dir}"
|
901 |
|
902 | if [[ -n "${KUBELET_PORT:-}" ]]; then
|
903 | flags+=" --port=${KUBELET_PORT}"
|
904 | fi
|
905 | if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
906 | flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
|
907 | flags+=" --enable-debugging-handlers=false"
|
908 | flags+=" --hairpin-mode=none"
|
909 | if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
|
910 | #TODO(mikedanese): allow static pods to start before creating a client
|
911 | #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
912 | #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
|
913 | flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
914 | flags+=" --register-schedulable=false"
|
915 | else
|
916 | # Standalone mode (not widely used?)
|
917 | flags+=" --pod-cidr=${MASTER_IP_RANGE}"
|
918 | fi
|
919 | else # For nodes
|
920 | flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
|
921 | flags+=" --enable-debugging-handlers=true"
|
922 | flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
|
923 | flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
|
924 | if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
|
925 | [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
|
926 | [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
|
927 | flags+=" --hairpin-mode=${HAIRPIN_MODE}"
|
928 | fi
|
929 | flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=${CA_CERT_BUNDLE_PATH}"
|
930 | fi
|
931 | # Network plugin
|
932 | if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then
|
933 | flags+=" --cni-bin-dir=/home/kubernetes/bin"
|
934 | if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
|
935 | # Calico uses CNI always.
|
936 | if [[ "${KUBERNETES_PRIVATE_MASTER:-}" == "true" ]]; then
|
937 | flags+=" --network-plugin=${NETWORK_PROVIDER}"
|
938 | else
|
939 | flags+=" --network-plugin=cni"
|
940 | fi
|
941 | else
|
942 | # Otherwise use the configured value.
|
943 | flags+=" --network-plugin=${NETWORK_PROVIDER}"
|
944 | fi
|
945 | fi
|
946 | if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then
|
947 | flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
|
948 | fi
|
949 | # FlexVolume plugin
|
950 | if [[ -n "${VOLUME_PLUGIN_DIR:-}" ]]; then
|
951 | flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
|
952 | fi
|
953 | if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then
|
954 | flags+=" --manifest-url=${MANIFEST_URL}"
|
955 | flags+=" --manifest-url-header=${MANIFEST_URL_HEADER}"
|
956 | fi
|
957 | if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then
|
958 | flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
|
959 | fi
|
960 | local node_labels=""
|
961 | if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then
|
962 | # Add kube-proxy daemonset label to node to avoid situation during cluster
|
963 | # upgrade/downgrade when there are two instances of kube-proxy running on a node.
|
964 | node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true"
|
965 | fi
|
966 | if [[ -n "${NODE_LABELS:-}" ]]; then
|
967 | node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
|
968 | fi
|
969 | if [[ -n "${node_labels:-}" ]]; then
|
970 | flags+=" --node-labels=${node_labels}"
|
971 | fi
|
972 | if [[ -n "${NODE_TAINTS:-}" ]]; then
|
973 | flags+=" --register-with-taints=${NODE_TAINTS}"
|
974 | fi
|
975 | if [[ -n "${EVICTION_HARD:-}" ]]; then
|
976 | flags+=" --eviction-hard=${EVICTION_HARD}"
|
977 | fi
|
978 | if [[ -n "${FEATURE_GATES:-}" ]]; then
|
979 | flags+=" --feature-gates=${FEATURE_GATES}"
|
980 | fi
|
981 | if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then
|
982 | flags+=" --rotate-certificates=true"
|
983 | fi
|
984 |
|
985 | local -r kubelet_env_file="/etc/default/kubelet"
|
986 | echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
|
987 |
|
988 | # Write the systemd service file for kubelet.
|
989 | cat <<EOF >/etc/systemd/system/kubelet.service
|
990 | [Unit]
|
991 | Description=Kubernetes kubelet
|
992 | Requires=network-online.target
|
993 | After=network-online.target
|
994 |
|
995 | [Service]
|
996 | Restart=always
|
997 | RestartSec=10
|
998 | EnvironmentFile=${kubelet_env_file}
|
999 | ExecStart=${kubelet_bin} \$KUBELET_OPTS
|
1000 |
|
1001 | [Install]
|
1002 | WantedBy=multi-user.target
|
1003 | EOF
|
1004 |
|
1005 | # Flush iptables nat table
|
1006 | iptables -t nat -F || true
|
1007 |
|
1008 | systemctl start kubelet.service
|
1009 | }
|
1010 |
|
1011 | # This function assembles the node problem detector systemd service file and
|
1012 | # starts it using systemctl.
|
1013 | function start-node-problem-detector {
|
1014 | echo "Start node problem detector"
|
1015 | local -r npd_bin="${KUBE_HOME}/bin/node-problem-detector"
|
1016 | local -r km_config="${KUBE_HOME}/node-problem-detector/config/kernel-monitor.json"
|
1017 | local -r dm_config="${KUBE_HOME}/node-problem-detector/config/docker-monitor.json"
|
1018 | echo "Using node problem detector binary at ${npd_bin}"
|
1019 | local flags="${NPD_TEST_LOG_LEVEL:-"--v=2"} ${NPD_TEST_ARGS:-}"
|
1020 | flags+=" --logtostderr"
|
1021 | flags+=" --system-log-monitors=${km_config},${dm_config}"
|
1022 | flags+=" --apiserver-override=https://${KUBERNETES_MASTER_NAME}?inClusterConfig=false&auth=/var/lib/node-problem-detector/kubeconfig"
|
1023 | local -r npd_port=${NODE_PROBLEM_DETECTOR_PORT:-20256}
|
1024 | flags+=" --port=${npd_port}"
|
1025 |
|
1026 | # Write the systemd service file for node problem detector.
|
1027 | cat <<EOF >/etc/systemd/system/node-problem-detector.service
|
1028 | [Unit]
|
1029 | Description=Kubernetes node problem detector
|
1030 | Requires=network-online.target
|
1031 | After=network-online.target
|
1032 |
|
1033 | [Service]
|
1034 | Restart=always
|
1035 | RestartSec=10
|
1036 | ExecStart=${npd_bin} ${flags}
|
1037 |
|
1038 | [Install]
|
1039 | WantedBy=multi-user.target
|
1040 | EOF
|
1041 |
|
1042 | systemctl start node-problem-detector.service
|
1043 | }
|
1044 |
|
1045 | # Create the log file and set its properties.
|
1046 | #
|
1047 | # $1 is the file to create.
|
1048 | function prepare-log-file {
|
1049 | touch $1
|
1050 | chmod 644 $1
|
1051 | chown root:root $1
|
1052 | }
|
1053 |
|
1054 | # Prepares parameters for kube-proxy manifest.
|
1055 | # $1 source path of kube-proxy manifest.
|
1056 | function prepare-kube-proxy-manifest-variables {
|
1057 | local -r src_file=$1;
|
1058 |
|
1059 | remove-salt-config-comments "${src_file}"
|
1060 |
|
1061 | local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig"
|
1062 | local kube_docker_registry="gcr.io/google_containers"
|
1063 | if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
|
1064 | kube_docker_registry=${KUBE_DOCKER_REGISTRY}
|
1065 | fi
|
1066 | local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag)
|
1067 | local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
|
1068 | local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}"
|
1069 | if [[ -n "${FEATURE_GATES:-}" ]]; then
|
1070 | params+=" --feature-gates=${FEATURE_GATES}"
|
1071 | fi
|
1072 | params+=" --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s"
|
1073 | if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then
|
1074 | params+=" ${KUBEPROXY_TEST_ARGS}"
|
1075 | fi
|
1076 | local container_env=""
|
1077 | local kube_cache_mutation_detector_env_name=""
|
1078 | local kube_cache_mutation_detector_env_value=""
|
1079 | if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
|
1080 | container_env="env:"
|
1081 | kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
|
1082 | kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
|
1083 | fi
|
1084 | local pod_priority=""
|
1085 | if [[ "${ENABLE_POD_PRIORITY:-}" == "true" ]]; then
|
1086 | pod_priority="priorityClassName: system-node-critical"
|
1087 | fi
|
1088 | sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file}
|
1089 | sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file}
|
1090 | sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file}
|
1091 | sed -i -e "s@{{params}}@${params}@g" ${src_file}
|
1092 | sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
1093 | sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file}
|
1094 | sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file}
|
1095 | sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file}
|
1096 | sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
|
1097 | sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
|
1098 | sed -i -e "s@{{kubernetes_service_host_env_value}}@${KUBERNETES_MASTER_NAME}@g" ${src_file}
|
1099 | if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
|
1100 | sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file}
|
1101 | fi
|
1102 | }
|
1103 |
|
1104 | # Starts kube-proxy static pod.
|
1105 | function start-kube-proxy {
|
1106 | echo "Start kube-proxy static pod"
|
1107 | prepare-log-file /var/log/kube-proxy.log
|
1108 | local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest"
|
1109 | prepare-kube-proxy-manifest-variables "${src_file}"
|
1110 |
|
1111 | cp "${src_file}" /etc/kubernetes/manifests
|
1112 | }
|
1113 |
|
1114 | # Replaces the variables in the etcd manifest file with the real values, and then
|
1115 | # copy the file to the manifest dir
|
1116 | # $1: value for variable 'suffix'
|
1117 | # $2: value for variable 'port'
|
1118 | # $3: value for variable 'server_port'
|
1119 | # $4: value for variable 'cpulimit'
|
1120 | # $5: pod name, which should be either etcd or etcd-events
|
1121 | function prepare-etcd-manifest {
|
1122 | local host_name=$(hostname)
|
1123 | local etcd_cluster=""
|
1124 | local cluster_state="new"
|
1125 | local etcd_protocol="http"
|
1126 | local etcd_creds=""
|
1127 |
|
1128 | if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
|
1129 | cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
|
1130 | fi
|
1131 | if [[ -n "${ETCD_CA_KEY:-}" && -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
|
1132 | etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
|
1133 | etcd_protocol="https"
|
1134 | fi
|
1135 |
|
1136 | for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
|
1137 | etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
|
1138 | if [[ -n "${etcd_cluster}" ]]; then
|
1139 | etcd_cluster+=","
|
1140 | fi
|
1141 | etcd_cluster+="${etcd_host}"
|
1142 | done
|
1143 |
|
1144 | local -r temp_file="/tmp/$5"
|
1145 | cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}"
|
1146 | remove-salt-config-comments "${temp_file}"
|
1147 | sed -i -e "s@{{ *suffix *}}@$1@g" "${temp_file}"
|
1148 | sed -i -e "s@{{ *port *}}@$2@g" "${temp_file}"
|
1149 | sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}"
|
1150 | sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}"
|
1151 | sed -i -e "s@{{ *hostname *}}@$host_name@g" "${temp_file}"
|
1152 | sed -i -e "s@{{ *srv_kube_path *}}@/etc/srv/kubernetes@g" "${temp_file}"
|
1153 | sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
|
1154 | # Get default storage backend from manifest file.
|
1155 | local -r default_storage_backend=$(cat "${temp_file}" | \
|
1156 | grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" | \
|
1157 | sed -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g")
|
1158 | if [[ -n "${STORAGE_BACKEND:-}" ]]; then
|
1159 | sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@${STORAGE_BACKEND}@g" "${temp_file}"
|
1160 | else
|
1161 | sed -i -e "s@{{ *pillar\.get('storage_backend', '\(.*\)') *}}@\1@g" "${temp_file}"
|
1162 | fi
|
1163 | if [[ "${STORAGE_BACKEND:-${default_storage_backend}}" == "etcd3" ]]; then
|
1164 | sed -i -e "s@{{ *quota_bytes *}}@--quota-backend-bytes=4294967296@g" "${temp_file}"
|
1165 | else
|
1166 | sed -i -e "s@{{ *quota_bytes *}}@@g" "${temp_file}"
|
1167 | fi
|
1168 | sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}"
|
1169 | if [[ -n "${ETCD_IMAGE:-}" ]]; then
|
1170 | sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@${ETCD_IMAGE}@g" "${temp_file}"
|
1171 | else
|
1172 | sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}"
|
1173 | fi
|
1174 | if [[ -n "${ETCD_DOCKER_REPOSITORY:-}" ]]; then
|
1175 | sed -i -e "s@{{ *pillar\.get('etcd_docker_repository', '\(.*\)') *}}@${ETCD_DOCKER_REPOSITORY}@g" "${temp_file}"
|
1176 | else
|
1177 | sed -i -e "s@{{ *pillar\.get('etcd_docker_repository', '\(.*\)') *}}@\1@g" "${temp_file}"
|
1178 | fi
|
1179 | sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
|
1180 | sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
|
1181 | if [[ -n "${ETCD_VERSION:-}" ]]; then
|
1182 | sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}"
|
1183 | else
|
1184 | sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@\1@g" "${temp_file}"
|
1185 | fi
|
1186 | # Replace the volume host path.
|
1187 | sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}"
|
1188 | mv "${temp_file}" /etc/kubernetes/manifests
|
1189 | }
|
1190 |
|
1191 | function start-etcd-empty-dir-cleanup-pod {
|
1192 | cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml" "/etc/kubernetes/manifests"
|
1193 | }
|
1194 |
|
1195 | # Starts etcd server pod (and etcd-events pod if needed).
|
1196 | # More specifically, it prepares dirs and files, sets the variable value
|
1197 | # in the manifests, and copies them to /etc/kubernetes/manifests.
|
1198 | function start-etcd-servers {
|
1199 | echo "Start etcd pods"
|
1200 | if [[ -d /etc/etcd ]]; then
|
1201 | rm -rf /etc/etcd
|
1202 | fi
|
1203 | if [[ -e /etc/default/etcd ]]; then
|
1204 | rm -f /etc/default/etcd
|
1205 | fi
|
1206 | if [[ -e /etc/systemd/system/etcd.service ]]; then
|
1207 | rm -f /etc/systemd/system/etcd.service
|
1208 | fi
|
1209 | if [[ -e /etc/init.d/etcd ]]; then
|
1210 | rm -f /etc/init.d/etcd
|
1211 | fi
|
1212 | prepare-log-file /var/log/etcd.log
|
1213 | prepare-etcd-manifest "" "2379" "2380" "200m" "etcd.manifest"
|
1214 |
|
1215 | prepare-log-file /var/log/etcd-events.log
|
1216 | prepare-etcd-manifest "-events" "4002" "2381" "100m" "etcd-events.manifest"
|
1217 | }
|
1218 |
|
1219 | # Calculates the following variables based on env variables, which will be used
|
1220 | # by the manifests of several kube-master components.
|
1221 | # CLOUD_CONFIG_OPT
|
1222 | # CLOUD_CONFIG_VOLUME
|
1223 | # CLOUD_CONFIG_MOUNT
|
1224 | # DOCKER_REGISTRY
|
1225 | function compute-master-manifest-variables {
|
1226 | CLOUD_CONFIG_OPT=""
|
1227 | CLOUD_CONFIG_VOLUME=""
|
1228 | CLOUD_CONFIG_MOUNT=""
|
1229 | if [[ -f /etc/gce.conf ]]; then
|
1230 | CLOUD_CONFIG_OPT="--cloud-config=/etc/gce.conf"
|
1231 | CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
|
1232 | CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
1233 | fi
|
1234 | DOCKER_REGISTRY="gcr.io/google_containers"
|
1235 | if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
|
1236 | DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}"
|
1237 | fi
|
1238 | }
|
1239 |
|
1240 | # A helper function that bind mounts kubelet dirs for running mount in a chroot
|
1241 | function prepare-mounter-rootfs {
|
1242 | echo "Prepare containerized mounter"
|
1243 | mount --bind "${CONTAINERIZED_MOUNTER_HOME}" "${CONTAINERIZED_MOUNTER_HOME}"
|
1244 | mount -o remount,exec "${CONTAINERIZED_MOUNTER_HOME}"
|
1245 | CONTAINERIZED_MOUNTER_ROOTFS="${CONTAINERIZED_MOUNTER_HOME}/rootfs"
|
1246 | mount --rbind /var/lib/kubelet/ "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
|
1247 | mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
|
1248 | mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc"
|
1249 | mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev"
|
1250 | cp /etc/resolv.conf "${CONTAINERIZED_MOUNTER_ROOTFS}/etc/"
|
1251 | }
|
1252 |
|
1253 | # A helper function for removing salt configuration and comments from a file.
|
1254 | # This is mainly for preparing a manifest file.
|
1255 | #
|
1256 | # $1: Full path of the file to manipulate
|
1257 | function remove-salt-config-comments {
|
1258 | # Remove salt configuration.
|
1259 | sed -i "/^[ |\t]*{[#|%]/d" $1
|
1260 | # Remove comments.
|
1261 | sed -i "/^[ |\t]*#/d" $1
|
1262 | }
|
1263 |
|
1264 | # Starts kubernetes apiserver.
|
1265 | # It prepares the log file, loads the docker image, calculates variables, sets them
|
1266 | # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
1267 | #
|
1268 | # Assumed vars (which are calculated in function compute-master-manifest-variables)
|
1269 | # CLOUD_CONFIG_OPT
|
1270 | # CLOUD_CONFIG_VOLUME
|
1271 | # CLOUD_CONFIG_MOUNT
|
1272 | # DOCKER_REGISTRY
|
1273 | function start-kube-apiserver {
|
1274 | echo "Start kubernetes api-server"
|
1275 | prepare-log-file /var/log/kube-apiserver.log
|
1276 | prepare-log-file /var/log/kube-apiserver-audit.log
|
1277 |
|
1278 | # Calculate variables and assemble the command line.
|
1279 | local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
|
1280 | params+=" --address=127.0.0.1"
|
1281 | params+=" --allow-privileged=true"
|
1282 | params+=" --cloud-provider=gce"
|
1283 | params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}"
|
1284 | params+=" --etcd-servers=http://127.0.0.1:2379"
|
1285 | params+=" --etcd-servers-overrides=/events#http://127.0.0.1:4002"
|
1286 | params+=" --secure-port=443"
|
1287 | params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}"
|
1288 | params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}"
|
1289 | if [[ -s "${REQUESTHEADER_CA_CERT_PATH:-}" ]]; then
|
1290 | params+=" --requestheader-client-ca-file=${REQUESTHEADER_CA_CERT_PATH}"
|
1291 | params+=" --requestheader-allowed-names=aggregator"
|
1292 | params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-"
|
1293 | params+=" --requestheader-group-headers=X-Remote-Group"
|
1294 | params+=" --requestheader-username-headers=X-Remote-User"
|
1295 | params+=" --proxy-client-cert-file=${PROXY_CLIENT_CERT_PATH}"
|
1296 | params+=" --proxy-client-key-file=${PROXY_CLIENT_KEY_PATH}"
|
1297 | fi
|
1298 | params+=" --enable-aggregator-routing=true"
|
1299 | if [[ -e "${APISERVER_CLIENT_CERT_PATH}" ]] && [[ -e "${APISERVER_CLIENT_KEY_PATH}" ]]; then
|
1300 | params+=" --kubelet-client-certificate=${APISERVER_CLIENT_CERT_PATH}"
|
1301 | params+=" --kubelet-client-key=${APISERVER_CLIENT_KEY_PATH}"
|
1302 | fi
|
1303 | if [[ -n "${SERVICEACCOUNT_CERT_PATH:-}" ]]; then
|
1304 | params+=" --service-account-key-file=${SERVICEACCOUNT_CERT_PATH}"
|
1305 | fi
|
1306 | params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv"
|
1307 | if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then
|
1308 | params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv"
|
1309 | fi
|
1310 | if [[ -n "${STORAGE_BACKEND:-}" ]]; then
|
1311 | params+=" --storage-backend=${STORAGE_BACKEND}"
|
1312 | fi
|
1313 | if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then
|
1314 | params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}"
|
1315 | fi
|
1316 | if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]]; then
|
1317 | params+=" --request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}s"
|
1318 | fi
|
1319 | if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then
|
1320 | params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
|
1321 | fi
|
1322 | if [[ -n "${NUM_NODES:-}" ]]; then
|
1323 | # If the cluster is large, increase max-requests-inflight limit in apiserver.
|
1324 | if [[ "${NUM_NODES}" -ge 1000 ]]; then
|
1325 | params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500"
|
1326 | fi
|
1327 | # Set amount of memory available for apiserver based on number of nodes.
|
1328 | # TODO: Once we start setting proper requests and limits for apiserver
|
1329 | # we should reuse the same logic here instead of current heuristic.
|
1330 | params+=" --target-ram-mb=$((${NUM_NODES} * 60))"
|
1331 | fi
|
1332 | if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
|
1333 | params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
|
1334 | fi
|
1335 | if [[ -n "${ETCD_QUORUM_READ:-}" ]]; then
|
1336 | params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}"
|
1337 | fi
|
1338 |
|
1339 | local audit_policy_config_mount=""
|
1340 | local audit_policy_config_volume=""
|
1341 | local audit_webhook_config_mount=""
|
1342 | local audit_webhook_config_volume=""
|
1343 | if [[ "${ENABLE_APISERVER_BASIC_AUDIT:-}" == "true" ]]; then
|
1344 | # We currently only support enabling with a fixed path and with built-in log
|
1345 | # rotation "disabled" (large value) so it behaves like kube-apiserver.log.
|
1346 | # External log rotation should be set up the same as for kube-apiserver.log.
|
1347 | params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
|
1348 | params+=" --audit-log-maxage=0"
|
1349 | params+=" --audit-log-maxbackup=0"
|
1350 | # Lumberjack doesn't offer any way to disable size-based rotation. It also
|
1351 | # has an in-memory counter that doesn't notice if you truncate the file.
|
1352 | # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log
|
1353 | # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver
|
1354 | # never restarts. Please manually restart apiserver before this time.
|
1355 | params+=" --audit-log-maxsize=2000000000"
|
1356 | # Disable AdvancedAuditing enabled by default
|
1357 | if [[ -z "${FEATURE_GATES:-}" ]]; then
|
1358 | FEATURE_GATES="AdvancedAuditing=false"
|
1359 | else
|
1360 | FEATURE_GATES="${FEATURE_GATES},AdvancedAuditing=false"
|
1361 | fi
|
1362 | elif [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
|
1363 | local -r audit_policy_file="/etc/audit_policy.config"
|
1364 | params+=" --audit-policy-file=${audit_policy_file}"
|
1365 | # Create the audit policy file, and mount it into the apiserver pod.
|
1366 | create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}"
|
1367 | audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true},"
|
1368 | audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\", \"type\": \"FileOrCreate\"}},"
|
1369 |
|
1370 | if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then
|
1371 | # The advanced audit log backend config matches the basic audit log config.
|
1372 | params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
|
1373 | params+=" --audit-log-maxage=0"
|
1374 | params+=" --audit-log-maxbackup=0"
|
1375 | # Lumberjack doesn't offer any way to disable size-based rotation. It also
|
1376 | # has an in-memory counter that doesn't notice if you truncate the file.
|
1377 | # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log
|
1378 | # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver
|
1379 | # never restarts. Please manually restart apiserver before this time.
|
1380 | params+=" --audit-log-maxsize=2000000000"
|
1381 | fi
|
1382 | if [[ "${ADVANCED_AUDIT_BACKEND:-}" == *"webhook"* ]]; then
|
1383 | params+=" --audit-webhook-mode=batch"
|
1384 |
|
1385 | # Create the audit webhook config file, and mount it into the apiserver pod.
|
1386 | local -r audit_webhook_config_file="/etc/audit_webhook.config"
|
1387 | params+=" --audit-webhook-config-file=${audit_webhook_config_file}"
|
1388 | create-master-audit-webhook-config "${audit_webhook_config_file}"
|
1389 | audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true},"
|
1390 | audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}},"
|
1391 | fi
|
1392 | fi
|
1393 |
|
1394 | if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then
|
1395 | params+=" --enable-logs-handler=false"
|
1396 | fi
|
1397 |
|
1398 | local admission_controller_config_mount=""
|
1399 | local admission_controller_config_volume=""
|
1400 | local image_policy_webhook_config_mount=""
|
1401 | local image_policy_webhook_config_volume=""
|
1402 | if [[ -n "${ADMISSION_CONTROL:-}" ]]; then
|
1403 | params+=" --admission-control=${ADMISSION_CONTROL}"
|
1404 | if [[ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]]; then
|
1405 | params+=" --admission-control-config-file=/etc/admission_controller.config"
|
1406 | # Mount the file to configure admission controllers if ImagePolicyWebhook is set.
|
1407 | admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false},"
|
1408 | admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\", \"type\": \"FileOrCreate\"}},"
|
1409 | # Mount the file to configure the ImagePolicyWebhook's webhook.
|
1410 | image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false},"
|
1411 | image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}},"
|
1412 | fi
|
1413 | fi
|
1414 |
|
1415 | if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then
|
1416 | params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
|
1417 | fi
|
1418 | if [[ -n "${RUNTIME_CONFIG:-}" ]]; then
|
1419 | params+=" --runtime-config=${RUNTIME_CONFIG}"
|
1420 | fi
|
1421 | if [[ -n "${FEATURE_GATES:-}" ]]; then
|
1422 | params+=" --feature-gates=${FEATURE_GATES}"
|
1423 | fi
|
1424 | if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then
|
1425 | local -r vm_external_ip=$(curl --retry 5 --retry-delay 3 --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
|
1426 | params+=" --advertise-address=${vm_external_ip}"
|
1427 | params+=" --ssh-user=${PROXY_SSH_USER}"
|
1428 | params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
|
1429 | elif [ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]; then
|
1430 | params="${params} --advertise-address=${MASTER_ADVERTISE_ADDRESS}"
|
1431 | fi
|
1432 |
|
1433 | local webhook_authn_config_mount=""
|
1434 | local webhook_authn_config_volume=""
|
1435 | if [[ -n "${GCP_AUTHN_URL:-}" ]]; then
|
1436 | params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config"
|
1437 | webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false},"
|
1438 | webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\", \"type\": \"FileOrCreate\"}},"
|
1439 | fi
|
1440 |
|
1441 |
|
1442 | local authorization_mode="Node,RBAC"
|
1443 | local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
|
1444 |
|
1445 | # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false
|
1446 | if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then
|
1447 | echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this."
|
1448 | # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions)
|
1449 | if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then
|
1450 | local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl"
|
1451 | remove-salt-config-comments "${abac_policy_json}"
|
1452 | if [[ -n "${KUBE_USER:-}" ]]; then
|
1453 | sed -i -e "s/{{kube_user}}/${KUBE_USER}/g" "${abac_policy_json}"
|
1454 | else
|
1455 | sed -i -e "/{{kube_user}}/d" "${abac_policy_json}"
|
1456 | fi
|
1457 | cp "${abac_policy_json}" /etc/srv/kubernetes/
|
1458 | fi
|
1459 |
|
1460 | params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl"
|
1461 | authorization_mode+=",ABAC"
|
1462 | fi
|
1463 |
|
1464 | local webhook_config_mount=""
|
1465 | local webhook_config_volume=""
|
1466 | if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then
|
1467 | authorization_mode+=",Webhook"
|
1468 | params+=" --authorization-webhook-config-file=/etc/gcp_authz.config"
|
1469 | webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false},"
|
1470 | webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}},"
|
1471 | fi
|
1472 | params+=" --authorization-mode=${authorization_mode}"
|
1473 |
|
1474 | local container_env=""
|
1475 | if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
|
1476 | container_env="\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
|
1477 | fi
|
1478 | if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then
|
1479 | if [[ -n "${container_env}" ]]; then
|
1480 | container_env="${container_env}, "
|
1481 | fi
|
1482 | container_env="\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\""
|
1483 | fi
|
1484 | if [[ -n "${container_env}" ]]; then
|
1485 | container_env="\"env\":[{${container_env}}],"
|
1486 | fi
|
1487 |
|
1488 | if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
1489 | local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
|
1490 | if [[ -n "${GOOGLE_CLOUD_KMS_CONFIG_FILE_NAME:-}" && -n "${GOOGLE_CLOUD_KMS_CONFIG:-}" ]]; then
|
1491 | echo "${GOOGLE_CLOUD_KMS_CONFIG}" | base64 --decode > "${GOOGLE_CLOUD_KMS_CONFIG_FILE_NAME}"
|
1492 | fi
|
1493 |
|
1494 | echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
|
1495 | params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
|
1496 | fi
|
1497 |
|
1498 | src_file="${src_dir}/kube-apiserver.manifest"
|
1499 | remove-salt-config-comments "${src_file}"
|
1500 | # Evaluate variables.
|
1501 | local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
|
1502 | sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
1503 | sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
1504 | sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
1505 | sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
|
1506 | sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
1507 | sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
1508 | sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
1509 | sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}"
|
1510 | sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
|
1511 | sed -i -e "s@{{secure_port}}@443@g" "${src_file}"
|
1512 | sed -i -e "s@{{secure_port}}@8080@g" "${src_file}"
|
1513 | sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
1514 | sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
1515 | sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}"
|
1516 | sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
|
1517 | sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"
|
1518 | sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}"
|
1519 | sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
|
1520 | sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
|
1521 | sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}"
|
1522 | sed -i -e "s@{{audit_webhook_config_volume}}@${audit_webhook_config_volume}@g" "${src_file}"
|
1523 | sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}"
|
1524 | sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}"
|
1525 | sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
|
1526 | sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
|
1527 | cp "${src_file}" /etc/kubernetes/manifests
|
1528 | }
|
1529 |
|
1530 | # Starts kubernetes controller manager.
|
1531 | # It prepares the log file, loads the docker image, calculates variables, sets them
|
1532 | # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
1533 | #
|
1534 | # Assumed vars (which are calculated in function compute-master-manifest-variables)
|
1535 | # CLOUD_CONFIG_OPT
|
1536 | # CLOUD_CONFIG_VOLUME
|
1537 | # CLOUD_CONFIG_MOUNT
|
1538 | # DOCKER_REGISTRY
|
1539 | function start-kube-controller-manager {
|
1540 | echo "Start kubernetes controller-manager"
|
1541 | create-kubecontrollermanager-kubeconfig
|
1542 | prepare-log-file /var/log/kube-controller-manager.log
|
1543 | # Calculate variables and assemble the command line.
|
1544 | local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
|
1545 | params+=" --use-service-account-credentials"
|
1546 | params+=" --cloud-provider=gce"
|
1547 | params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig"
|
1548 | params+=" --root-ca-file=${CA_CERT_BUNDLE_PATH}"
|
1549 | params+=" --service-account-private-key-file=${SERVICEACCOUNT_KEY_PATH}"
|
1550 | if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then
|
1551 | params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
|
1552 | fi
|
1553 | if [[ -n "${INSTANCE_PREFIX:-}" ]]; then
|
1554 | params+=" --cluster-name=${INSTANCE_PREFIX}"
|
1555 | fi
|
1556 | if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
|
1557 | params+=" --cluster-cidr=${CLUSTER_IP_RANGE}"
|
1558 | fi
|
1559 | if [[ -n "${CA_KEY:-}" ]]; then
|
1560 | params+=" --cluster-signing-cert-file=${CA_CERT_PATH}"
|
1561 | params+=" --cluster-signing-key-file=${CA_KEY_PATH}"
|
1562 | fi
|
1563 | if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
|
1564 | params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
|
1565 | fi
|
1566 | if [[ -n "${CONCURRENT_SERVICE_SYNCS:-}" ]]; then
|
1567 | params+=" --concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}"
|
1568 | fi
|
1569 | if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then
|
1570 | params+=" --allocate-node-cidrs=true"
|
1571 | elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then
|
1572 | params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
|
1573 | fi
|
1574 | if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
|
1575 | params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
|
1576 | fi
|
1577 | if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
|
1578 | params+=" --cidr-allocator-type=CloudAllocator"
|
1579 | params+=" --configure-cloud-routes=false"
|
1580 | fi
|
1581 | if [[ -n "${FEATURE_GATES:-}" ]]; then
|
1582 | params+=" --feature-gates=${FEATURE_GATES}"
|
1583 | fi
|
1584 | if [[ -n "${VOLUME_PLUGIN_DIR:-}" ]]; then
|
1585 | params+=" --flex-volume-plugin-dir=${VOLUME_PLUGIN_DIR}"
|
1586 | fi
|
1587 | if [[ -n "${CLUSTER_SIGNING_DURATION:-}" ]]; then
|
1588 | params+=" --experimental-cluster-signing-duration=$CLUSTER_SIGNING_DURATION"
|
1589 | fi
|
1590 | # disable using HPA metrics REST clients if metrics-server isn't enabled
|
1591 | if [[ "${ENABLE_METRICS_SERVER:-}" != "true" ]]; then
|
1592 | params+=" --horizontal-pod-autoscaler-use-rest-clients=false"
|
1593 | fi
|
1594 |
|
1595 | local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
|
1596 | local container_env=""
|
1597 | if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
|
1598 | container_env="\"env\":[{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}],"
|
1599 | fi
|
1600 |
|
1601 | local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
|
1602 | remove-salt-config-comments "${src_file}"
|
1603 | # Evaluate variables.
|
1604 | sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
1605 | sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
1606 | sed -i -e "s@{{pillar\['kube-controller-manager_docker_tag'\]}}@${kube_rc_docker_tag}@g" "${src_file}"
|
1607 | sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
1608 | sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
1609 | sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
1610 | sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
1611 | sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
|
1612 | sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
|
1613 | cp "${src_file}" /etc/kubernetes/manifests
|
1614 | }
|
1615 |
|
1616 | # Starts kubernetes scheduler.
|
1617 | # It prepares the log file, loads the docker image, calculates variables, sets them
|
1618 | # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
|
1619 | #
|
1620 | # Assumed vars (which are calculated in compute-master-manifest-variables)
|
1621 | # DOCKER_REGISTRY
|
1622 | function start-kube-scheduler {
|
1623 | echo "Start kubernetes scheduler"
|
1624 | create-kubescheduler-kubeconfig
|
1625 | prepare-log-file /var/log/kube-scheduler.log
|
1626 |
|
1627 | # Calculate variables and set them in the manifest.
|
1628 | params="${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"} ${SCHEDULER_TEST_ARGS:-}"
|
1629 | params+=" --kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig"
|
1630 | if [[ -n "${FEATURE_GATES:-}" ]]; then
|
1631 | params+=" --feature-gates=${FEATURE_GATES}"
|
1632 | fi
|
1633 | if [[ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]]; then
|
1634 | params+=" --algorithm-provider=${SCHEDULING_ALGORITHM_PROVIDER}"
|
1635 | fi
|
1636 | local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag")
|
1637 |
|
1638 | # Remove salt comments and replace variables with values.
|
1639 | local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-scheduler.manifest"
|
1640 | remove-salt-config-comments "${src_file}"
|
1641 |
|
1642 | sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
1643 | sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
1644 | sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
|
1645 | sed -i -e "s@{{pillar\['kube-scheduler_docker_tag'\]}}@${kube_scheduler_docker_tag}@g" "${src_file}"
|
1646 | cp "${src_file}" /etc/kubernetes/manifests
|
1647 | }
|
1648 |
|
1649 | # Starts cluster autoscaler.
|
1650 | # Assumed vars (which are calculated in function compute-master-manifest-variables)
|
1651 | # CLOUD_CONFIG_OPT
|
1652 | # CLOUD_CONFIG_VOLUME
|
1653 | # CLOUD_CONFIG_MOUNT
|
1654 | function start-cluster-autoscaler {
|
1655 | if [[ "${ENABLE_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
|
1656 | echo "Start kubernetes cluster autoscaler"
|
1657 | prepare-log-file /var/log/cluster-autoscaler.log
|
1658 |
|
1659 | # Remove salt comments and replace variables with values
|
1660 | local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest"
|
1661 | remove-salt-config-comments "${src_file}"
|
1662 |
|
1663 | local params="${AUTOSCALER_MIG_CONFIG} ${CLOUD_CONFIG_OPT} ${AUTOSCALER_EXPANDER_CONFIG:---expander=price}"
|
1664 | sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
1665 | sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
1666 | sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
1667 | sed -i -e "s@{%.*%}@@g" "${src_file}"
|
1668 |
|
1669 | cp "${src_file}" /etc/kubernetes/manifests
|
1670 | fi
|
1671 | }
|
1672 |
|
1673 | # A helper function for copying addon manifests and set dir/files
|
1674 | # permissions.
|
1675 | #
|
1676 | # $1: addon category under /etc/kubernetes
|
1677 | # $2: manifest source dir
|
1678 | function setup-addon-manifests {
|
1679 | local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/$2"
|
1680 | local -r dst_dir="/etc/kubernetes/$1/$2"
|
1681 | if [[ ! -d "${dst_dir}" ]]; then
|
1682 | mkdir -p "${dst_dir}"
|
1683 | fi
|
1684 | local files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml")
|
1685 | if [[ -n "${files}" ]]; then
|
1686 | cp "${src_dir}/"*.yaml "${dst_dir}"
|
1687 | fi
|
1688 | files=$(find "${src_dir}" -maxdepth 1 -name "*.json")
|
1689 | if [[ -n "${files}" ]]; then
|
1690 | cp "${src_dir}/"*.json "${dst_dir}"
|
1691 | fi
|
1692 | files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml.in")
|
1693 | if [[ -n "${files}" ]]; then
|
1694 | cp "${src_dir}/"*.yaml.in "${dst_dir}"
|
1695 | fi
|
1696 | chown -R root:root "${dst_dir}"
|
1697 | chmod 755 "${dst_dir}"
|
1698 | chmod 644 "${dst_dir}"/*
|
1699 | }
|
1700 |
|
1701 | # Fluentd manifest is modified using kubectl, which may not be available at
|
1702 | # this point. Run this as a background process.
|
1703 | function wait-for-apiserver-and-update-fluentd {
|
1704 | until kubectl get nodes
|
1705 | do
|
1706 | sleep 10
|
1707 | done
|
1708 | kubectl set resources --dry-run --local -f ${fluentd_gcp_yaml} \
|
1709 | --limits=memory=${FLUENTD_GCP_MEMORY_LIMIT} \
|
1710 | --requests=cpu=${FLUENTD_GCP_CPU_REQUEST},memory=${FLUENTD_GCP_MEMORY_REQUEST} \
|
1711 | --containers=fluentd-gcp -o yaml > ${fluentd_gcp_yaml}.tmp
|
1712 | mv ${fluentd_gcp_yaml}.tmp ${fluentd_gcp_yaml}
|
1713 | }
|
1714 |
|
1715 | # Trigger background process that will ultimately update fluentd resource
|
1716 | # requirements.
|
1717 | function start-fluentd-resource-update {
|
1718 | wait-for-apiserver-and-update-fluentd &
|
1719 | }
|
1720 |
|
1721 | # Updates parameters in yaml file for prometheus-to-sd configuration, or
|
1722 | # removes component if it is disabled.
|
1723 | function update-prometheus-to-sd-parameters {
|
1724 | if [[ "${ENABLE_PROMETHEUS_TO_SD:-}" == "true" ]]; then
|
1725 | sed -i -e "s@{{ *prometheus_to_sd_prefix *}}@${PROMETHEUS_TO_SD_PREFIX}@g" "$1"
|
1726 | sed -i -e "s@{{ *prometheus_to_sd_endpoint *}}@${PROMETHEUS_TO_SD_ENDPOINT}@g" "$1"
|
1727 | else
|
1728 | # Removes all lines between two patterns (throws away prometheus-to-sd)
|
1729 | sed -i -e "/# BEGIN_PROMETHEUS_TO_SD/,/# END_PROMETHEUS_TO_SD/d" "$1"
|
1730 | fi
|
1731 | }
|
1732 |
|
1733 | # Prepares the manifests of k8s addons, and starts the addon manager.
|
1734 | # Vars assumed:
|
1735 | # CLUSTER_NAME
|
1736 | function start-kube-addons {
|
1737 | echo "Prepare kube-addons manifests and start kube addon manager"
|
1738 | local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
|
1739 | local -r dst_dir="/etc/kubernetes/addons"
|
1740 |
|
1741 | # prep addition kube-up specific rbac objects
|
1742 | setup-addon-manifests "addons" "rbac"
|
1743 |
|
1744 | # Set up manifests of other addons.
|
1745 | if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then
|
1746 | prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml"
|
1747 | setup-addon-manifests "addons" "kube-proxy"
|
1748 | fi
|
1749 | if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \
|
1750 | [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \
|
1751 | [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \
|
1752 | [[ "${ENABLE_CLUSTER_MONITORING:-}" == "standalone" ]] || \
|
1753 | [[ "${ENABLE_CLUSTER_MONITORING:-}" == "googleinfluxdb" ]]; then
|
1754 | local -r file_dir="cluster-monitoring/${ENABLE_CLUSTER_MONITORING}"
|
1755 | setup-addon-manifests "addons" "cluster-monitoring"
|
1756 | setup-addon-manifests "addons" "${file_dir}"
|
1757 | # Replace the salt configurations with variable values.
|
1758 | base_metrics_memory="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}"
|
1759 | base_eventer_memory="190Mi"
|
1760 | base_metrics_cpu="${HEAPSTER_GCP_BASE_CPU:-80m}"
|
1761 | nanny_memory="90Mi"
|
1762 | local -r metrics_memory_per_node="${HEAPSTER_GCP_MEMORY_PER_NODE:-4}"
|
1763 | local -r metrics_cpu_per_node="${HEAPSTER_GCP_CPU_PER_NODE:-0.5}"
|
1764 | local -r eventer_memory_per_node="500"
|
1765 | local -r nanny_memory_per_node="200"
|
1766 | if [[ -n "${NUM_NODES:-}" && "${NUM_NODES}" -ge 1 ]]; then
|
1767 | num_kube_nodes="$((${NUM_NODES}+1))"
|
1768 | nanny_memory="$((${num_kube_nodes} * ${nanny_memory_per_node} + 90 * 1024))Ki"
|
1769 | fi
|
1770 | controller_yaml="${dst_dir}/${file_dir}"
|
1771 | if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "googleinfluxdb" ]]; then
|
1772 | controller_yaml="${controller_yaml}/heapster-controller-combined.yaml"
|
1773 | else
|
1774 | controller_yaml="${controller_yaml}/heapster-controller.yaml"
|
1775 | fi
|
1776 | remove-salt-config-comments "${controller_yaml}"
|
1777 | sed -i -e "s@{{ cluster_name }}@${CLUSTER_NAME}@g" "${controller_yaml}"
|
1778 | sed -i -e "s@{{ *base_metrics_memory *}}@${base_metrics_memory}@g" "${controller_yaml}"
|
1779 | sed -i -e "s@{{ *base_metrics_cpu *}}@${base_metrics_cpu}@g" "${controller_yaml}"
|
1780 | sed -i -e "s@{{ *base_eventer_memory *}}@${base_eventer_memory}@g" "${controller_yaml}"
|
1781 | sed -i -e "s@{{ *metrics_memory_per_node *}}@${metrics_memory_per_node}@g" "${controller_yaml}"
|
1782 | sed -i -e "s@{{ *eventer_memory_per_node *}}@${eventer_memory_per_node}@g" "${controller_yaml}"
|
1783 | sed -i -e "s@{{ *nanny_memory *}}@${nanny_memory}@g" "${controller_yaml}"
|
1784 | sed -i -e "s@{{ *metrics_cpu_per_node *}}@${metrics_cpu_per_node}@g" "${controller_yaml}"
|
1785 | update-prometheus-to-sd-parameters ${controller_yaml}
|
1786 | fi
|
1787 | if [[ "${ENABLE_METRICS_SERVER:-}" == "true" ]]; then
|
1788 | setup-addon-manifests "addons" "metrics-server"
|
1789 | fi
|
1790 | if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then
|
1791 | setup-addon-manifests "addons" "dns"
|
1792 | local -r kubedns_file="${dst_dir}/dns/kube-dns.yaml"
|
1793 | mv "${dst_dir}/dns/kube-dns.yaml.in" "${kubedns_file}"
|
1794 | # Replace the salt configurations with variable values.
|
1795 | sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${kubedns_file}"
|
1796 | sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${kubedns_file}"
|
1797 |
|
1798 | if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then
|
1799 | setup-addon-manifests "addons" "dns-horizontal-autoscaler"
|
1800 | fi
|
1801 | fi
|
1802 | if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then
|
1803 | setup-addon-manifests "addons" "registry"
|
1804 | local -r registry_pv_file="${dst_dir}/registry/registry-pv.yaml"
|
1805 | local -r registry_pvc_file="${dst_dir}/registry/registry-pvc.yaml"
|
1806 | mv "${dst_dir}/registry/registry-pv.yaml.in" "${registry_pv_file}"
|
1807 | mv "${dst_dir}/registry/registry-pvc.yaml.in" "${registry_pvc_file}"
|
1808 | # Replace the salt configurations with variable values.
|
1809 | remove-salt-config-comments "${controller_yaml}"
|
1810 | sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pv_file}"
|
1811 | sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}"
|
1812 | sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}"
|
1813 | fi
|
1814 | if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
|
1815 | [[ "${LOGGING_DESTINATION:-}" == "elasticsearch" ]] && \
|
1816 | [[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]]; then
|
1817 | setup-addon-manifests "addons" "fluentd-elasticsearch"
|
1818 | fi
|
1819 | if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
|
1820 | [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
|
1821 | setup-addon-manifests "addons" "fluentd-gcp"
|
1822 | local -r event_exporter_yaml="${dst_dir}/fluentd-gcp/event-exporter.yaml"
|
1823 | local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml"
|
1824 | update-prometheus-to-sd-parameters ${event_exporter_yaml}
|
1825 | update-prometheus-to-sd-parameters ${fluentd_gcp_yaml}
|
1826 | start-fluentd-resource-update
|
1827 | fi
|
1828 | if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then
|
1829 | setup-addon-manifests "addons" "dashboard"
|
1830 | fi
|
1831 | if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "daemonset" ]]; then
|
1832 | setup-addon-manifests "addons" "node-problem-detector"
|
1833 | fi
|
1834 | if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
1835 | # Setup role binding for standalone node problem detector.
|
1836 | setup-addon-manifests "addons" "node-problem-detector/standalone"
|
1837 | fi
|
1838 | if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
|
1839 | setup-addon-manifests "admission-controls" "limit-range"
|
1840 | fi
|
1841 | if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
|
1842 | setup-addon-manifests "addons" "calico-policy-controller"
|
1843 |
|
1844 | # Configure Calico CNI directory.
|
1845 | local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
|
1846 | sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}"
|
1847 | fi
|
1848 | if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
|
1849 | setup-addon-manifests "addons" "storage-class/gce"
|
1850 | fi
|
1851 | if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then
|
1852 | setup-addon-manifests "addons" "ip-masq-agent"
|
1853 | fi
|
1854 | if [[ "${ENABLE_METADATA_PROXY:-}" == "simple" ]]; then
|
1855 | setup-addon-manifests "addons" "metadata-proxy/gce"
|
1856 | fi
|
1857 |
|
1858 | # Place addon manager pod manifest.
|
1859 | cp "${src_dir}/kube-addon-manager.yaml" /etc/kubernetes/manifests
|
1860 | }
|
1861 |
|
1862 | # Starts an image-puller - used in test clusters.
|
1863 | function start-image-puller {
|
1864 | echo "Start image-puller"
|
1865 | cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/e2e-image-puller.manifest" \
|
1866 | /etc/kubernetes/manifests/
|
1867 | }
|
1868 |
|
1869 | # Starts kube-registry proxy
|
1870 | function start-kube-registry-proxy {
|
1871 | echo "Start kube-registry-proxy"
|
1872 | cp "${KUBE_HOME}/kube-manifests/kubernetes/kube-registry-proxy.yaml" /etc/kubernetes/manifests
|
1873 | }
|
1874 |
|
1875 | # Starts a l7 loadbalancing controller for ingress.
|
1876 | function start-lb-controller {
|
1877 | if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
|
1878 | echo "Start GCE L7 pod"
|
1879 | prepare-log-file /var/log/glbc.log
|
1880 | setup-addon-manifests "addons" "cluster-loadbalancing/glbc"
|
1881 | cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest" \
|
1882 | /etc/kubernetes/manifests/
|
1883 | fi
|
1884 | }
|
1885 |
|
1886 | # Starts rescheduler.
|
1887 | function start-rescheduler {
|
1888 | if [[ "${ENABLE_RESCHEDULER:-}" == "true" ]]; then
|
1889 | echo "Start Rescheduler"
|
1890 | prepare-log-file /var/log/rescheduler.log
|
1891 | cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/rescheduler.manifest" \
|
1892 | /etc/kubernetes/manifests/
|
1893 | fi
|
1894 | }
|
1895 |
|
1896 | # Setup working directory for kubelet.
|
1897 | function setup-kubelet-dir {
|
1898 | echo "Making /var/lib/kubelet executable for kubelet"
|
1899 | mount -B /var/lib/kubelet /var/lib/kubelet/
|
1900 | mount -B -o remount,exec,suid,dev /var/lib/kubelet
|
1901 | }
|
1902 |
|
1903 | function reset-motd {
|
1904 | # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
|
1905 | local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
|
1906 | # This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1),
|
1907 | # or the git hash that's in the build info.
|
1908 | local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")"
|
1909 | local devel=""
|
1910 | if [[ "${gitref}" != "${version}" ]]; then
|
1911 | devel="
|
1912 | Note: This looks like a development version, which might not be present on GitHub.
|
1913 | If it isn't, the closest tag is at:
|
1914 | https://github.com/kubernetes/kubernetes/tree/${gitref}
|
1915 | "
|
1916 | gitref="${version//*+/}"
|
1917 | fi
|
1918 | cat > /etc/motd <<EOF
|
1919 |
|
1920 | Welcome to Kubernetes ${version}!
|
1921 |
|
1922 | You can find documentation for Kubernetes at:
|
1923 | http://docs.kubernetes.io/
|
1924 |
|
1925 | The source for this release can be found at:
|
1926 | /home/kubernetes/kubernetes-src.tar.gz
|
1927 | Or you can download it at:
|
1928 | https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-src.tar.gz
|
1929 |
|
1930 | It is based on the Kubernetes source at:
|
1931 | https://github.com/kubernetes/kubernetes/tree/${gitref}
|
1932 | ${devel}
|
1933 | For Kubernetes copyright and licensing information, see:
|
1934 | /home/kubernetes/LICENSES
|
1935 |
|
1936 | EOF
|
1937 | }
|
1938 |
|
1939 | function override-kubectl {
|
1940 | echo "overriding kubectl"
|
1941 | echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
|
1942 | }
|
1943 |
|
1944 | ########### Main Function ###########
|
1945 | echo "Start to configure instance for kubernetes"
|
1946 |
|
1947 | KUBE_HOME="/home/kubernetes"
|
1948 | CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
1949 | if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
1950 | echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
1951 | exit 1
|
1952 | fi
|
1953 |
|
1954 | source "${KUBE_HOME}/kube-env"
|
1955 |
|
1956 | if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
1957 | source "${KUBE_HOME}/kube-master-certs"
|
1958 | fi
|
1959 |
|
1960 | if [[ -n "${KUBE_USER:-}" ]]; then
|
1961 | if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
1962 | echo "Bad KUBE_USER format."
|
1963 | exit 1
|
1964 | fi
|
1965 | fi
|
1966 |
|
1967 | # generate the controller manager and scheduler tokens here since they are only used on the master.
|
1968 | KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
1969 | KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
1970 |
|
1971 | setup-os-params
|
1972 | config-ip-firewall
|
1973 | create-dirs
|
1974 | setup-kubelet-dir
|
1975 | ensure-local-ssds
|
1976 | setup-logrotate
|
1977 | if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
1978 | mount-master-pd
|
1979 | create-node-pki
|
1980 | create-master-pki
|
1981 | create-master-auth
|
1982 | create-master-kubelet-auth
|
1983 | create-master-etcd-auth
|
1984 | else
|
1985 | create-node-pki
|
1986 | create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
1987 | if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
1988 | create-kubeproxy-user-kubeconfig
|
1989 | fi
|
1990 | if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
1991 | create-node-problem-detector-kubeconfig
|
1992 | fi
|
1993 | fi
|
1994 |
|
1995 | override-kubectl
|
1996 | # Run the containerized mounter once to pre-cache the container image.
|
1997 | assemble-docker-flags
|
1998 | start-kubelet
|
1999 |
|
2000 | if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
2001 | compute-master-manifest-variables
|
2002 | start-etcd-servers
|
2003 | start-etcd-empty-dir-cleanup-pod
|
2004 | start-kube-apiserver
|
2005 | start-kube-controller-manager
|
2006 | start-kube-scheduler
|
2007 | start-kube-addons
|
2008 | start-cluster-autoscaler
|
2009 | start-lb-controller
|
2010 | start-rescheduler
|
2011 | else
|
2012 | if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
2013 | start-kube-proxy
|
2014 | fi
|
2015 | # Kube-registry-proxy.
|
2016 | if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then
|
2017 | start-kube-registry-proxy
|
2018 | fi
|
2019 | if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
2020 | start-image-puller
|
2021 | fi
|
2022 | if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
2023 | start-node-problem-detector
|
2024 | fi
|
2025 | fi
|
2026 | reset-motd
|
2027 | prepare-mounter-rootfs
|
2028 | modprobe configs
|
2029 | echo "Done for the configuration for kubernetes"
|