1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
|
#!/bin/bash
# Script works in the following scenario:
# 1. Machine is a debian jessie or more recent, and has systemd
# 2. Machine has rkhunter running
# 3. Optional if update for gitlab-ce available: Machine has a Gitlab instance which is stopped
# by default and needs to be started during upgrade and stopped afterwards
# 4. Optional if needrestart available: needrestart will restart all services it can restart,
# and we will warn the user if there are too critical services remaining and/or
# the running kernel is outdated after upgrading
# Remark for 3.: This is only necessary because Gitlab does a database backup before actually
# upgrading (which is good), for which it needs to shutdown the Gitlab sub-instances. If
# they are already stopped, this fails (which is not good aka. stupid). If this code was
# better, we'd only need to start the systemd service. But what can you do ¯\_(ツ)_/¯
# (did I mention already that I'm happy with cgit?)
APTBIN="$(which apt||echo "/usr/bin/apt")"
RKHBIN="$(which rkhunter||echo "/usr/bin/rkhunter")"
# using -e here because we are not to be blamed if the executable isn't - executable
[ ! -e "$RKHBIN" ]&&unset RKHBIN
#RKHCKP=( "-c" "--sk" "--cronjob" "--rwo" )
RKHCKP=( "-c" "--sk" "--cronjob" "-q" )
RKHUDP=( "--propupd" "--cronjob" "-q" )
STDBIN="$(which systemctl||echo "/bin/systemctl")"
GTLBIN="$(which gitlab-ctl||echo "/usr/bin/gitlab-ctl")"
GTLUNT="gitlab-runsvdir.service"
NDRBIN="$(which needrestart||echo "/usr/sbin/needrestart")"
# -e: see RKHBIN
[ ! -e "$NDRBIN" ]&&unset NDRBIN
NDRCLP=( "-ra" "-blqp" )
NDRCKP=( "-ra" "-bkqp" )
RETSAV="\\033[s"
RETCLR="\\033[K"
RETRES="\\033[u"
RETOGE="$(tput cub 666)[ $(tput setaf 2)OK$(tput sgr0) ]$RETRES$RETCLR""."
RETWRN="$(tput cub 666)[$(tput setaf 3)WARN$(tput sgr0)]$RETRES$RETCLR""."
RETERR="$(tput cub 666)[$(tput setaf 1)FAIL$(tput sgr0)]$RETRES$RETCLR""."
SHUTUP=0
LOGDMP=0;LOGTARG="ln-update"
GTLUPD=0
RETVAL=0
retinit() {
echo -ne "[....] $1$RETSAV""..."
}
retinf() {
# Delivers the initial log output to be adjusted by return values
echo -e "$(tput cub 666)[$(tput setaf 7)INFO$(tput sgr0)]$RETRES$RETCLR"" ($(tput setaf 7)$1$(tput sgr0))."
}
gitlab-start() {
# Starts a stopped gitlab instance: unmasks gitlab-runsvdir, starts that service and executes
# gitlab-ctl start.
# RC:
# 3: Unmasking or starting the systemd service failed
# 4: gitlab-ctl start failed
[ "$SHUTUP" -ne 1 ]&&retinit "Enabling and launching Gitlab"
#"$SUDBIN" "$STDBIN" unmask "$GTLUNT" >/dev/null 2>&1
"$STDBIN" unmask "$GTLUNT" >/dev/null 2>&1
if [ "$?" -ne 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Unmasking Gitlab systemd unit failed."
return 3
fi
#"$SUDBIN" "$STDBIN" start "$GTLUNT" >/dev/null 2>&1
"$STDBIN" start "$GTLUNT" >/dev/null 2>&1
if [ "$?" -ne 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Launching Gitlab systemd unit failed."
return 3
fi
sleep 5
#"$SUDBIN" "$GTLBIN" start >/dev/null 2>&1
"$GTLBIN" start >/dev/null 2>&1
if [ "$?" -ne 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Launching Gitlab sub-units failed."
return 4
fi
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Gitlab enabled and started."
}
gitlab-stop() {
# Stops a started gitlab instance: execution of gitlab-ctl stop succeeded by a stop of
# gitlab-runvsdir.service and masking the latter.
# RC:
# 6: gitlab-ctl stop failed
# 7: systemd unit cannot be stopped or cannot be masked.
# 8: gitlab sub-units still running, i.e. gitlab-ctl stop did not complete everything; or
# the systemd unit is still running
[ "$SHUTUP" -ne 1 ]&&retinit "Stopping Gitlab"
#"$SUDBIN" "$GTLBIN" stop >/dev/null 2>&1
"$GTLBIN" stop >/dev/null 2>&1
if [ "$?" -ne 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Stopping Gitlab sub-units failed."
return 6
fi
#"$SUDBIN" "$STDBIN" stop "$GTLUNT" >/dev/null 2>&1
"$STDBIN" stop "$GTLUNT" >/dev/null 2>&1
if [ "$?" -ne 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Stopping Gitlab systemd unit failed."
return 7
fi
#"$SUDBIN" "$STDBIN" mask "$GTLUNT" >/dev/null 2>&1
"$STDBIN" mask "$GTLUNT" >/dev/null 2>&1
case "$?" in
0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE";;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Masking Gitlab systemd unit failed."
return 7
;;
esac
[ "$SHUTUP" -ne 1 ]&&retinit "Checking if Gitlab sub-units are stopped"
GLSVCCNT="$("$GTLBIN" status 2>&1|wc -l)"
"$GTLBIN" status >/dev/null 2>&1
GLSVCSTS="$?"
if [ "$GLSVCSTS" -lt "$GLSVCCNT" ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR (Stopped Gitlab units: $GLSVCSTS/$GLSVCCNT)"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Not all Gitlab sub-units stopped ($GLSVCSTS/$GLSVCCNT)."
return 8
fi
"$STDBIN" status "$GTLUNT" >/dev/null 2>&1
TRETVAL="$?"
case "$TRETVAL" in
3) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE (Stopped Gitlab units: $GLSVCSTS/$GLSVCCNT, systemd service shut down.)";;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR (systemd: $TRETVAL)"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Gitlab systemd unit cannot be stopped ($TRETVAL)."
return 8
;;
esac
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Gitlab stopped and disabled."
unset TRETVAL
}
rkhunter-ck() {
# Checks whether RKHunter reports any warnings or errors prior to updating.
# This is mainly because after the update we will do a property update to have
# the updated binaries and libraries updated in the RKHunter database. To -NOT-
# trash any previous errors we will report an error right away and stop any
# further execution so the admin can fix the situation and then re-run.
# RC:
# 1: RKHunter has any current error or warnings
[ "$SHUTUP" -ne 1 ]&&retinit "Checking RKHunter state"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Starting RKHunter status check."
#"$SUDBIN" "$RKHBIN" "${RKHCKP[@]}"
"$RKHBIN" "${RKHCKP[@]}"
case "$?" in
0)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "RKHunter status check finished."
;;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETWRN (check rkhunter logfile)"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "RKHunter state inconsistent."
return 1
;;
esac
}
rkhunter-upd() {
# Updates the RKHunter property database after the dist-upgrade so we will not confuse
# this with our mandatory system changes.
# RC:
# 1: if this step fails
[ "$SHUTUP" -ne 1 ]&&retinit "Updating RKHunter database"
"$RKHBIN" "${RKHUDP[@]}"
case "$?" in
0)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "RKHunter properties update successful."
;;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR (check rkhunter logfile)"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "RKHunter properties update failed."
return 1
;;
esac
}
needrestart-ck() {
# Uses needrestart (Debian package) to check whether any outdated kernel or service with outdated
# libraries are active. For this, we will use needrestart's autostart features, so be aware that we
# -WILL- restart services during the process.
# If there is any service that won't be restarted (needrestart is not unhealthily offensive, honest
# cheers to the devs from here) or if the running kernel is outdated posterior to our upgrades:
# RC:
# 255: Outdated services still active or running kernel outdated
LOCRETVAL=0
[ "$SHUTUP" -ne 1 ]&&retinit "Checking for services in need of restart"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Checking for services in need of restart..."
LLIBRES="$("$NDRBIN" "${NDRCLP[@]}")"
NAGRETVAL="$?"
case "$NAGRETVAL" in
0)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "No remaining services running with outdated libs."
;;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETWRN"&&echo "$LLIBRES"|sed 's/^[^-]\+-\ \([^|]\+\)|.*/\1/g'
[ "$LOGDMP" -eq 1 ]&&logger -p"warning" -t"$LOGTARG" "Some services could not be restarted: $LLIBRES"
LOCRETVAL=255
;;
esac
unset LLIBRES
[ "$SHUTUP" -ne 1 ]&&retinit "Checking for outdated kernel..."
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Checking for outdated kernel..."
LKRNRES="$("$NDRBIN" "${NDRCKP[@]}")"
NAGRETVAL="$?"
case "$NAGRETVAL" in
0)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Kernel is up-to-date."
;;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETWRN"&&echo "$LKRNRES"|sed 's/^[^-]\+-\ \([^|]\+\)|.*/\1/g'
[ "$LOGDMP" -eq 1 ]&&logger -p"warning" -t"$LOGTARG" "Kernel is outdated: $LKRNRES"
LOCRETVAL=255
;;
esac
unset LKRNRES
return "$LOCRETVAL"
}
while getopts :lq SHOPT;do
case "$SHOPT" in
q)SHUTUP=1;;
l)LOGDMP=1;;
esac
done
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Starting new run..."
[ "$SHUTUP" -ne 1 ]&&retinit "Checking root privileges"
#ROOTCK="$(sudo id -u)"
ROOTCK="$(id -u)"
case "$ROOTCK" in
0) [ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE";;
*)
[ "$SHUTUP" -ne 1 ]&&echo -ne "$RETERR"&&echo -e " (We need root rights)"
[ "$LOGDMP" -eq 1 ]&&logger -p"warning" -t"$LOGTARG" "Cancelled due to insufficient rights."
exit 1
;;
esac
[ "$SHUTUP" -ne 1 ]&&retinit "Updating Apt database"
#"$SUDBIN" "$APTBIN" -qqq update
"$APTBIN" -qqq update
if [ "$?" -ne 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Cannot update apt database."
exit 2
fi
# Make bash not break up the output into array elements on whitespaces but only on encountering line breaks
IFS=$'\n'
UPDARR=( $("$APTBIN" list --upgradable 2>/dev/null) )
# ...done.
unset IFS
UPDCNT="${#UPDARR[@]}"
unset UPDARR
case "$UPDCNT" in
# 0|1)
# [ "$SHUTUP" -ne 1 ]&&retinf "No updates available :)"
# exit 0
# ;;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Apt database uodated."
;;
esac
"$APTBIN" list --upgradable 2>/dev/null|grep gitlab-ce >/dev/null
if [ "$?" -eq 0 ];then
[ "$SHUTUP" -ne 1 ]&&echo "Encountered Gitlab update."
GTLUPD=1
fi
if [ ! -z "$RKHBIN" ];then rkhunter-ck||exit "$?";fi
if [ "$GTLUPD" -eq 1 ];then gitlab-start||exit "$?";fi
[ "$SHUTUP" -ne 1 ]&&retinit "Upgrading packages"
#"$SUDBIN" "$APTBIN" -qqq -y dist-upgrade >/dev/null 2>&1
"$APTBIN" -qqq -y dist-upgrade >/dev/null 2>&1
case "$?" in
0)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETOGE"
[ "$LOGDMP" -eq 1 ]&&logger -p"info" -t"$LOGTARG" "Apt dist-upgrade successful."
;;
*)
[ "$SHUTUP" -ne 1 ]&&echo -e "$RETERR"
[ "$LOGDMP" -eq 1 ]&&logger -p"error" -t"$LOGTARG" "Apt dist-upgrade failed."
exit 5
;;
esac
if [ "$GTLUPD" -eq 1 ];then gitlab-stop||exit "$?";fi
if [ ! -z "$RKHBIN" ];then rkhunter-upd||exit "$?";fi
if [ ! -z "$NDRBIN" ];then
needrestart-ck;NDRRETVAL="$?"
# because needrestart-ck only returns warning (RC255), we will only change the script RETVAL
# if it has been 0 prior to this:
if [ "$RETVAL" -eq 0 ];then RETVAL="$NDRRETVAL";fi
unset NDRRETVAL
fi
RETVAL="$?"
exit "$RETVAL"
|