From ff7221fc239feff781ccb133d65d5ddf9ab99fe5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 15 Dec 2021 13:27:23 +0100 Subject: [PATCH 001/215] Update 400_copy_modules.sh For MODULES=() nor MODULES=( 'moduleX' 'moduleY' ) include the currently loaded kernel modules, see https://github.com/rear/rear/issues/2727 --- usr/share/rear/build/GNU/Linux/400_copy_modules.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/build/GNU/Linux/400_copy_modules.sh b/usr/share/rear/build/GNU/Linux/400_copy_modules.sh index 8db9a69779..aac69b50b2 100644 --- a/usr/share/rear/build/GNU/Linux/400_copy_modules.sh +++ b/usr/share/rear/build/GNU/Linux/400_copy_modules.sh @@ -135,6 +135,10 @@ for dummy in "once" ; do # MODULES=( 'moduleX' 'moduleY' ) where additional kernel modules can be specified # to be included in the recovery system in addition to the ones via an empty MODULES=() setting: LogPrint "Copying kernel modules as specified by MODULES" + # Kernel modules that should be loaded during recovery system startup must be always copied into the recovery system: + MODULES+=( "${MODULES_LOAD[@]}" ) + # Kernel modules that are currently loaded are always copied into the recovery system: + MODULES+=( $( lsmod | tail -n +2 | cut -d ' ' -f 1 ) ) # Before ReaR version 2.5 the below added modules had been added via conf/GNU/Linux.conf # which is sourced in usr/sbin/rear before user config files like etc/rear/local.conf # so that the user had to specify MODULES+=( 'moduleX' 'moduleY' ) @@ -160,8 +164,7 @@ for dummy in "once" ; do zlib zlib-inflate zlib-deflate libcrc32c crc32c crc32c-intel ) # Include the modules in MODULES plus their dependant modules. - # Kernel modules that should be loaded during recovery system startup must be always copied into the recovery system: - for module in "${MODULES_LOAD[@]}" "${MODULES[@]}" ; do + for module in "${MODULES[@]}" ; do # Strip trailing ".o" if there: module=${module#.o} # Strip trailing ".ko" if there: From 8e8d77fb8251e5e9c174a12812ee279b85876f32 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 15 Dec 2021 13:50:39 +0100 Subject: [PATCH 002/215] Update 400_copy_modules.sh Fixed misleading comment. --- usr/share/rear/build/GNU/Linux/400_copy_modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/build/GNU/Linux/400_copy_modules.sh b/usr/share/rear/build/GNU/Linux/400_copy_modules.sh index aac69b50b2..fbcbf27081 100644 --- a/usr/share/rear/build/GNU/Linux/400_copy_modules.sh +++ b/usr/share/rear/build/GNU/Linux/400_copy_modules.sh @@ -127,7 +127,7 @@ for dummy in "once" ; do # Finally the fallback cases, i.e. when the user has specified # MODULES=() which means the currently loaded kernel modules get included in the recovery system - # plus the modules that get added above plus kernel modules for certain kernel drivers like + # plus the kernel modules in MODULES_LOAD plus kernel modules for certain kernel drivers like # storage drivers, network drivers, crypto drivers, virtualization drivers, and some extra drivers # (see rescue/GNU/Linux/230_storage_and_network_modules.sh # and rescue/GNU/Linux/240_kernel_modules.sh) From 3d504374a8e8bf45445025112d25bbcbad6d1958 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 8 Feb 2022 12:02:07 +0100 Subject: [PATCH 003/215] Update default.conf Simpler and better formatted description text for PXE_HTTP_URL --- usr/share/rear/conf/default.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 77fa93a41f..b99f48b288 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1146,7 +1146,8 @@ PXE_TFTP_PREFIX=$HOSTNAME. # # Optional HTTP download source for PXE (URL style) # for example PXE_HTTP_URL="http://pxe-over-http-srv:7777" -# If set an additional pxe boot option in the pxeconfig for this client is provided ('rear-http') which uses this url as base path for the kernel and initrd files. +# If set an additional PXE boot option 'rear-http' in the pxeconfig is provided +# which uses the specified URL as base path for kernel and initrd. PXE_HTTP_URL= # # Create pxelinux config symlinks for MAC addresses or for IP addresses ? [MAC|IP|''] From 53757eab1447c712fb7c8e44be9c8b3b3ffd9faa Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Feb 2022 13:24:36 +0100 Subject: [PATCH 004/215] Update 06-layout-configuration.adoc Enhance the "disk layout file syntax" description: Describe that one cannot rely on backward compatibility. Describe positional parameters vs. option=value parameters. --- doc/user-guide/06-layout-configuration.adoc | 72 ++++++++++++++++----- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/doc/user-guide/06-layout-configuration.adoc b/doc/user-guide/06-layout-configuration.adoc index 2cca537eeb..c1660b299c 100644 --- a/doc/user-guide/06-layout-configuration.adoc +++ b/doc/user-guide/06-layout-configuration.adoc @@ -575,13 +575,22 @@ system. == Disk layout file syntax == This section describes the syntax of all components in the Relax-and-Recover -layout file at +/var/lib/rear/layout/disklayout.conf+. +layout file /var/lib/rear/layout/disklayout.conf + +One cannot rely on backward compatibility between ReaR versions. +Normally the layout file /var/lib/rear/layout/disklayout.conf is created from scratch +for each run of "rear mkrescue/mkbackup" so a newer ReaR version creates it anew +with the right syntax for this ReaR version which is the exact same ReaR +that gets included in the ReaR recovery system together with this layout file +to recreate the disk layout during "rear recover". +Only when a selfmade /etc/rear/disklayout.conf is used then it must be adapted +by the user when he upgrades to a newer ReaR version. The syntax is of the form ---------------------------------- keyword value1 value2 ... ---------------------------------- -where keyword denotes one kind of component (dik, partition, filesystem, ...) +where keyword denotes one kind of component (disk, partition, filesystem, ...) and keyword and the values are separated by single space characters so that one can get the lines that belong to a particular component with a particular value1 via simple commands like @@ -590,24 +599,43 @@ grep "^keyword value1 " /var/lib/rear/layout/disklayout.conf ---------------------------------- (provided there is a value2 after value1 so there is a space after value1). -Normal text has to be present verbatim in the file. Angle -brackets "<" and ">" delimit a value that can be edited. Quotes " inside the -angle brackets indicate a verbatim option, often used together with a / to -indicate multiple options. Parenthesis "(" ")" inside explain the expected unit. No -unit suffix should be present, unless specifically indicated. Square brackets -"[" and "]" indicate an optional parameter. They can be excluded when -hand-crafting a layout file line. - No whitespace is allowed at the beginning of lines in the disklayout.conf file. Lines that start with a # (number sign, hash, or pound sign) are comments. -All other lines start with a component keyword. None of the component keywords -is a leading substring of another component keyword (e.g. disk is not a -leading substring of opaldisk) so that one can get the lines that -belong to a particular component via simple commands like +All other lines start with a component keyword. + +None of the component keywords is a leading substring of another component keyword +(e.g. disk is not a leading substring of opaldisk) so that one can get the lines +that belong to a particular component via simple commands like ---------------------------------- grep ^keyword /var/lib/rear/layout/disklayout.conf ---------------------------------- +For most component keywords the values are positional parameters +(if there is no value a dummy value like 'none' must be used) +so empty values in between are invalid syntax +which can result arbitrarily bad failures during "rear recover". + +For some component keywords the parameters have a form like +---------------------------------- +keyword value1 value2 optionA=valueA optionB=valueB ... +---------------------------------- +where the first ones are positional parameters but not the option=value parameters +so empty option=value parameters are allowed, for example see the 'raid' keyword. + +For details the matching scripts need to be inspected how things actually work +for a particular component keyword (i.e. what is implemented in the code). + +Syntax of the individual component keyword descriptions below: + +Normal text has to be present verbatim. +Angle brackets "<" and ">" delimit a value that can be edited. +Quotes " inside the angle brackets indicate a verbatim option, +often used together with a / to indicate multiple options. +Parenthesis "(" ")" inside explain the expected unit. +No unit suffix should be present, unless specifically indicated. +Square brackets "[" and "]" indicate an optional parameter. +They can be excluded when hand-crafting a layout file line. + === Disks === ---------------------------------- disk @@ -693,9 +721,19 @@ logicaldrive || raid= [boot=<[yn]>] [password=] ---------------------------------- + == Disk Restore Script (recover mode) == -The +/var/lib/rear/layout/disklayout.conf+ file is being used as input during +rear recover+ to create on-the-fly a script called +/var/lib/rear/layout/diskrestore.sh+. -When something goes wrong during the recreation of partitions, volume groups you will be thrown in edit mode and you can make the modification needed. However, it is desirable to have a preview mode before doing the recovery so you can review the +diskrestore.sh+ script before doing any recovery. It is better to find mistakes, obsolete arguments and so on before then later, right? -Gratien wrote a script to accomplish this (script is not part of ReaR) and is meant for debugging reasons only. For more details see http://www.it3.be/2016/06/08/rear-diskrestore/ +The +/var/lib/rear/layout/disklayout.conf+ file is being used as input +during +rear recover+ to create on-the-fly a script called +/var/lib/rear/layout/diskrestore.sh+. + +When something goes wrong during the recreation of partitions, volume groups, ... +you will be thrown in edit mode and you can make the modification needed. +However, it is desirable to have a preview mode before doing the recovery +so you can review the +diskrestore.sh+ script before doing any recovery. +It is better to find mistakes, obsolete arguments and so on before then later, right? + +Gratien wrote a script to accomplish this (script is not part of ReaR) +and is meant for debugging reasons only. +For more details see http://www.it3.be/2016/06/08/rear-diskrestore/ From 71ac3b4a9041c198c841663d0b149e9d608c6a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Renaud=20M=C3=A9trich?= Date: Tue, 1 Feb 2022 15:12:47 +0100 Subject: [PATCH 005/215] multipath: fix exclusion of still wanted devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current code excluding multipath devices is broken when a device being excluded matches other devices. This leads to excluding wanted devices. This happens when having custom alias for multipath devices *or* there are more than 26 multipath devices and 'mpatha' is getting excluded, which leads to excluding all 'mpathaX' devices are well. See example below: - have /boot on a dedicated multipath device 'mpathba' - have / on a dedicated multipath device 'mpatha' and LVM vg 'rhel' - have a dedicated multipath device 'mpathb' and LVM vg 'data' - tell ReaR to only include 'rhel' VG (and not 'data' VG) - tell ReaR to not auto-exclude multipath devices (otherwise everything gets commented out) ~~~ $ lsblk NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT sda 8:0 0 5G 0 disk └─mpathba 253:1 0 5G 0 mpath └─mpathba1 253:2 0 1G 0 part /boot sdb 8:16 0 5G 0 disk └─mpathba 253:1 0 5G 0 mpath └─mpathba1 253:2 0 1G 0 part /boot sdc 8:32 0 20G 0 disk └─mpatha 253:0 0 20G 0 mpath ├─mpatha1 253:3 0 1G 0 part └─mpatha2 253:4 0 19G 0 part ├─rhel-root 253:5 0 10G 0 lvm / └─rhel-swap 253:6 0 2G 0 lvm [SWAP] sdd 8:48 0 20G 0 disk └─mpatha 253:0 0 20G 0 mpath ├─mpatha1 253:3 0 1G 0 part └─mpatha2 253:4 0 19G 0 part ├─rhel-root 253:5 0 10G 0 lvm / └─rhel-swap 253:6 0 2G 0 lvm [SWAP] sde 8:64 0 2G 0 disk └─mpathb 253:7 0 2G 0 mpath sdf 8:80 0 2G 0 disk └─mpathb 253:7 0 2G 0 mpath sr0 11:0 1 1024M 0 rom $ vgs VG #PV #LV #SN Attr VSize VFree data 1 0 0 wz--n- <2.00g <2.00g rhel 1 2 0 wz--n- <19.00g <7.00g $ cat /etc/rear/local.conf ONLY_INCLUDE_VG=("rhel") AUTOEXCLUDE_MULTIPATH=n ~~~ With original code: ~~~ $ rear mkrescue $ grep -v ^# /var/lib/rear/layout/disklayout.conf | grep "mpathba" --> device excluded even though it hosts /boot ~~~ With the fix: ~~~ $ rear mkrescue $ grep -v ^# /var/lib/rear/layout/disklayout.conf | grep "mpathba" fs /dev/mapper/mpathba1 /boot xfs ... multipath /dev/mapper/mpathba 5368709120 msdos /dev/sda,/dev/sdb part /dev/mapper/mpathba 1073741824 1048576 primary none /dev/mapper/mpathba1 ~~~ The root cause behind this is ReaR excludes 'mpathb' (hosting the 'data' VG) through using a non-word matching *grep* command, which causes 'mpathba' (hosting /boot) to be excluded as well. Signed-off-by: Renaud Métrich --- .../save/default/335_remove_excluded_multipath_vgs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/layout/save/default/335_remove_excluded_multipath_vgs.sh b/usr/share/rear/layout/save/default/335_remove_excluded_multipath_vgs.sh index 040e9eecae..e731c994b8 100644 --- a/usr/share/rear/layout/save/default/335_remove_excluded_multipath_vgs.sh +++ b/usr/share/rear/layout/save/default/335_remove_excluded_multipath_vgs.sh @@ -19,9 +19,9 @@ while read lvmdev name mpdev junk ; do # Remember, multipath devices from a volume group that is "excluded" should be 'commented out' device=$(echo $mpdev | cut -c1-45) while read LINE ; do - # Now we need to comment all lines that contain "$devices" in the LAYOUT_FILE + # Now we need to comment all lines that contain "$device" in the LAYOUT_FILE sed -i "s|^$LINE|\#$LINE|" "$LAYOUT_FILE" - done < <(grep "$device" $LAYOUT_FILE | grep -v "^#") + done < <(grep " $device " $LAYOUT_FILE | grep -v "^#") DebugPrint "Disabling multipath device $device belonging to disabled 'lvmdev $name' in $LAYOUT_FILE" done < <(grep "^#lvmdev" $LAYOUT_FILE) @@ -31,7 +31,7 @@ done < <(grep "^#lvmdev" $LAYOUT_FILE) while read LINE ; do # multipath /dev/mapper/360060e8007e2e3000030e2e300002065 /dev/sdae,/dev/sdat,/dev/sdbi,/dev/sdp device=$(echo $LINE | awk '{print $2}' | cut -c1-45) - num=$(grep "$device" $LAYOUT_FILE | grep -v "^#" | wc -l) + num=$(grep " $device " $LAYOUT_FILE | grep -v "^#" | wc -l) if [ $num -lt 2 ] ; then # If the $device is only seen once (in a uncommented line) then the multipath is not in use sed -i "s|^$LINE|\#$LINE|" "$LAYOUT_FILE" From 09627f9a5a8ea3e427d3dc573148e2b4487687cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Renaud=20M=C3=A9trich?= Date: Tue, 28 Dec 2021 17:11:17 +0100 Subject: [PATCH 006/215] New EXCLUDE_IP_ADDRESSES and EXCLUDE_NETWORK_INTERFACES directives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These new array variables enable to exclude specific IP addresses or network interfaces when building the network configuration used in the rescue environment. This is typically useful when floating IP addresses are used. Not excluding these may lead to outage if the floating IP address is used by another system at time the system is getting recovered. Signed-off-by: Renaud Métrich --- usr/share/rear/conf/default.conf | 9 +++++++++ .../rear/rescue/GNU/Linux/310_network_devices.sh | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 30c2887dd6..7886cb07f3 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -2963,6 +2963,15 @@ WARN_MISSING_VOL_ID=1 USE_CFG2HTML= # The SKIP_CFG2HTML variable is no longer supported since ReaR 1.18 +# IP addresses that are present on the system but must be excluded when +# building the network configuration used in recovery mode; this is typically +# used when floating IP addresses are used on the system +EXCLUDE_IP_ADDRESSES=() + +# Network interfaces that are present on the system but must be excluded when +# building the network configuration used in recovery mode +EXCLUDE_NETWORK_INTERFACES=() + # Simplify bonding setups by configuring always the first active device of a # bond, except when mode is 4 (IEEE 802.3ad policy) SIMPLIFY_BONDING=no diff --git a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh index f806bfbf21..2385f5b6dc 100644 --- a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh +++ b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh @@ -355,6 +355,11 @@ function is_interface_up () { local network_interface=$1 local sysfspath=/sys/class/net/$network_interface + if IsInArray "$network_interface" "${EXCLUDE_NETWORK_INTERFACES[@]}"; then + LogPrint "Excluding '$network_interface' per EXCLUDE_NETWORK_INTERFACES directive." + return 1 + fi + local state=$( cat $sysfspath/operstate ) if [ "$state" = "down" ] ; then return 1 @@ -403,11 +408,19 @@ function ipaddr_setup () { if [ -n "$ipaddrs" ] ; then # If some IP is found for the network interface, then use them for ipaddr in $ipaddrs ; do + if IsInArray "${ipaddr%%/*}" "${EXCLUDE_IP_ADDRESSES[@]}"; then + LogPrint "Excluding IP address '$ipaddr' per EXCLUDE_IP_ADDRESSES directive even through it's defined in mapping file '$CONFIG_DIR/mappings/ip_addresses'." + continue + fi echo "ip addr add $ipaddr dev $mapped_as" done else # Otherwise, collect IP addresses for the network interface on the system for ipaddr in $( ip a show dev $network_interface scope global | grep "inet.*\ " | tr -s " " | cut -d " " -f 3 ) ; do + if IsInArray "${ipaddr%%/*}" "${EXCLUDE_IP_ADDRESSES[@]}"; then + LogPrint "Excluding IP address '$ipaddr' per EXCLUDE_IP_ADDRESSES directive." + continue + fi echo "ip addr add $ipaddr dev $mapped_as" done fi From ea0a74bd50493c7472600e2a523b06ae62ee508c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 23 Feb 2022 14:12:29 +0100 Subject: [PATCH 007/215] Update 305_include_s390_tools.sh Simpler code in prep/Linux-s390/305_include_s390_tools.sh to set a fixed bootdir="/boot" that also fixes ShellCheck SC1066 "Don't use $ on the left side of assignments", cf. https://github.com/rear/rear/issues/1040#issuecomment-1034890880 --- usr/share/rear/prep/Linux-s390/305_include_s390_tools.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/prep/Linux-s390/305_include_s390_tools.sh b/usr/share/rear/prep/Linux-s390/305_include_s390_tools.sh index 084ea31046..b4ab37866c 100644 --- a/usr/share/rear/prep/Linux-s390/305_include_s390_tools.sh +++ b/usr/share/rear/prep/Linux-s390/305_include_s390_tools.sh @@ -3,8 +3,13 @@ test -d $VAR_DIR/recovery || mkdir -p $VAR_DIR/recovery -local bootdir="$( echo -n /boot/ )" -test -d "$bootdir" || $bootdir='/boot/' +# See the code in prep/GNU/Linux/300_include_grub_tools.sh +# that sets grubdir via +# local grubdir="$( echo -n /boot/grub* )" +# where 'shopt -s nullglob' results nothing when nothing matches +# but that is not needed here to set a fixed bootdir="/boot" +# cf. https://github.com/rear/rear/issues/1040#issuecomment-1034890880 +local bootdir="/boot/" # cf. https://github.com/rear/rear/issues/2137 # findmnt is used the same as grub-probe to find the device where /boot is mounted From 8a9d0f8f3b1855ea9c5546e9c944896373a793e5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 23 Feb 2022 14:44:55 +0100 Subject: [PATCH 008/215] Update 220_lvm_layout.sh Added missing closing " in comment. --- usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh b/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh index 56e7956de0..278ffe2422 100644 --- a/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh @@ -2,7 +2,7 @@ # TODO: What if there are logical volumes on the system but there is no 'lvm' binary? # Shouldn't then "rear mkrescue" better error out here than to silently skip LVM altogether? -# Cf. "Try hard to care about possible errors in https://github.com/rear/rear/wiki/Coding-Style +# Cf. "Try hard to care about possible errors" in https://github.com/rear/rear/wiki/Coding-Style # Think about a minimal system that was set up by a (full featured) installation system # but tools to set up things were not installed in the (now running) installed system. # For example 'parted' is usually no longer needed in the installed system. From 0d871f8f6faf13897ccdd31b8a1c9bebd7560607 Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Mon, 28 Feb 2022 17:02:36 +0100 Subject: [PATCH 009/215] Update stale.yml Enlarge the inactivity days to 120 and 30 days before closure - give us more time to think about it --- .github/stale.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 527597ffed..14199129c1 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,9 +1,9 @@ # Number of days of inactivity before an issue becomes stale -# three month plus one weekend plus two more workdays 31+30+31+2+2=96 -daysUntilStale: 96 +# Let us go for 120 days +daysUntilStale: 120 # Number of days of inactivity before a stale issue is closed -# two weeks plus one weekend plus two more workdays 7+7+2+2=18 -daysUntilClose: 18 +# Pin it to 30 days +daysUntilClose: 30 # Issues with these labels will never be considered stale exemptLabels: - Dedicated Priority Support From 5b95e72e645a755a6eb8f55d91ef48ddab94e8ea Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 1 Mar 2022 11:43:45 +0100 Subject: [PATCH 010/215] Update stale.yml In stale.yml adapt the markComment according to the current daysUntilStale (=120) and daysUntilClose (=30) values, cf. https://github.com/rear/rear/commit/0d871f8f6faf13897ccdd31b8a1c9bebd7560607 --- .github/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 14199129c1..261d5fac66 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -15,8 +15,8 @@ staleLabel: ready-to-close? # Comment to post when marking an issue as stale. Set to `false` to disable markComment: > This issue has been automatically marked as stale - because it has not had recent activity for three month. - It will be closed if no further activity occurs within two weeks. + because it has not had recent activity for 120 days. + It will be closed if no further activity occurs within 30 days. Thank you for your contribution. # Comment to post when closing a stale issue. Set to `false` to disable closeComment: false From aea1b223c5298b30882b9a4f3c5d23d6853c9e6a Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Fri, 4 Mar 2022 16:43:28 +0100 Subject: [PATCH 011/215] #2500 - foresee a way to overrule the grub2 searcg string with EFI boot Signed-off-by: Gratien D'haese --- usr/share/rear/conf/default.conf | 7 +++++++ usr/share/rear/lib/bootloader-functions.sh | 3 +++ usr/share/rear/lib/uefi-functions.sh | 5 ++++- usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index d2ec193bfe..2d339229ab 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1063,6 +1063,13 @@ USB_BOOTLOADER= # if an USB device was manually mounted to avoid recursive backups: AUTOEXCLUDE_USB_PATH=() +# USB EFI booting can benifit with a better search string then the default: +# GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label REAR-EFI +# as hardcoded in script output/USB/Linux-i386/100_create_efiboot.sh +# Only to be used by experts. An example of a different setup could be: +# GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label REAR-EFI --hint hd0,msdos1" +GRUB2_SET_USB_ROOT="" + ## # OUTPUT=RAWDISK stuff ## diff --git a/usr/share/rear/lib/bootloader-functions.sh b/usr/share/rear/lib/bootloader-functions.sh index 2ac774ac65..a2ca4cd7ae 100644 --- a/usr/share/rear/lib/bootloader-functions.sh +++ b/usr/share/rear/lib/bootloader-functions.sh @@ -622,6 +622,7 @@ function create_grub2_cfg { cat << EOF menuentry "Relax-and-Recover (BIOS or UEFI without Secure Boot)" --id=rear { insmod gzio + insmod xzio echo 'Loading kernel $grub2_kernel ...' linux $grub2_kernel root=UUID=$root_uuid $KERNEL_CMDLINE echo 'Loading initial ramdisk $grub2_initrd ...' @@ -630,6 +631,7 @@ menuentry "Relax-and-Recover (BIOS or UEFI without Secure Boot)" --id=rear { menuentry "Relax-and-Recover (UEFI and Secure Boot)" --id=rear_secure_boot { insmod gzio + insmod xzio echo 'Loading kernel $grub2_kernel ...' linuxefi $grub2_kernel root=UUID=$root_uuid $KERNEL_CMDLINE echo 'Loading initial ramdisk $grub2_initrd ...' @@ -640,6 +642,7 @@ EOF cat << EOF menuentry "Relax-and-Recover (BIOS or UEFI in legacy BIOS mode)" --id=rear { insmod gzio + insmod xzio echo 'Loading kernel $grub2_kernel ...' linux $grub2_kernel root=UUID=$root_uuid $KERNEL_CMDLINE echo 'Loading initial ramdisk $grub2_initrd ...' diff --git a/usr/share/rear/lib/uefi-functions.sh b/usr/share/rear/lib/uefi-functions.sh index c109f618fe..6998d5d5f1 100644 --- a/usr/share/rear/lib/uefi-functions.sh +++ b/usr/share/rear/lib/uefi-functions.sh @@ -90,10 +90,13 @@ function build_bootx86_efi { fi if [ -n "$gprobe" ]; then - # this is unfortunately only a crude approximation of the Grub internal probe_mods() function + # This is unfortunately only a crude approximation of the Grub internal probe_mods() function. + # $gprobe --target=partmap "$p" | sed -e 's/^/part_/' does not always returns part_msdos + # Therefore, we explicit do an echo 'part_msdos' (the sort -u will make sure it is listed only once) modules=( $( for p in "${dirs[@]}" ; do $gprobe --target=fs "$p" $gprobe --target=partmap "$p" | sed -e 's/^/part_/' + echo 'part_msdos' $gprobe --target=abstraction "$p" done | sort -u ) ) fi diff --git a/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh b/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh index f0e27bc088..1f89473190 100644 --- a/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh +++ b/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh @@ -83,7 +83,7 @@ EOF # We need to explicitly set GRUB 2 'root' variable to $efi_label (hardcoded "REAR-EFI") # because default $root would point to memdisk, where kernel and initrd are NOT present. # GRUB2_SET_USB_ROOT is used in the create_grub2_cfg() function: - GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label $efi_label" + [[ -z "$GRUB2_SET_USB_ROOT" ]] && GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label $efi_label" # Create config for GRUB 2 create_grub2_cfg $efi_dir/kernel $efi_dir/$REAR_INITRD_FILENAME > $efi_dst/grub.cfg # Create bootloader, this overwrite BOOTX64.efi copied in previous step ... From b3822d9c493aa74e524f106b9888282371c226aa Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 7 Mar 2022 13:38:59 +0100 Subject: [PATCH 012/215] Update 500_make_backup.sh In backup/YUM/default/500_make_backup.sh fixed ShellCheck SC2000: See if you can use ${#variable} instead https://github.com/koalaman/shellcheck/wiki/SC2000 The code before was curCmdLen=$(echo "$cmd2" | wc -c) so curCmdLen is ${#cmd2} + 1 because of the newline of 'echo' but I don't know for sure if + 1 is needed or not so I keep it via curCmdLen=(( ${#cmd2} + 1 )) cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- usr/share/rear/backup/YUM/default/500_make_backup.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/backup/YUM/default/500_make_backup.sh b/usr/share/rear/backup/YUM/default/500_make_backup.sh index 18b5d105d0..1687a0223f 100644 --- a/usr/share/rear/backup/YUM/default/500_make_backup.sh +++ b/usr/share/rear/backup/YUM/default/500_make_backup.sh @@ -61,7 +61,14 @@ if is_true "$YUM_BACKUP_FILES_FULL_EXCL" ; then } || { cmd2=$(echo -n "$cmd2 -samefile $fname") } - curCmdLen=$(echo "$cmd2" | wc -c) + # Aviod ShellCheck + # SC2000: See if you can use ${#variable} instead + # https://github.com/koalaman/shellcheck/wiki/SC2000 + # The code before was + # curCmdLen=$(echo "$cmd2" | wc -c) + # so curCmdLen is ${#cmd2} + 1 because of the newline of 'echo' + # but I don't know for sure if + 1 is needed or not so I keep it: + curCmdLen=$(( ${#cmd2} + 1 )) [ $curCmdLen -gt $maxArgLen ] && { # Simple "something is still going on" indicator by printing dots # directly to stdout which is fd7 (see lib/_input-output-functions.sh) From 8965c0a1e486f55c1712a8a70753af66410d5fda Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 7 Mar 2022 13:52:06 +0100 Subject: [PATCH 013/215] Update 380_request_client_destination.sh In verify/NBU/default/380_request_client_destination.sh fixed ShellCheck SC2018: Use '[:lower:]' to support accents and foreign alphabets and SC2019: Use '[:upper:]' to support accents and foreign alphabets https://github.com/koalaman/shellcheck/wiki/SC2018 cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 and have the while loop on several lines as usual. --- .../verify/NBU/default/380_request_client_destination.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/verify/NBU/default/380_request_client_destination.sh b/usr/share/rear/verify/NBU/default/380_request_client_destination.sh index f0929558d0..6aa5ba6edc 100644 --- a/usr/share/rear/verify/NBU/default/380_request_client_destination.sh +++ b/usr/share/rear/verify/NBU/default/380_request_client_destination.sh @@ -4,7 +4,12 @@ # OR Request the user to hit ENTER to do a normal restore to the same client. # read NBU vars from NBU config file bp.conf -while read KEY VALUE ; do echo "$KEY" | grep -qi '^#' && continue ; test -z "$KEY" && continue ; KEY="$(echo "$KEY" | tr a-z A-Z)" ; export NBU_$KEY="$(echo "$VALUE" | sed -e 's/=//' -e 's/ //g')" ; done Date: Mon, 7 Mar 2022 14:20:34 +0100 Subject: [PATCH 014/215] Update 230_filesystem_layout.sh In layout/save/GNU/Linux/230_filesystem_layout.sh fixed ShellCheck SC2030: Modification of tunefs is local (to subshell caused by (..) group) https://github.com/koalaman/shellcheck/wiki/SC2031 cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- .../rear/layout/save/GNU/Linux/230_filesystem_layout.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh b/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh index 28548110b7..fb627a19f4 100644 --- a/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh @@ -545,15 +545,17 @@ grep -q '^fs ' $DISKLAYOUT_FILE && REQUIRED_PROGS+=( mkfs ) # (see above supported_filesystems="ext2,ext3,ext4,vfat,xfs,reiserfs,btrfs"): required_mkfs_tools="" for filesystem_type in $( echo $supported_filesystems | tr ',' ' ' ) ; do - grep -q "^fs .* $filesystem_type " $DISKLAYOUT_FILE && required_mkfs_tools="$required_mkfs_tools mkfs.$filesystem_type" + grep -q "^fs .* $filesystem_type " $DISKLAYOUT_FILE && required_mkfs_tools+=" mkfs.$filesystem_type" done # Remove duplicates because in disklayout.conf there can be many entries with same filesystem type: required_mkfs_tools="$( echo $required_mkfs_tools | tr ' ' '\n' | sort -u | tr '\n' ' ' )" REQUIRED_PROGS+=( $required_mkfs_tools ) # mke2fs is also required in the recovery system if any 'mkfs.ext*' filesystem creating tool is required # and tune2fs or tune4fs is used to set tunable filesystem parameters on ext2/ext3/ext4 -# see above how $tunefs is set to tune2fs or tune4fs -echo $required_mkfs_tools | grep -q 'mkfs.ext' && REQUIRED_PROGS+=( mke2fs $tunefs ) +# cf. above how $tunefs is set to tune2fs or tune4fs inside the subshell +# i.e. $tunefs is not set here so REQUIRED_PROGS+=( $tunefs ) would do nothing +# but tune2fs and tune4fs get included via PROGS in conf/GNU/Linux.conf which should be sufficient: +echo $required_mkfs_tools | grep -q 'mkfs.ext' && REQUIRED_PROGS+=( mke2fs ) # xfs_admin is also required in the recovery system if 'mkfs.xfs' is required: echo $required_mkfs_tools | grep -q 'mkfs.xfs' && REQUIRED_PROGS+=( xfs_admin ) # reiserfstune is also required in the recovery system if 'mkfs.reiserfs' is required: From 41386828bcc4b48c7734a891e118feb98e9c2db3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 7 Mar 2022 14:37:33 +0100 Subject: [PATCH 015/215] Update 400_create_local_efi_dir.sh In output/ISO/Linux-ia64/400_create_local_efi_dir.sh fixed ShellCheck SC2050: This expression is constant. Did you forget the $ on a variable? https://github.com/koalaman/shellcheck/wiki/SC2050 cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 The old code was [ CREATE_LOCAL_EFI_DIR = false ] && return and now it is is_true $CREATE_LOCAL_EFI_DIR || return 0 because CREATE_LOCAL_EFI_DIR=true is set in usr/share/rear/conf/Linux-ia64.conf plus LogPrint user info when local /boot/efi/efi/rear gets polpulated because of this. --- .../ISO/Linux-ia64/400_create_local_efi_dir.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/usr/share/rear/output/ISO/Linux-ia64/400_create_local_efi_dir.sh b/usr/share/rear/output/ISO/Linux-ia64/400_create_local_efi_dir.sh index 2ef81e5939..7b0788a059 100644 --- a/usr/share/rear/output/ISO/Linux-ia64/400_create_local_efi_dir.sh +++ b/usr/share/rear/output/ISO/Linux-ia64/400_create_local_efi_dir.sh @@ -1,9 +1,10 @@ # create_local_efi_dir.sh script -# useful for testing the rescue boot procedure on local disk instead of -# burning an ISO image to CD to boot from +# useful for testing the rescue boot procedure on local disk +# instead of burning an ISO image to CD to boot from # Lives under /boot/efi/efi/rear -# set the variable in config file /etc/rear/Linux-ia64.conf -[ CREATE_LOCAL_EFI_DIR = false ] && return -[ ! -d /boot/efi/efi/rear ] && mkdir -p /boot/efi/efi/rear -cp $v $TMP_DIR/mnt/boot/* /boot/efi/efi/rear/ >&2 -Log "Populated the local EFI boot directory /boot/efi/efi/rear" +# CREATE_LOCAL_EFI_DIR=true is set in usr/share/rear/conf/Linux-ia64.conf +# but not mentioned in default.conf - nevertheless the user may set it to false: +is_true $CREATE_LOCAL_EFI_DIR || return 0 +test -d /boot/efi/efi/rear || mkdir -p /boot/efi/efi/rear +cp $v $TMP_DIR/mnt/boot/* /boot/efi/efi/rear/ +LogPrint "Populated local EFI boot directory /boot/efi/efi/rear (CREATE_LOCAL_EFI_DIR is true)" From 201ff677b9215e8546bae77d1a63b54bc817b22e Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Mon, 7 Mar 2022 14:55:45 +0100 Subject: [PATCH 016/215] #2500 rename GRUB2_SET_USB_ROOT into GRUB2_SEARCH_ROOT_COMMAND Signed-off-by: Gratien D'haese --- usr/share/rear/conf/default.conf | 10 ++++++---- usr/share/rear/lib/bootloader-functions.sh | 16 ++++++++-------- .../output/USB/Linux-i386/100_create_efiboot.sh | 4 ++-- .../output/USB/Linux-i386/300_create_grub.sh | 4 ++-- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 2d339229ab..b56a67c4a2 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1063,12 +1063,14 @@ USB_BOOTLOADER= # if an USB device was manually mounted to avoid recursive backups: AUTOEXCLUDE_USB_PATH=() -# USB EFI booting can benifit with a better search string then the default: -# GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label REAR-EFI +# USB EFI booting can benifit with a better search string than the default: +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI # as hardcoded in script output/USB/Linux-i386/100_create_efiboot.sh # Only to be used by experts. An example of a different setup could be: -# GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label REAR-EFI --hint hd0,msdos1" -GRUB2_SET_USB_ROOT="" +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI --hint hd0,msdos1" +# or +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --file /EFI/BOOT/BOOTX86.efi" +GRUB2_SEARCH_ROOT_COMMAND="" ## # OUTPUT=RAWDISK stuff diff --git a/usr/share/rear/lib/bootloader-functions.sh b/usr/share/rear/lib/bootloader-functions.sh index a2ca4cd7ae..435c15ea6e 100644 --- a/usr/share/rear/lib/bootloader-functions.sh +++ b/usr/share/rear/lib/bootloader-functions.sh @@ -542,15 +542,15 @@ function create_grub2_cfg { local grub2_initrd="$2" test "$grub2_initrd" || BugError "create_grub2_cfg function called without grub2_initrd argument" DebugPrint "Configuring GRUB2 initrd $grub2_initrd" - local grub2_set_root_command="$3" - if ! test "$grub2_set_root_command" ; then - test "$grub2_set_root" && grub2_set_root_command="set root=$grub2_set_root" + local grub2_search_root_command="$3" + if ! test "$grub2_search_root_command" ; then + test "$grub2_set_root" && grub2_search_root_command="set root=$grub2_set_root" fi - if ! test "$grub2_set_root_command" ; then - test "$GRUB2_SET_USB_ROOT" && grub2_set_root_command="$GRUB2_SET_USB_ROOT" + if ! test "$grub2_search_root_command" ; then + test "$GRUB2_SEARCH_ROOT_COMMAND" && grub2_search_root_command="$GRUB2_SEARCH_ROOT_COMMAND" fi - test "$grub2_set_root_command" || grub2_set_root_command="search --no-floppy --set=root --file /boot/efiboot.img" - DebugPrint "Configuring GRUB2 root device as '$grub2_set_root_command'" + test "$grub2_search_root_command" || grub2_search_root_command="search --no-floppy --set=root --file /boot/efiboot.img" + DebugPrint "Configuring GRUB2 root device as '$grub2_search_root_command'" local grub2_default_menu_entry="$GRUB2_DEFAULT_BOOT" test "$grub2_default_menu_entry" || grub2_default_menu_entry="chainloader" @@ -713,7 +713,7 @@ EOF # Sleep 3 seconds before the GRUB2 menu replaces what there is on the screen # so that the user has a chance to see possible (error) messages on the screen. cat << EOF -$grub2_set_root_command +$grub2_search_root_command insmod all_video set gfxpayload=keep insmod part_gpt diff --git a/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh b/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh index 1f89473190..7015ff44ad 100644 --- a/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh +++ b/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh @@ -82,8 +82,8 @@ EOF DebugPrint "Configuring GRUB2 for EFI boot" # We need to explicitly set GRUB 2 'root' variable to $efi_label (hardcoded "REAR-EFI") # because default $root would point to memdisk, where kernel and initrd are NOT present. - # GRUB2_SET_USB_ROOT is used in the create_grub2_cfg() function: - [[ -z "$GRUB2_SET_USB_ROOT" ]] && GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label $efi_label" + # GRUB2_SEARCH_ROOT_COMMAND is used in the create_grub2_cfg() function: + [[ -z "$GRUB2_SEARCH_ROOT_COMMAND" ]] && GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label $efi_label" # Create config for GRUB 2 create_grub2_cfg $efi_dir/kernel $efi_dir/$REAR_INITRD_FILENAME > $efi_dst/grub.cfg # Create bootloader, this overwrite BOOTX64.efi copied in previous step ... diff --git a/usr/share/rear/output/USB/Linux-i386/300_create_grub.sh b/usr/share/rear/output/USB/Linux-i386/300_create_grub.sh index 16eceac018..9b6e3dc658 100644 --- a/usr/share/rear/output/USB/Linux-i386/300_create_grub.sh +++ b/usr/share/rear/output/USB/Linux-i386/300_create_grub.sh @@ -59,6 +59,6 @@ test "$USB_DEVICE_BOOT_LABEL" || USB_DEVICE_BOOT_LABEL="REARBOOT" # We need to set the GRUB environment variable 'root' to the partition device with filesystem label USB_DEVICE_BOOT_LABEL # because GRUB's default 'root' (or GRUB's 'root' identifcation heuristics) would point to the ramdisk but neither kernel # nor initrd are located on the ramdisk but on the partition device with filesystem label USB_DEVICE_BOOT_LABEL. -# GRUB2_SET_USB_ROOT is used in the create_grub2_cfg() function: -GRUB2_SET_USB_ROOT="search --no-floppy --set=root --label $USB_DEVICE_BOOT_LABEL" +# GRUB2_SEARCH_ROOT_COMMAND is used in the create_grub2_cfg() function: +GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label $USB_DEVICE_BOOT_LABEL" create_grub2_cfg /$USB_PREFIX/kernel /$USB_PREFIX/$REAR_INITRD_FILENAME > $usb_boot_dir/$grub_cfg || Error "Failed to create $usb_boot_dir/$grub_cfg" From c253f8d4d816e35d6e21a779ac5fb94779572df2 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 8 Mar 2022 15:59:40 +0100 Subject: [PATCH 017/215] Update 210_raid_layout.sh In layout/save/GNU/Linux/210_raid_layout.sh fixed the RAID10 layout support code as far as possible for now. Currently only 'near=...' support is (and was) implemented, cf. https://github.com/rear/rear/commit/4dbbb288057189076d82cb8e477cd994a4ce1851 This fix was triggered by ShellCheck SC2034: param appears unused. Verify use (or export if used externally) https://github.com/koalaman/shellcheck/wiki/SC2034 SC2066: Since you double quoted this, it will not word split, and the loop will only run once https://github.com/koalaman/shellcheck/wiki/SC2066 cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- .../layout/save/GNU/Linux/210_raid_layout.sh | 63 ++++++++++++++++--- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh index ea482b86f3..03c81b89c2 100644 --- a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh @@ -17,7 +17,7 @@ local basename local line local metadata level raid_devices uuid name spare_devices layout chunksize component_devices container_size local container array_size -local param copies number +local layout_option layout_option_name layout_near_option_value layout_near_option local component_device local raid_layout_entry local raid_dev_size raid_dev_label @@ -186,19 +186,64 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d uuid=${line[2]} test $uuid && raid_layout_entry+=" uuid=$uuid" + # A "Layout :" line in the detailed mdadm output looks like + # Layout : near=2 + # or + # Layout : near=2, far=2 + # or + # Layout : near=2, far=1 + # cf. https://linux-blog.anracom.com/tag/far-layout/ + # and https://unix.stackexchange.com/questions/280283/is-it-possible-to-create-a-mdadm-raid10-with-both-near-and-far-layout-options + # and https://ubuntuforums.org/showthread.php?t=1689828&page=4 + # where for those examples the line array becomes ("declare -p line" outputs): + # declare -a line=([0]="Layout" [1]=":" [2]="near=2") + # declare -a line=([0]="Layout" [1]=":" [2]="near=2," [3]="far=2") + # declare -a line=([0]="Layout" [1]=":" [2]="near=2," [3]="far=1") + # so "near=..." is array element number 2 and "far=..." is array element number 3 if it exists: line=( $( grep "Layout :" $mdadm_details ) ) - layout=${line[2]} - # fix up layout for RAID10: - # > near=2,far=1 -> n2 + # We use ${line[3]:-} to be safe against "bash: line[3]: unbound variable" in case of 'set -eu' + # so for the above examples the layout string becomes: + # near=2 + # near=2,far=2 + # near=2,far=1 + layout="${line[2]}${line[3]:-}" + # For RAID10 have the layout value what the mdadm command needs as --layout option value + # so with the above examples the mdadm command option --layout=... value has to become + # near=2 -> n2 + # near=2,far=2 -> n2 + # near=2,far=1 -> n2 + # TODO: Currently the RAID10 layout "far=..." value is ignored because according to + # https://unix.stackexchange.com/questions/280283/is-it-possible-to-create-a-mdadm-raid10-with-both-near-and-far-layout-options + # it seems it is not possible (or it does not make sense in practice) to set both "near=..." and "far=..." + # see also the initial implementation https://github.com/rear/rear/commit/4dbbb288057189076d82cb8e477cd994a4ce1851 + # where no support for the "far=..." value was implemented which had been OK since then (dated May 24, 2011): if test "$level" = "raid10" ; then + layout_near_option="" OIFS=$IFS IFS="," - for param in "$layout" ; do - copies=${layout%=*} - number=${layout#*=} - test "$number" -gt 1 && layout="${copies:0:1}$number" + for layout_option in $layout ; do + layout_option_name=${layout_option%=*} + # Only use the "near=..." value but at least inform the user: + if ! test "$layout_option_name" = "near" ; then + LogPrintError "RAID10 layout option '$layout_option' is ignored (only 'near=...' support is implemented)" + continue + fi + layout_near_option_value=${layout_option#*=} + # TODO: I do not understand why layout=n1 is not set by ReaR because + # the default is layout=n2 (according to 'man mdadm' with mdadm v4.1 on openSUSE Leap 15.3) + # so when a user has "Layout : near=1" then ReaR should recreate it as it was before + # regardless if such a RAID10 layout makes sense in practice + # so we tell the user at least in debug mode: + if ! test "$layout_near_option_value" -gt 1 ; then + DebugPrint "RAID10 layout option '$layout_option' is ignored (its value must be greater than 1)" + continue + fi + layout_near_option="n$layout_near_option_value" done IFS=$OIFS + # Ensure $layout_near_option is a single non empty and non blank word + # (no quoting because test " " returns zero exit code): + test $layout_near_option && layout="$layout_near_option" || layout="" fi # mdadm can print '-unknown-' for a RAID layout # which got recently (2019-12-02) added to RAID0 (it existed before for RAID5 and RAID6 and RAID10) see @@ -207,6 +252,8 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d # will create a 'mdadm' command in diskrestore.sh like "mdadm ... --layout=-unknown- ..." which would fail # during "rear recover" with something like "mdadm: layout -unknown- not understood for raid0" # see https://github.com/rear/rear/issues/2616 + # and ensure $layout is a single non empty and non blank word + # (no quoting because test " " returns zero exit code): test $layout -a '-unknown-' != "$layout" && raid_layout_entry+=" layout=$layout" chunksize=$( grep "Chunk Size" $mdadm_details | tr -d " " | cut -d ":" -f "2" | sed -r 's/^([0-9]+).+/\1/') From dc842a4a5f0d3874cdb2f1843d8f1aa7922896aa Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 08:26:46 +0100 Subject: [PATCH 018/215] Update 100_include_partition_code.sh In layout/prepare/GNU/Linux/100_include_partition_code.sh fixed ShellCheck SC2199: Arrays implicitly concatenate in [[ ]]. Use a loop (or explicit * instead of @) https://github.com/koalaman/shellcheck/wiki/SC2199 and SC2076: Don't quote right-hand side of =~, it'll match literally rather than as a regex https://github.com/koalaman/shellcheck/wiki/SC2076 cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- .../layout/prepare/GNU/Linux/100_include_partition_code.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/layout/prepare/GNU/Linux/100_include_partition_code.sh b/usr/share/rear/layout/prepare/GNU/Linux/100_include_partition_code.sh index d25bc40908..84a7cd3328 100644 --- a/usr/share/rear/layout/prepare/GNU/Linux/100_include_partition_code.sh +++ b/usr/share/rear/layout/prepare/GNU/Linux/100_include_partition_code.sh @@ -218,8 +218,10 @@ EOF end=$( mathlib_calculate "$end - 1" ) fi if [[ "$ARCH" == "Linux-s390" ]] ; then - # if dasd disk is LDL formated, then do not partition it, because it is partitioned and can take only partition - if [[ ! "${listDasdLdl[@]}" =~ "$device" ]] ; then + # LDL formatted disks are already partitioned and should not be partitioned with parted or fdasd , it will fail + # the listDasdLdl array contains devices such as /dev/dasdb that are formatted as LDL + # listDasdLdl is set in layout/prepare/Linux-s390/205_s390_enable_disk.sh + if ! IsInArray "$device" "${listDasdLdl[@]}" ; then echo "not LDL dasd formated disk, create a partition" cat >> "$LAYOUT_CODE" < Date: Wed, 9 Mar 2022 09:20:20 +0100 Subject: [PATCH 019/215] Update 150_restore_duply.sh In restore/DUPLICITY/default/150_restore_duply.sh avoid ShellCheck SC2103: Use a ( subshell ) to avoid having to cd back https://github.com/koalaman/shellcheck/wiki/SC2103 by using 'pushd' and 'popd' as we do at many other places instead of 'cd' and 'cd -' which was only used here, cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- usr/share/rear/restore/DUPLICITY/default/150_restore_duply.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/restore/DUPLICITY/default/150_restore_duply.sh b/usr/share/rear/restore/DUPLICITY/default/150_restore_duply.sh index 5727eb744b..12303f3c95 100644 --- a/usr/share/rear/restore/DUPLICITY/default/150_restore_duply.sh +++ b/usr/share/rear/restore/DUPLICITY/default/150_restore_duply.sh @@ -22,7 +22,7 @@ if [ "$BACKUP_PROG" = "duply" ] && has_binary duply; then DUPLY_RESTORE_OK="y" # we need to move up one dir (to get restore almost empty) - cd $TARGET_FS_ROOT + pushd $TARGET_FS_ROOT >/dev/null # file $VAR_DIR/recovery/mountpoint_device contains the mount points in / /boot etc order # we need to reverse it - to avoid tac we use sed instead @@ -36,7 +36,7 @@ if [ "$BACKUP_PROG" = "duply" ] && has_binary duply; then [[ ! -d $TARGET_FS_ROOT/proc ]] && mkdir -m 555 $TARGET_FS_ROOT/proc [[ ! -d $TARGET_FS_ROOT/tmp ]] && mkdir -m 4777 $TARGET_FS_ROOT/tmp - cd - >/dev/null + popd >/dev/null fi fi fi From 5f2f908f60122ae5b4eeb9bc68df931a0fcf5f2e Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Wed, 9 Mar 2022 09:42:42 +0100 Subject: [PATCH 020/215] #2500 fix typo in conf/default.conf Signed-off-by: Gratien D'haese --- usr/share/rear/conf/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index b56a67c4a2..60181cd09b 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1063,7 +1063,7 @@ USB_BOOTLOADER= # if an USB device was manually mounted to avoid recursive backups: AUTOEXCLUDE_USB_PATH=() -# USB EFI booting can benifit with a better search string than the default: +# USB EFI booting can benefit with a better search string than the default: # GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI # as hardcoded in script output/USB/Linux-i386/100_create_efiboot.sh # Only to be used by experts. An example of a different setup could be: From cdbc35713c56c52b05a396c1c5d0d8be6ec071e0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 10:19:02 +0100 Subject: [PATCH 021/215] Update 00-functions.sh In skel/default/etc/scripts/system-setup.d/00-functions.sh ignore ShellCheck SC2119: Use my_udevsettle "$@" if function's $1 should mean script's $1 SC2120: my_udevsettle references arguments, but none are ever passed cf. https://github.com/rear/rear/issues/1040#issuecomment-1062703092 --- .../skel/default/etc/scripts/system-setup.d/00-functions.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh b/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh index 4a353dafd9..5bd5f2429c 100644 --- a/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh +++ b/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh @@ -6,12 +6,18 @@ my_udevtrigger() { # If systemd is running, this should help to rename devices if [[ $(ps --no-headers -C systemd) ]]; then sleep 1 + # See https://github.com/rear/rear/issues/1040#issuecomment-1062703092 + # SC2119: Use my_udevsettle "$@" if function's $1 should mean script's $1. + # shellcheck disable=SC2119 my_udevsettle udevadm trigger --action=add fi } # call udevsettle +# See https://github.com/rear/rear/issues/1040#issuecomment-1062703092 +# SC2120: my_udevsettle references arguments, but none are ever passed. +# shellcheck disable=SC2120 my_udevsettle() { type -p udevadm >/dev/null && udevadm settle --timeout=10 $@ || udevsettle $@ } From f13f2d1bc7e17a93bdf22143f8e2d9f039fe1870 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 11:03:07 +0100 Subject: [PATCH 022/215] Update layout-functions.sh In lib/layout-functions.sh fixed ShellCheck SC2178: Variable was used as an array but is now assigned a string https://github.com/koalaman/shellcheck/wiki/SC2178 by not using the 'disks' variable as an array for the 'raid' case but also there as a string as in the 'multipath' case in that code because device nodes are single words so a string is sufficient cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- usr/share/rear/lib/layout-functions.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/usr/share/rear/lib/layout-functions.sh b/usr/share/rear/lib/layout-functions.sh index 8643ddffa2..ffaef148df 100644 --- a/usr/share/rear/lib/layout-functions.sh +++ b/usr/share/rear/lib/layout-functions.sh @@ -164,8 +164,8 @@ generate_layout_dependencies() { ;; raid) name=$(echo "$remainder" | cut -d " " -f "1") - disks=( $(echo "$remainder" | sed -r "s/.*devices=([^ ]+).*/\1/" | tr ',' ' ') ) - for disk in "${disks[@]}" ; do + disks=$(echo "$remainder" | sed -r "s/.*devices=([^ ]+).*/\1/" | tr ',' ' ') + for disk in $disks ; do add_dependency "$name" "$disk" done add_component "$name" "raid" @@ -211,9 +211,7 @@ generate_layout_dependencies() { multipath) name=$(echo "$remainder" | cut -d " " -f "1") disks=$(echo "$remainder" | cut -d " " -f "4" | tr "," " ") - add_component "$name" "multipath" - for disk in $disks ; do add_dependency "$name" "$disk" done From 208147017cd60a9a6b688f36c1c594e4d3bb60dd Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 11:40:11 +0100 Subject: [PATCH 023/215] Update filesystems-functions.sh In lib/filesystems-functions.sh in function xfs_parse fixed ShellCheck SC2179: Use array+=("item") to append items to an array https://github.com/koalaman/shellcheck/wiki/SC2179 and SC2128: Expanding an array without an index only gives the first element https://github.com/koalaman/shellcheck/wiki/SC2128 by not initializing 'xfs_opts' as an empty array but as an empty string because 'xfs_opts' is used as a string everywhere in the actual code cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 furthermore 'xfs_opts' is local in both functions xfs_parse and create_fs in layout/prepare/GNU/Linux/131_include_filesystem_code.sh --- usr/share/rear/lib/filesystems-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/filesystems-functions.sh b/usr/share/rear/lib/filesystems-functions.sh index afdd3f24c7..985a9f26c1 100644 --- a/usr/share/rear/lib/filesystems-functions.sh +++ b/usr/share/rear/lib/filesystems-functions.sh @@ -36,8 +36,8 @@ function btrfs_subvolume_exists() { function xfs_parse { - xfs_opts=() local xfs_opt_file=$1 + local xfs_opts="" # Check if we can read configuration file produced by xfs_info. # Fall back to mkfs.xfs defaults if trouble with configuration file occur. From 5abc34bb6bc3adb18f55ea07a1825783ce16fe59 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 11:51:23 +0100 Subject: [PATCH 024/215] Update sesam-functions.sh In rear/lib/sesam-functions.sh fixed ShellCheck SC2221: This pattern always overrides a later one on line 28 https://github.com/koalaman/shellcheck/wiki/SC2221 by removing the duplicated specified case (gv_rw_work) SESAM_WORK_DIR="$value" ;; which is already specified on line 24, cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- usr/share/rear/lib/sesam-functions.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/usr/share/rear/lib/sesam-functions.sh b/usr/share/rear/lib/sesam-functions.sh index 63a1575109..b6f6c43e2d 100644 --- a/usr/share/rear/lib/sesam-functions.sh +++ b/usr/share/rear/lib/sesam-functions.sh @@ -25,7 +25,6 @@ while IFS== read key value ; do (gv_rw_tmp) SESAM_TMP_DIR="$value" ;; (gv_rw_lis) SESAM_LIS_DIR="$value" ;; (gv_rw_lgc) SESAM_LGC_DIR="$value" ;; - (gv_rw_work) SESAM_WORK_DIR="$value" ;; (gv_rw_stpd) SESAM_SMS_DIR="$value" ;; (gv_rw_prot) SESAM_PROT_DIR="$value" ;; esac From 83d2944ce73cc0ee0e22d15b4afe22f8a9b95d77 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 12:10:37 +0100 Subject: [PATCH 025/215] Update 260_create_syslinux_efi_bootloader.sh In output/RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh fixed ShellCheck SC2235: Use { ..; } instead of (..) to avoid subshell overhead https://github.com/koalaman/shellcheck/wiki/SC2235 cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 --- .../RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/output/RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh b/usr/share/rear/output/RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh index cca8caf6b0..e08184139f 100644 --- a/usr/share/rear/output/RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh +++ b/usr/share/rear/output/RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh @@ -15,7 +15,9 @@ local syslinux_efi="$(find /usr/lib /usr/share -iname syslinux.efi -print | grep local ldlinux_e64="$(find /usr/lib /usr/share -iname ldlinux.e64 -print | head -n 1 )" # Pass if required syslinux EFI files cannot be found -([[ -f "$syslinux_efi" ]] && [[ -f "$ldlinux_e64" ]]) || return 0 +# Avoid SC2235: Use { ..; } instead of (..) to avoid subshell overhead +# cf. https://github.com/koalaman/shellcheck/wiki/SC2235 +{ [[ -f "$syslinux_efi" ]] && [[ -f "$ldlinux_e64" ]] ; } || return 0 if is_true "${RAWDISK_BOOT_EXCLUDE_SYSLINUX_EFI:-no}"; then LogPrint "DISABLED: Using syslinux to create an EFI bootloader" From 8a170cd5b48cd2b4e27c95ebca18138547840258 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 9 Mar 2022 13:14:08 +0100 Subject: [PATCH 026/215] Update rear In usr/sbin/rear avoid ShellCheck SC1075: Use 'elif' instead of 'else if' (or put 'if' on new line if nesting) https://github.com/koalaman/shellcheck/wiki/SC1075 and fixed ShellCheck SC2145: Argument mixes string and array. Use * or separate argument https://github.com/koalaman/shellcheck/wiki/SC2145 cf. https://github.com/rear/rear/issues/1040 --- usr/sbin/rear | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/usr/sbin/rear b/usr/sbin/rear index 5ce1d8c1e4..04979257f9 100755 --- a/usr/sbin/rear +++ b/usr/sbin/rear @@ -494,7 +494,7 @@ done if test "$WORKFLOW" != "help" ; then LogPrint "$PRODUCT $VERSION / $RELEASE_DATE" LogPrint "Running $PROGRAM $WORKFLOW (PID $MASTER_PID date $START_DATE_TIME_STRING)" - DebugPrint "Command line options: $0 ${CMD_OPTS[@]}" + DebugPrint "Command line options: $0 ${CMD_OPTS[*]}" LogPrint "Using log file: $RUNTIME_LOGFILE" DebugPrint "Using build area: $BUILD_DIR" fi @@ -567,11 +567,10 @@ if test "$CONFIG_APPEND_FILES" ; then # try if 'foo.conf' exists and if yes, use that: if test -r "$config_append_file_path" ; then COPY_AS_IS+=( "$config_append_file_path" ) - else if test -r "$config_append_file_path.conf" ; then - COPY_AS_IS+=( "$config_append_file_path.conf" ) - else - LogPrintError "There is '-C $config_append_file' but neither '$config_append_file_path' nor '$config_append_file_path.conf' can be read." - fi + elif test -r "$config_append_file_path.conf" ; then + COPY_AS_IS+=( "$config_append_file_path.conf" ) + else + LogPrintError "There is '-C $config_append_file' but neither '$config_append_file_path' nor '$config_append_file_path.conf' can be read." fi ;; (*) @@ -583,12 +582,11 @@ if test "$CONFIG_APPEND_FILES" ; then if test -r "$config_append_file_path" ; then LogPrint "Sourcing additional configuration file '$config_append_file_path'" Source "$config_append_file_path" || Error "Failed to Source $config_append_file_path" - else if test -r "$config_append_file_path.conf" ; then - LogPrint "Sourcing additional configuration file '$config_append_file_path.conf'" - Source "$config_append_file_path.conf" || Error "Failed to Source $config_append_file_path.conf" - else - LogPrintError "There is '-C $config_append_file' but neither '$config_append_file_path' nor '$config_append_file_path.conf' can be read." - fi + elif test -r "$config_append_file_path.conf" ; then + LogPrint "Sourcing additional configuration file '$config_append_file_path.conf'" + Source "$config_append_file_path.conf" || Error "Failed to Source $config_append_file_path.conf" + else + LogPrintError "There is '-C $config_append_file' but neither '$config_append_file_path' nor '$config_append_file_path.conf' can be read." fi done fi From 69e04d8e2f5963451d83f80a7d610c37555c141a Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 10:29:36 +0100 Subject: [PATCH 027/215] Update _input-output-functions.sh In lib/_input-output-functions.sh fixed ShellCheck reported errors SC2145: Argument mixes string and array. Use * or separate argument https://github.com/koalaman/shellcheck/wiki/SC2145 SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 and avoided ShellCheck false error indication SC1087: Use braces when expanding arrays, e.g. ${array[idx]} ... https://github.com/koalaman/shellcheck/wiki/SC1087 for code like string_appended="$string[*]" by appending '[*]' to a string by a separated string+='[*]' command, cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/lib/_input-output-functions.sh | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/usr/share/rear/lib/_input-output-functions.sh b/usr/share/rear/lib/_input-output-functions.sh index b5601c6bd0..d4100bf6fc 100644 --- a/usr/share/rear/lib/_input-output-functions.sh +++ b/usr/share/rear/lib/_input-output-functions.sh @@ -365,7 +365,7 @@ builtin trap "EXIT_FAIL_MESSAGE=0 ; echo '${MESSAGE_PREFIX}Aborting due to an er # Make sure nobody else can use trap: function trap () { - BugError "Forbidden usage of trap with '$@'. Use AddExitTask instead." + BugError "Forbidden usage of trap with '$*'. Use AddExitTask instead." } # For actually intended user messages output to the original STDOUT @@ -502,7 +502,7 @@ function LogToSyslog () { # an alias, builtin, function, or a disk file that would be executed # see https://github.com/rear/rear/issues/729 function has_binary () { - for bin in $@ ; do + for bin in "$@" ; do # Suppress success output via stdout which is crucial when has_binary is called # in other functions that provide their intended function results via stdout # to not pollute intended function results with intermixed has_binary stdout @@ -762,7 +762,7 @@ function BugError () { Error " ==================== BUG in $caller_source: -'$@' +'$*' -------------------- Please report it at $BUG_REPORT_SITE and include all related parts from $RUNTIME_LOGFILE @@ -1020,9 +1020,8 @@ function cleanup_build_area_and_end_program () { # or the user can interupt any automated response within a relatively short time (minimum is only 1 second). function UserInput () { # First and foremost log that UserInput was called (but be confidential here): - # Have caller_source as an array so that plain $caller_source is only the filename (with path): - local caller_source=( $( CallerSource ) ) - Log "UserInput: called in ${caller_source[@]}" + local caller_source="$( CallerSource )" + Log "UserInput: called in $caller_source" # Set defaults or fallback values: # Have a relatively big default timeout of 5 minutes to avoid that the timeout interrupts ongoing user input: local timeout=300 @@ -1156,7 +1155,7 @@ function UserInput () { # The actual work: # In debug mode show the user the script that called UserInput and what user_input_ID was specified # so that the user can prepare an automated response for that UserInput call (without digging in the code): - DebugPrint "UserInput -I $user_input_ID needed in ${caller_source[@]}" + DebugPrint "UserInput -I $user_input_ID needed in $caller_source" # First of all show the prompt unless an empty prompt was specified (via -p '') # so that the prompt can be used as some kind of header line that introduces the user input # and separates the following user input from arbitrary other output lines before: @@ -1246,8 +1245,13 @@ function UserInput () { # Regarding how to get all array elements when the array name is in a variable, see # https://unix.stackexchange.com/questions/60584/how-to-use-a-variable-as-part-of-an-array-name # Assume input_words_array_name="myarr" then input_words_array_name_dereferenced="myarr[*]" - # and "${!input_words_array_name_dereferenced}" becomes "${myarr[*]}": - local input_words_array_name_dereferenced="$input_words_array_name[*]" + # and "${!input_words_array_name_dereferenced}" becomes "${myarr[*]}" + # Avoid ShellCheck false error indication for code like + # string_appended="$string[*]" + # ^-- SC1087: Use braces when expanding arrays, e.g. ${array[idx]} + # by appending '[*]' to a string variable in a separated command: + local input_words_array_name_dereferenced="$input_words_array_name" + input_words_array_name_dereferenced+='[*]' input_string="${!input_words_array_name_dereferenced}" fi # When there is no user input or when the user input is only spaces use the "best" fallback or default that exists. From 63c1d5fd17406b39348147bb5a2c4e0dd28d1b1a Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Thu, 10 Mar 2022 10:33:11 +0100 Subject: [PATCH 028/215] #2763 fix typo in /EFI/BOOT/BOOTX86.efi (-> BOOTX64.efi) Signed-off-by: Gratien D'haese --- usr/share/rear/conf/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 60181cd09b..8f10f80757 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1069,7 +1069,7 @@ AUTOEXCLUDE_USB_PATH=() # Only to be used by experts. An example of a different setup could be: # GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI --hint hd0,msdos1" # or -# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --file /EFI/BOOT/BOOTX86.efi" +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --file /EFI/BOOT/BOOTX64.efi" GRUB2_SEARCH_ROOT_COMMAND="" ## From 5cebac5a13b519d7224b2cf50a4742e4c0e0700c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 11:15:34 +0100 Subject: [PATCH 029/215] Update 400_prep_nbkdc.sh In prep/NBKDC/default/400_prep_nbkdc.s avoid ShellCheck false error indication SC1097: Unexpected ==. For assignment, use = https://github.com/koalaman/shellcheck/wiki/SC1097 for code like: while IFS== read key value by quoting the assigned character, cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 +# TODO: Explain what the CDV variable is +# cf. https://github.com/rear/rear/commit/4c8fd6f6aafbec9aacc94e704a2227f7fc4e3302#r68375270 +# Aviod ShellCheck false error indication +# SC1097: Unexpected ==. For assignment, use = +# for code like +# while IFS== read key value +# by quoting the assigned character: +while CDV='=' read key value ; do --- .../rear/prep/NBKDC/default/400_prep_nbkdc.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/prep/NBKDC/default/400_prep_nbkdc.sh b/usr/share/rear/prep/NBKDC/default/400_prep_nbkdc.sh index 757f029393..096ef08c6c 100644 --- a/usr/share/rear/prep/NBKDC/default/400_prep_nbkdc.sh +++ b/usr/share/rear/prep/NBKDC/default/400_prep_nbkdc.sh @@ -30,7 +30,12 @@ fi CLIENT_INI=$NBKDC_DIR/conf/client.properties if [ -r "$CLIENT_INI" ]; then - while IFS== read key value ; do + # Avoid ShellCheck false error indication + # SC1097: Unexpected ==. For assignment, use = + # for code like + # while IFS== read key value + # by quoting the assigned character: + while IFS='=' read key value ; do case "$key" in hiback_install_dir) NBKDC_HIB_DIR="$value" ;; hiback_version) NBKDC_HIB_VER="$value" ;; @@ -46,7 +51,14 @@ fi COND=$NBKDC_HIB_DIR/CONDEV [[ -r "$COND" ]] || Error "CONDEV file '$COND' can not be read" -while CDV== read key value ; do +# TODO: Explain what the CDV variable is +# cf. https://github.com/rear/rear/commit/4c8fd6f6aafbec9aacc94e704a2227f7fc4e3302#r68375270 +# Avoid ShellCheck false error indication +# SC1097: Unexpected ==. For assignment, use = +# for code like +# while IFS== read key value +# by quoting the assigned character: +while CDV='=' read key value ; do case "$key" in "&listdir:") NBKDC_HIBLST_DIR="$value" ;; "&tmpdir:") NBKDC_HIBTMP_DIR="$value" ;; From eddfadbc60832c0d18e1d055dff8d594f3f051fc Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 11:18:53 +0100 Subject: [PATCH 030/215] Update sesam-functions.sh In lib/sesam-functions.sh avoid ShellCheck false error indication SC1097: Unexpected ==. For assignment, use = https://github.com/koalaman/shellcheck/wiki/SC1097 for code like: while IFS== read key value by quoting the assigned character, cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/lib/sesam-functions.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/lib/sesam-functions.sh b/usr/share/rear/lib/sesam-functions.sh index b6f6c43e2d..c2e4c82077 100644 --- a/usr/share/rear/lib/sesam-functions.sh +++ b/usr/share/rear/lib/sesam-functions.sh @@ -17,7 +17,12 @@ SESAM_LD_LIBRARY_PATH=$SM_BIN_SESAM SM_INI="$( grep SM_INI $sesam2000ini_file 2>/dev/null | cut -d '=' -f 2 )" test -z "$SM_INI" && return 0 -while IFS== read key value ; do +# Avoid ShellCheck false error indication +# SC1097: Unexpected ==. For assignment, use = +# for code like +# while IFS== read key value +# by quoting the assigned character: +while IFS='=' read key value ; do case "$key" in (gv_ro) SESAM_BIN_DIR="$value" ;; (gv_rw) SESAM_VAR_DIR="$value" ;; From 7f879c42c5b7f1d67dd346032c9c2b094cbca541 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 11:28:05 +0100 Subject: [PATCH 031/215] Update 300_create_extlinux.sh In output/USB/Linux-i386/300_create_extlinux.sh avoid ShellCheck reported error SC2199: Arrays implicitly concatenate in [[ ]]. Use a loop (or explicit * instead of @) https://github.com/koalaman/shellcheck/wiki/SC2199 by using "$*" instead of "$@", cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh b/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh index 88d60b7f75..21ccefce80 100644 --- a/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh +++ b/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh @@ -57,7 +57,7 @@ function syslinux_has { # FIXME: Syslinux older than 3.62 do have menu.c32 but not submenu support # We simplify by disabling MENU support for everything older than 3.62 function syslinux_write { - if [[ "$@" ]]; then + if [[ "$*" ]]; then echo "$@" | syslinux_write elif [[ "$FEATURE_SYSLINUX_SUBMENU" ]]; then cat >&4 From c9362dd27ac968a095fa930d12e40142c95cdfdd Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 11:50:45 +0100 Subject: [PATCH 032/215] Update 310_network_devices.sh In rescue/GNU/Linux/310_network_devices.sh avoid ShellCheck reported error SC2199: Arrays implicitly concatenate in [[ ]]. Use a loop (or explicit * instead of @) https://github.com/koalaman/shellcheck/wiki/SC2199 by simpler code, cf, https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/rescue/GNU/Linux/310_network_devices.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh index 2385f5b6dc..7bc8d1e98c 100644 --- a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh +++ b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh @@ -552,9 +552,7 @@ function handle_bridge () { if is_true $ip_link_supports_bridge ; then echo "ip link add name $network_interface type bridge stp_state $stp" elif has_binary brctl ; then - if [[ " ${REQUIRED_PROGS[@]} " != *\ brctl\ * ]] ; then - REQUIRED_PROGS+=( "brctl" ) - fi + IsInArray "brctl" "${REQUIRED_PROGS[@]}" || REQUIRED_PROGS+=( "brctl" ) echo "brctl addbr $network_interface" echo "brctl stp $network_interface $stp" else From 78e7ea0d7b3116deaa10bb21bbab7d2f5930049b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 11:57:55 +0100 Subject: [PATCH 033/215] Update 260_copy_log_and_report.sh In restore/FDRUPSTREAM/default/260_copy_log_and_report.sh fixed ShellCheck reported error SC2199: Arrays implicitly concatenate in [[ ]]. Use a loop (or explicit * instead of @) https://github.com/koalaman/shellcheck/wiki/SC2199 and fixed possible "bash: conditional binary operator expected" error when ${SERVICENAME[@]} evaluates to more than one word in [[...]] by using "${SERVICENAME[*]}" instead of unquoted ${SERVICENAME[@]} cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/restore/FDRUPSTREAM/default/260_copy_log_and_report.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/restore/FDRUPSTREAM/default/260_copy_log_and_report.sh b/usr/share/rear/restore/FDRUPSTREAM/default/260_copy_log_and_report.sh index 3a933a259c..b1a884b66d 100644 --- a/usr/share/rear/restore/FDRUPSTREAM/default/260_copy_log_and_report.sh +++ b/usr/share/rear/restore/FDRUPSTREAM/default/260_copy_log_and_report.sh @@ -8,7 +8,7 @@ REARLOGPATH="$FDRUPSTREAM_DATA_PATH/rear/logs" # If SERVICENAME is empty, then FDR/Upstream is not running and we won't # be able to find the log files. -if [[ ! ${SERVICENAME[@]} ]]; then +if [[ ! "${SERVICENAME[*]}" ]]; then echo LogPrintError "***************" LogPrintError "***************" From c50eefeb0d1defa2a0c01466ff89aa0225fcded5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 10 Mar 2022 12:42:21 +0100 Subject: [PATCH 034/215] Update 950_email_result_files.sh In output/default/950_email_result_files.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 and used ${array[*]} instead of ${array[@]} in message stings to avoid SC2145: Argument mixes string and array. Use * or separate argument https://github.com/koalaman/shellcheck/wiki/SC2145 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 and replaced ...IfError function calls by calling Error, see "Using the ...IfError functions can result unexpected behaviour ..." in lib/_input-output-functions.s --- .../output/default/950_email_result_files.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/usr/share/rear/output/default/950_email_result_files.sh b/usr/share/rear/output/default/950_email_result_files.sh index f481dd91ef..f739d7e727 100644 --- a/usr/share/rear/output/default/950_email_result_files.sh +++ b/usr/share/rear/output/default/950_email_result_files.sh @@ -4,13 +4,12 @@ test -z "$RESULT_MAILTO" && return -[ ${#RESULT_FILES[@]} -gt 0 ] -StopIfError "No files to send (RESULT_FILES is empty)" +[ ${#RESULT_FILES[@]} -gt 0 ] || Error "No files to send (RESULT_FILES is empty)" -[ -x "$RESULT_SENDMAIL" ] -StopIfError "No mailer [$RESULT_SENDMAIL] found !" +[ -x "$RESULT_SENDMAIL" ] || Error "No mailer [$RESULT_SENDMAIL] found !" + +Log "Sending e-mail from $RESULT_MAILFROM to ${RESULT_MAILTO[*]}" -Log "Sending Email from $RESULT_MAILFROM to ${RESULT_MAILTO[@]}" # We will remove the ISO files from the RESULT_FILES array (is becoming too big - issue #397) c=${#RESULT_FILES[@]} # amount of element is array RESULT_FILES i=0 @@ -19,7 +18,7 @@ while (( $i < $c )) ; do i=$(( i + 1 )) done -Log "Attaching files: ${MAIL_FILES[@]}" +Log "Attaching files: ${MAIL_FILES[*]}" test -z "$RESULT_MAILSUBJECT" && RESULT_MAILSUBJECT="Relax-and-Recover $HOSTNAME ($OUTPUT)" @@ -41,6 +40,7 @@ test -z "$RESULT_MAILSUBJECT" && RESULT_MAILSUBJECT="Relax-and-Recover $HOSTNAME MAIL_SIZE=( $(du -h $TMP_DIR/email.bin) ) -LogPrint "Mailing resulting files ($MAIL_SIZE) to ${RESULT_MAILTO[@]}" -$RESULT_SENDMAIL "${RESULT_SENDMAIL_OPTIONS[@]}" <$TMP_DIR/email.bin -LogPrintIfError "WARNING ! Sending Email with '$RESULT_SENDMAIL "${RESULT_SENDMAIL_OPTIONS[@]}"' failed." +LogPrint "Mailing resulting files ($MAIL_SIZE) to ${RESULT_MAILTO[*]}" +if ! $RESULT_SENDMAIL "${RESULT_SENDMAIL_OPTIONS[@]}" <$TMP_DIR/email.bin ; then + LogPrintError "WARNING: Sending e-mail with '$RESULT_SENDMAIL ${RESULT_SENDMAIL_OPTIONS[*]}' failed" +fi From f8e75766d88ba9778a8eea23edb189096c3a9ad4 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 12:25:42 +0100 Subject: [PATCH 035/215] Update 940_grub2_rescue.sh In output/default/940_grub2_rescue.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/output/default/940_grub2_rescue.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/output/default/940_grub2_rescue.sh b/usr/share/rear/output/default/940_grub2_rescue.sh index 41c495973f..f0650af547 100644 --- a/usr/share/rear/output/default/940_grub2_rescue.sh +++ b/usr/share/rear/output/default/940_grub2_rescue.sh @@ -53,7 +53,7 @@ local grub_config_dir="$boot_dir/grub${grub_num}" # Esure there is sufficient disk space available in /boot for the local Relax-and-Recover rescue system: function total_filesize { - stat --format '%s' $@ 2>/dev/null | awk 'BEGIN { t=0 } { t+=$1 } END { print t }' + stat --format '%s' "$@" 2>/dev/null | awk 'BEGIN { t=0 } { t+=$1 } END { print t }' } # Free space in /boot: local free_space=$( df -Pkl $boot_dir | awk 'END { print $4 * 1024 }' ) From 578338b77105d57d9cab258a9aa41286cde142fb Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 12:26:32 +0100 Subject: [PATCH 036/215] Update 940_grub_rescue.sh In output/default/940_grub_rescue.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/output/default/940_grub_rescue.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/output/default/940_grub_rescue.sh b/usr/share/rear/output/default/940_grub_rescue.sh index 5ec142df49..8f9504282d 100644 --- a/usr/share/rear/output/default/940_grub_rescue.sh +++ b/usr/share/rear/output/default/940_grub_rescue.sh @@ -33,7 +33,7 @@ test -r "$KERNEL_FILE" || Error "Failed to find kernel '$KERNEL_FILE', updating test -r "$TMP_DIR/$REAR_INITRD_FILENAME" || Error "Failed to find initrd '$REAR_INITRD_FILENAME', updating GRUB failed." function total_filesize { - stat --format '%s' $@ 2>/dev/null | awk 'BEGIN { t=0 } { t+=$1 } END { print t }' + stat --format '%s' "$@" 2>/dev/null | awk 'BEGIN { t=0 } { t+=$1 } END { print t }' } available_space=$(df -Pkl /boot | awk 'END { print $4 * 1024 }') From 203fba69c583d9e9267d26bbd82c329d16f12cbf Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 12:57:47 +0100 Subject: [PATCH 037/215] Update 960_dsmc_verify_isofile.sh In output/TSM/default/960_dsmc_verify_isofile.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 and avoid SC2145: Argument mixes string and array. Use * or separate argument https://github.com/koalaman/shellcheck/wiki/SC2145 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 plus better error checking and messaging and other code cleanup --- .../TSM/default/960_dsmc_verify_isofile.sh | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/usr/share/rear/output/TSM/default/960_dsmc_verify_isofile.sh b/usr/share/rear/output/TSM/default/960_dsmc_verify_isofile.sh index 8357aa5239..436a0a9f08 100644 --- a/usr/share/rear/output/TSM/default/960_dsmc_verify_isofile.sh +++ b/usr/share/rear/output/TSM/default/960_dsmc_verify_isofile.sh @@ -1,21 +1,23 @@ # 960_dsmc_verify_isofile.sh -if [[ ! "$TSM_RM_ISOFILE" =~ [yY1] ]] ; then - return -fi -Log "Verify if the files '$TSM_RESULT_FILE_PATH/$ISO_PREFIX.iso' were archived correctly with dsmc" +is_true $TSM_RM_ISOFILE || return 0 + +Log "Verify if the ISO file '$TSM_RESULT_FILE_PATH/$ISO_PREFIX.iso' was archived correctly with dsmc" if [[ -z "$TSM_ARCHIVE_MGMT_CLASS" ]]; then LC_ALL=${LANG_RECOVER} dsmc q backup "$TSM_RESULT_FILE_PATH/$ISO_PREFIX.iso" >/dev/null else LC_ALL=${LANG_RECOVER} dsmc q archive "$TSM_RESULT_FILE_PATH/$ISO_PREFIX.iso" >/dev/null fi if [[ $? -eq 0 ]]; then - Log "Removing the '${TSM_RESULT_FILES[@]}' files to preserve space" - rm $v -f ${TSM_RESULT_FILES[@]} >/dev/null - Log "Remove the $ISO_DIR/$ISO_PREFIX.iso to preserve space" - rm $v -f $ISO_DIR/$ISO_PREFIX.iso >/dev/null - LogPrint "The only remaining copy of the ISO file is under TSM:$TSM_RESULT_FILE_PATH/$ISO_PREFIX.iso" + Log "Removing $ISO_DIR/$ISO_PREFIX.iso to preserve space" + if rm $v -f $ISO_DIR/$ISO_PREFIX.iso ; then + LogPrint "The only remaining copy of the ISO file is under TSM:$TSM_RESULT_FILE_PATH/$ISO_PREFIX.iso" + Log "To preserve space also removing the TSM_RESULT_FILES ${TSM_RESULT_FILES[*]}" + rm $v -f "${TSM_RESULT_FILES[@]}" + else + Log "Could not remove $ISO_DIR/$ISO_PREFIX.iso so the local files are kept" + fi else - LogPrint "TSM did not confirm correctly if the ISO file was stored properly - not remove local ISO files" + LogPrint "TSM did not confirm that the ISO file was stored properly so the local files are kept" fi From 9a0580480f5710f950e85189e94f1a36276c1eda Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:02:01 +0100 Subject: [PATCH 038/215] Update 800_create_isofs.sh In output/ISO/Linux-i386/800_create_isofs.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 and avoid SC2145: Argument mixes string and array. Use * or separate argument https://github.com/koalaman/shellcheck/wiki/SC2145 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/output/ISO/Linux-i386/800_create_isofs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/output/ISO/Linux-i386/800_create_isofs.sh b/usr/share/rear/output/ISO/Linux-i386/800_create_isofs.sh index b004686bcb..ca017e3d6a 100644 --- a/usr/share/rear/output/ISO/Linux-i386/800_create_isofs.sh +++ b/usr/share/rear/output/ISO/Linux-i386/800_create_isofs.sh @@ -21,7 +21,7 @@ fi # in case the user populates this array manually we must not forget to copy # these files to our temporary isofs if test "${#ISO_FILES[@]}" -gt 0 ; then - cp -pL $v ${ISO_FILES[@]} $TMP_DIR/isofs/isolinux/ || Error "Failed to copy ISO_FILES '${ISO_FILES[@]}'" + cp -pL $v "${ISO_FILES[@]}" $TMP_DIR/isofs/isolinux/ || Error "Failed to copy ISO_FILES ${ISO_FILES[*]}" fi mkdir -p $v "$ISO_DIR" || Error "Failed to create ISO_DIR '$ISO_DIR'" From 3148b9f065eb6d97db88bdf6be3dd10ae633bce2 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:07:21 +0100 Subject: [PATCH 039/215] Update 120_confirm_wipedisk_disks.sh In layout/recreate/default/120_confirm_wipedisk_disks.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/layout/recreate/default/120_confirm_wipedisk_disks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh b/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh index 641054e1a7..c400b79382 100644 --- a/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh +++ b/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh @@ -85,7 +85,7 @@ else esac done local component_device parent_device added_parent_device="no" - for component_device in ${component_devices[@]} ; do + for component_device in "${component_devices[@]}" ; do # component_device is a disk like /dev/sdc or a partition like /dev/sdc1 (cf. above) # so we get the parent device of it (the parent of a disk will be the disk itself) # cf. the code of the function write_protection_ids() in lib/write-protect-functions.sh From 25fbb44653833fe4ef5e581d5d3e5d35556bea92 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:16:15 +0100 Subject: [PATCH 040/215] Update 205_s390_enable_disk.sh In layout/prepare/Linux-s390/205_s390_enable_disk.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/layout/prepare/Linux-s390/205_s390_enable_disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/prepare/Linux-s390/205_s390_enable_disk.sh b/usr/share/rear/layout/prepare/Linux-s390/205_s390_enable_disk.sh index c4037e0245..0ff2fd5867 100644 --- a/usr/share/rear/layout/prepare/Linux-s390/205_s390_enable_disk.sh +++ b/usr/share/rear/layout/prepare/Linux-s390/205_s390_enable_disk.sh @@ -21,7 +21,7 @@ format_s390_disk() { # LDL formatted disks are already partitioned and should not be partitioned with parted or fdasd , it will fail # this var, listDasdLdl, is used by 100_include_partition_code.sh to exclude writing partition code to diskrestore.sh for LDL disks listDasdLdl+=( $device ) - LogPrint "LDL disk added to listDasdLdl:" ${listDasdLdl[@]} + LogPrint "LDL disk '$device' will not be partitioned (LDL disks are already partitioned)" fi LogPrint 'dasdfmt:' $device ', blocksize:' $blocksize ', layout:' $layout # dasd format From fa3ea439465c4a64d030f247277c5f4a7daf56a0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:21:07 +0100 Subject: [PATCH 041/215] Update 310_network_devices.sh In rescue/GNU/Linux/310_network_devices.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/rescue/GNU/Linux/310_network_devices.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh index 7bc8d1e98c..afc08193b9 100644 --- a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh +++ b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh @@ -124,11 +124,11 @@ EOT # e.g. RHEL6 doesn't support that if readlink /foo /bar 2>/dev/null ; then function resolve () { - readlink -e $@ + readlink -e "$@" } else function resolve () { - for path in $@ ; do + for path in "$@" ; do readlink -e $path done } From ebc988f42736ca055bfedca4f73d1a2b666a4e88 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:34:46 +0100 Subject: [PATCH 042/215] Update 230_storage_and_network_modules.sh In rescue/GNU/Linux/230_storage_and_network_modules.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/rescue/GNU/Linux/230_storage_and_network_modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/rescue/GNU/Linux/230_storage_and_network_modules.sh b/usr/share/rear/rescue/GNU/Linux/230_storage_and_network_modules.sh index 2fa6d6fe9c..b00539a30b 100644 --- a/usr/share/rear/rescue/GNU/Linux/230_storage_and_network_modules.sh +++ b/usr/share/rear/rescue/GNU/Linux/230_storage_and_network_modules.sh @@ -9,7 +9,7 @@ function find_modules_in_dirs () { # and the # ... | sed -e 's/^\(.*\)\.ko.*/\1/' # removes the trailing .ko faster via one sed call than many basename calls or shell code: - find $@ -type f -name '*.ko*' -printf '%f\n' 2>/dev/null | sed -e 's/^\(.*\)\.ko.*/\1/' + find "$@" -type f -name '*.ko*' -printf '%f\n' 2>/dev/null | sed -e 's/^\(.*\)\.ko.*/\1/' } # Include storage drivers From 553995e89ddf4acdaef7c97641b96586ad09b786 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:36:45 +0100 Subject: [PATCH 043/215] Update 990_move_away_restored_files.sh In restore/default/990_move_away_restored_files.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/restore/default/990_move_away_restored_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/restore/default/990_move_away_restored_files.sh b/usr/share/rear/restore/default/990_move_away_restored_files.sh index 44ecee00d5..5b7a013fda 100644 --- a/usr/share/rear/restore/default/990_move_away_restored_files.sh +++ b/usr/share/rear/restore/default/990_move_away_restored_files.sh @@ -46,7 +46,7 @@ for dummy in "once" ; do # because the files therein could contain security relevant information: mkdir -p -m 0700 $move_away_dir || continue # Copy each file or directory in BACKUP_RESTORE_MOVE_AWAY_FILES with full path: - for file in ${BACKUP_RESTORE_MOVE_AWAY_FILES[@]} ; do + for file in "${BACKUP_RESTORE_MOVE_AWAY_FILES[@]}" ; do # Strip leading '/' from $file to get it with relative path that is needed inside the recovery system: file_relative="${file#/}" # Skip files or directories listed in BACKUP_RESTORE_MOVE_AWAY_FILES that do not actually exist: From 1eb7785766504e7c9d20728803b0826558107d31 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:38:29 +0100 Subject: [PATCH 044/215] Update 300_create_nbu_restore_fs_list.sh In restore/NBU/default/300_create_nbu_restore_fs_list.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/restore/NBU/default/300_create_nbu_restore_fs_list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/restore/NBU/default/300_create_nbu_restore_fs_list.sh b/usr/share/rear/restore/NBU/default/300_create_nbu_restore_fs_list.sh index a7e792c3ec..cf4b4f2503 100644 --- a/usr/share/rear/restore/NBU/default/300_create_nbu_restore_fs_list.sh +++ b/usr/share/rear/restore/NBU/default/300_create_nbu_restore_fs_list.sh @@ -25,7 +25,7 @@ then fi if [ ${#EXCLUDE_MOUNTPOINTS[@]} -gt 0 ] then - for FS in ${EXCLUDE_MOUNTPOINTS[@]} + for FS in "${EXCLUDE_MOUNTPOINTS[@]}" do echo "${FS}/" >> $TMP_DIR/restore_fs_list echo "!${FS}/*" >> $TMP_DIR/restore_fs_list From fd19a450c99fc739b0bd76dee349f9a5b764d0e2 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:40:58 +0100 Subject: [PATCH 045/215] Update 00-functions.sh In skel/default/etc/scripts/system-setup.d/00-functions.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../skel/default/etc/scripts/system-setup.d/00-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh b/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh index 5bd5f2429c..969b3d1557 100644 --- a/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh +++ b/usr/share/rear/skel/default/etc/scripts/system-setup.d/00-functions.sh @@ -1,7 +1,7 @@ # helper functions # call udevtrigger my_udevtrigger() { - type -p udevadm >/dev/null && udevadm trigger $@ || udevtrigger $@ + type -p udevadm >/dev/null && udevadm trigger "$@" || udevtrigger "$@" # If systemd is running, this should help to rename devices if [[ $(ps --no-headers -C systemd) ]]; then @@ -19,7 +19,7 @@ my_udevtrigger() { # SC2120: my_udevsettle references arguments, but none are ever passed. # shellcheck disable=SC2120 my_udevsettle() { - type -p udevadm >/dev/null && udevadm settle --timeout=10 $@ || udevsettle $@ + type -p udevadm >/dev/null && udevadm settle --timeout=10 "$@" || udevsettle "$@" } # call udevinfo From 4047b23215fa246c51b151171c9e5a92133b53d5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:54:16 +0100 Subject: [PATCH 046/215] Update dhcp-setup-functions.sh In skel/default/etc/scripts/dhcp-setup-functions.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/skel/default/etc/scripts/dhcp-setup-functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/skel/default/etc/scripts/dhcp-setup-functions.sh b/usr/share/rear/skel/default/etc/scripts/dhcp-setup-functions.sh index 49ac81c38b..93fe3f0294 100644 --- a/usr/share/rear/skel/default/etc/scripts/dhcp-setup-functions.sh +++ b/usr/share/rear/skel/default/etc/scripts/dhcp-setup-functions.sh @@ -206,7 +206,7 @@ my_ipcalc() { if [[ "$2" ]]; then declare -i DEC MASK="$(ip2num "$2")" - for BITS in ${!NETMASKS[@]}; do + for BITS in "${!NETMASKS[@]}" ; do DEC=${NETMASKS[$BITS]} (( MASK == DEC )) && break done @@ -420,7 +420,7 @@ dhconfig() { gateway=${static_routes[$i+1]} metric='' - for t in ${route_targets[@]}; do + for t in "${route_targets[@]}" ; do if [ ${t} = ${target} ]; then if [ -z "${metric}" ]; then metric=1 @@ -460,7 +460,7 @@ dhconfig() { for router in ${new_routers} ; do added_router=- - for r in ${default_routers[@]} ; do + for r in "${default_routers[@]}" ; do if [ "${r}" = "${router}" ]; then added_router=1 fi From a1f093cd9451d4863e760e88923ac8acee1c6369 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:55:49 +0100 Subject: [PATCH 047/215] Update 990_copy_logfile.sh In wrapup/default/990_copy_logfile.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/wrapup/default/990_copy_logfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/wrapup/default/990_copy_logfile.sh b/usr/share/rear/wrapup/default/990_copy_logfile.sh index dea24155ed..500e42f263 100644 --- a/usr/share/rear/wrapup/default/990_copy_logfile.sh +++ b/usr/share/rear/wrapup/default/990_copy_logfile.sh @@ -8,7 +8,7 @@ # cf. https://github.com/rear/rear/issues/987 # and https://github.com/rear/rear/issues/1088 recovery_workflows=( "recover" "layoutonly" "restoreonly" ) -IsInArray $WORKFLOW ${recovery_workflows[@]} || return 0 +IsInArray $WORKFLOW "${recovery_workflows[@]}" || return 0 # Copy the logfile: # Usually RUNTIME_LOGFILE=/var/log/rear/rear-$HOSTNAME.log From 510af53b84f9e16e03bef8f3bbc535fdcbbaeaae Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 13:59:51 +0100 Subject: [PATCH 048/215] Update output-functions.sh In usr/share/rear/lib/output-functions.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/lib/output-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/output-functions.sh b/usr/share/rear/lib/output-functions.sh index c2648c4af9..923fb065a2 100644 --- a/usr/share/rear/lib/output-functions.sh +++ b/usr/share/rear/lib/output-functions.sh @@ -42,7 +42,7 @@ function assert_ISO_FILE_SIZE_LIMIT () { # Skip when there is no usable ISO_FILE_SIZE_LIMIT set (in particular for ISO_FILE_SIZE_LIMIT=0): is_positive_integer $ISO_FILE_SIZE_LIMIT || return 0 local file_for_iso file_for_iso_size - for file_for_iso in $@ ; do + for file_for_iso in "$@" ; do file_for_iso_size=$( stat -L -c '%s' $file_for_iso ) # Continue "bona fide" with testing the next one if size could not be determined (assume the current one is OK): is_positive_integer $file_for_iso_size || continue From 042c330d0ce3bd7237b12bbfd69cab468c83b9ef Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 14:14:31 +0100 Subject: [PATCH 049/215] Update layout-functions.sh In lib/layout-functions.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/lib/layout-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/lib/layout-functions.sh b/usr/share/rear/lib/layout-functions.sh index ffaef148df..0c633663e0 100644 --- a/usr/share/rear/lib/layout-functions.sh +++ b/usr/share/rear/lib/layout-functions.sh @@ -563,7 +563,7 @@ version_newer() { # Function to get version from tool. get_version() { - TERM=dumb $@ 2>&1 | sed -rn 's/^[^0-9\.]*([0-9]+\.[-0-9a-z\.]+).*$/\1/p' | head -1 + TERM=dumb "$@" 2>&1 | sed -rn 's/^[^0-9\.]*([0-9]+\.[-0-9a-z\.]+).*$/\1/p' | head -1 } # Translate a device name to a sysfs name. @@ -1380,7 +1380,7 @@ delete_dummy_partitions_and_resize_real_ones() { # Delete dummy partitions local -i num - for num in ${dummy_partitions_to_delete[@]} ; do + for num in "${dummy_partitions_to_delete[@]}" ; do LogPrint "Disk '$current_disk': deleting dummy partition number $num" parted -s -m $current_disk rm $num done From 5059fcfb60b3efdb443c8352df8e0eefac0cbe50 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 14:25:29 +0100 Subject: [PATCH 050/215] Update 940_grub_rescue.sh In output/default/940_grub_rescue.sh removed a misleading comment that shows a wrong get_version function call get_version "grub --version" where "grub --version" is one argumet instead of correctly separated arguments get_version grub --version that is needed since the ShellCheck SC2068 fixed get_version function via https://github.com/rear/rear/commit/042c330d0ce3bd7237b12bbfd69cab468c83b9ef --- usr/share/rear/output/default/940_grub_rescue.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/output/default/940_grub_rescue.sh b/usr/share/rear/output/default/940_grub_rescue.sh index 8f9504282d..0d24caadcd 100644 --- a/usr/share/rear/output/default/940_grub_rescue.sh +++ b/usr/share/rear/output/default/940_grub_rescue.sh @@ -20,8 +20,8 @@ if [[ -z "$grub_binary" ]]; then return fi -### Use strings as grub --version syncs all disks -#grub_version=$(get_version "grub --version") +# Use strings because "grub --version" would sync all disks +# cf. the get_version function in lib/layout-functions.sh grub_version=$(strings $grub_binary | sed -rn 's/^[^0-9\.]*([0-9]+\.[-0-9a-z\.]+).*$/\1/p' | tail -n 1) if version_newer "$grub_version" 1.0; then # only for grub-legacy we make special ReaR boot entry in menu.lst From 0ed0915f1d8d97890d24ba4802c479c97eb2d422 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 14:28:37 +0100 Subject: [PATCH 051/215] Update 850_make_USB_bootable.sh In output/USB/Linux-i386/850_make_USB_bootable.sh fixed a wrong get_version function call get_version "dd --version" where "dd --version" is one argumet instead of correctly separated arguments get_version dd --version that is needed since the ShellCheck SC2068 fixed get_version function via https://github.com/rear/rear/commit/042c330d0ce3bd7237b12bbfd69cab468c83b9ef --- usr/share/rear/output/USB/Linux-i386/850_make_USB_bootable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/output/USB/Linux-i386/850_make_USB_bootable.sh b/usr/share/rear/output/USB/Linux-i386/850_make_USB_bootable.sh index afe099ac13..f243b14a72 100644 --- a/usr/share/rear/output/USB/Linux-i386/850_make_USB_bootable.sh +++ b/usr/share/rear/output/USB/Linux-i386/850_make_USB_bootable.sh @@ -5,7 +5,7 @@ test "$USB_BOOTLOADER" = "grub" && return # Test for features in dd # true if dd supports oflag= option FEATURE_DD_OFLAG= -dd_version=$( get_version "dd --version" ) +dd_version=$( get_version dd --version ) version_newer "$dd_version" 5.3.0 && FEATURE_DD_OFLAG="y" # We assume REAL_USB_DEVICE and RAW_USB_DEVICE are both set by prep/USB/Linux-i386/350_check_usb_disk.sh From 8e419ac1b03caf4fba2381fcbbb03a9c7c50a8b0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 14:35:06 +0100 Subject: [PATCH 052/215] Update filesystems-functions.sh In lib/filesystems-functions.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/lib/filesystems-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/filesystems-functions.sh b/usr/share/rear/lib/filesystems-functions.sh index 985a9f26c1..8aca934c1f 100644 --- a/usr/share/rear/lib/filesystems-functions.sh +++ b/usr/share/rear/lib/filesystems-functions.sh @@ -165,7 +165,7 @@ function xfs_parse | grep -oE "crc=[0-9]*" | cut -d "=" -f2) # Count how many parameter we have - for i in ${xfs_param_iname[@]}; do + for i in "${xfs_param_iname[@]}" ; do xfs_param_count=$((xfs_param_count+1)) done From c673df65cddb599fe457ab918cfd3aeaae643d05 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 14:42:23 +0100 Subject: [PATCH 053/215] Update linux-functions.sh In lib/linux-functions.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- usr/share/rear/lib/linux-functions.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usr/share/rear/lib/linux-functions.sh b/usr/share/rear/lib/linux-functions.sh index 41bfc6defc..937d9199ea 100644 --- a/usr/share/rear/lib/linux-functions.sh +++ b/usr/share/rear/lib/linux-functions.sh @@ -27,9 +27,9 @@ function have_udev () { # see https://github.com/rear/rear/issues/791 function my_udevtrigger () { # first try the most current way, newer systems (e.g. SLES11) have 'udevadm trigger' - has_binary udevadm && udevadm trigger $@ && return 0 + has_binary udevadm && udevadm trigger "$@" && return 0 # then try an older way, older systems (e.g. SLES10) have 'udevtrigger' - has_binary udevtrigger && udevtrigger $@ && return 0 + has_binary udevtrigger && udevtrigger "$@" && return 0 # as first fallback do what start_udev does on RHEL 4 if has_binary udevstart ; then local udevd_pid=$( pidof -x udevd ) @@ -45,9 +45,9 @@ function my_udevtrigger () { # see https://github.com/rear/rear/issues/791 function my_udevsettle () { # first try the most current way, newer systems (e.g. SLES11) have 'udevadm settle' - has_binary udevadm && udevadm settle $@ && return 0 + has_binary udevadm && udevadm settle "$@" && return 0 # then try an older way, older systems (e.g. SLES10) have 'udevsettle' - has_binary udevsettle && udevsettle $@ && return 0 + has_binary udevsettle && udevsettle "$@" && return 0 # as first fallback re-implement udevsettle for older systems if [ -e /sys/kernel/uevent_seqnum ] && [ -e /dev/.udev/uevent_seqnum ] ; then local tries=0 @@ -177,7 +177,7 @@ function RequiredSharedObjects () { local file_for_ldd="" local file_owner_name="" # It is crucial to append to /dev/$DISPENSABLE_OUTPUT_DEV (cf. 'Print' in lib/_input-output-functions.sh): - for file_for_ldd in $@ ; do + for file_for_ldd in "$@" ; do # Skip non-regular files like directories, device files, and non-existent files # cf. similar code in build/GNU/Linux/100_copy_as_is.sh # but here symbolic links must not be skipped (e.g. /sbin/mkfs.ext2 -> /usr/sbin/mkfs.ext2) From c93a4d55c730ec428fff01525b566b9724e6af59 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 11 Mar 2022 15:02:25 +0100 Subject: [PATCH 054/215] Update 550_rebuild_initramfs.sh In finalize/Fedora/i386/550_rebuild_initramfs.sh avoid ShellCheck SC2068: Double quote array expansions to avoid re-splitting elements https://github.com/koalaman/shellcheck/wiki/SC2068 and avoid SC2145: Argument mixes string and array. Use * or separate argument https://github.com/koalaman/shellcheck/wiki/SC2145 cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 --- .../rear/finalize/Fedora/i386/550_rebuild_initramfs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh index 3476b77f19..11c1a460b2 100644 --- a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh +++ b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh @@ -47,7 +47,7 @@ if [ -f $VAR_DIR/recovery/initrd_modules ]; then OLD_INITRD_MODULES=() fi -Log "Original OLD_INITRD_MODULES='${OLD_INITRD_MODULES[@]}'" +Log "Original OLD_INITRD_MODULES are ${OLD_INITRD_MODULES[*]}" # To see what has been added by the migration process, the new modules are added to the # end of the list. To achieve this, we list the old modules twice in the variable # NEW_INITRD_MODULES and then add the new modules. Then we use "uniq -u" to filter out @@ -58,10 +58,10 @@ NEW_INITRD_MODULES=( ${OLD_INITRD_MODULES[@]} ${OLD_INITRD_MODULES[@]} $( cat $T # uniq INITRD_MODULES NEW_INITRD_MODULES=( $(tr " " "\n" <<< "${NEW_INITRD_MODULES[*]}" | sort | uniq -u) ) -Log "New INITRD_MODULES='${OLD_INITRD_MODULES[@]} ${NEW_INITRD_MODULES[@]}'" +Log "New INITRD_MODULES are ${OLD_INITRD_MODULES[*]} ${NEW_INITRD_MODULES[*]}" INITRD_MODULES="${OLD_INITRD_MODULES[@]} ${NEW_INITRD_MODULES[@]}" -WITH_INITRD_MODULES=$( printf '%s\n' ${INITRD_MODULES[@]} | awk '{printf "--with=%s ", $1}' ) +WITH_INITRD_MODULES=$( printf '%s\n' ${INITRD_MODULES[*]} | awk '{printf "--with=%s ", $1}' ) # Recreate any initrd or initramfs image under $TARGET_FS_ROOT/boot/ with new drivers # Images ignored: @@ -88,7 +88,7 @@ for INITRD_IMG in $( ls $TARGET_FS_ROOT/boot/initramfs-*.img $TARGET_FS_ROOT/boo # and then "rear recover" cannot be aborted with the usual [Ctrl]+[C] keys. # Use plain $var because when var contains only blanks test "$var" results true because test " " results true: if test $mkinitrd_binary ; then - if chroot $TARGET_FS_ROOT $mkinitrd_binary -v -f ${WITH_INITRD_MODULES[@]} $INITRD $kernel_version >&2 ; then + if chroot $TARGET_FS_ROOT $mkinitrd_binary -v -f ${WITH_INITRD_MODULES[*]} $INITRD $kernel_version ; then LogPrint "Updated initrd with new drivers for kernel $kernel_version." else LogPrint "WARNING: From 2fbb8ed8230e4af972d3a5933c6505c852950866 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 15 Mar 2022 09:37:28 +0100 Subject: [PATCH 055/215] Update 550_rebuild_initramfs.sh Handle INITRD_MODULES and WITH_INITRD_MODULES as strings, cf. https://github.com/rear/rear/pull/2771/files#r826060892 and https://github.com/rear/rear/pull/2771/files#r826063485 --- .../Fedora/i386/550_rebuild_initramfs.sh | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh index 11c1a460b2..1287dd2abf 100644 --- a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh +++ b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh @@ -47,29 +47,29 @@ if [ -f $VAR_DIR/recovery/initrd_modules ]; then OLD_INITRD_MODULES=() fi -Log "Original OLD_INITRD_MODULES are ${OLD_INITRD_MODULES[*]}" -# To see what has been added by the migration process, the new modules are added to the -# end of the list. To achieve this, we list the old modules twice in the variable -# NEW_INITRD_MODULES and then add the new modules. Then we use "uniq -u" to filter out -# the modules which only appear once in the list. The resulting array -# contains the new modules also. +Log "Original OLD_INITRD_MODULES=( ${OLD_INITRD_MODULES[*]} )" +# To see what has been added by the migration process, the new modules are added to the end of the list. +# To achieve this, we list the old modules twice in the variable NEW_INITRD_MODULES and then add the new modules. +# Then we use "uniq -u" to filter out the modules which only appear once in the list. +# The resulting array contains only the new modules: NEW_INITRD_MODULES=( ${OLD_INITRD_MODULES[@]} ${OLD_INITRD_MODULES[@]} $( cat $TMP_DIR/storage_drivers ) ) +NEW_INITRD_MODULES=( $( tr " " "\n" <<< "${NEW_INITRD_MODULES[*]}" | sort | uniq -u ) ) -# uniq INITRD_MODULES -NEW_INITRD_MODULES=( $(tr " " "\n" <<< "${NEW_INITRD_MODULES[*]}" | sort | uniq -u) ) +# Concatenate the old and new modules into a string: +INITRD_MODULES="${OLD_INITRD_MODULES[*]} ${NEW_INITRD_MODULES[*]}" +Log "New INITRD_MODULES='$INITRD_MODULES'" -Log "New INITRD_MODULES are ${OLD_INITRD_MODULES[*]} ${NEW_INITRD_MODULES[*]}" -INITRD_MODULES="${OLD_INITRD_MODULES[@]} ${NEW_INITRD_MODULES[@]}" - -WITH_INITRD_MODULES=$( printf '%s\n' ${INITRD_MODULES[*]} | awk '{printf "--with=%s ", $1}' ) +# Do not quote $INITRD_MODULES otherwise printf could not split words as separated arguments on separated lines: +WITH_INITRD_MODULES=$( printf '%s\n' $INITRD_MODULES | awk '{printf "--with=%s ", $1}' ) # Recreate any initrd or initramfs image under $TARGET_FS_ROOT/boot/ with new drivers # Images ignored: # kdump images as they are build by kdump -# initramfs rescue images (>= Rhel 7), which need all modules and -# are created by new-kernel-pkg +# initramfs rescue images (>= Rhel 7), which need all modules and are created by new-kernel-pkg # initrd-plymouth.img (>= Rhel 7), which contains only files needed for graphical boot via plymouth - +# The unalias originates from https://github.com/rear/rear/commit/d029e0dbe98b9fed496d8f3857bef132c8f165ad +# This is the only place in ReaR where unalias is called and it unaliases ls also for all subsequent scripts. +# TODO: Explain why this unalias is needed here and why it is also needed in all subsequent scripts: unalias ls 2>/dev/null for INITRD_IMG in $( ls $TARGET_FS_ROOT/boot/initramfs-*.img $TARGET_FS_ROOT/boot/initrd-*.img | egrep -v '(kdump|rescue|plymouth)' ) ; do @@ -88,7 +88,7 @@ for INITRD_IMG in $( ls $TARGET_FS_ROOT/boot/initramfs-*.img $TARGET_FS_ROOT/boo # and then "rear recover" cannot be aborted with the usual [Ctrl]+[C] keys. # Use plain $var because when var contains only blanks test "$var" results true because test " " results true: if test $mkinitrd_binary ; then - if chroot $TARGET_FS_ROOT $mkinitrd_binary -v -f ${WITH_INITRD_MODULES[*]} $INITRD $kernel_version ; then + if chroot $TARGET_FS_ROOT $mkinitrd_binary -v -f $WITH_INITRD_MODULES $INITRD $kernel_version ; then LogPrint "Updated initrd with new drivers for kernel $kernel_version." else LogPrint "WARNING: From 855675ebcb4beba22dfccef2c99ef12c8768e1f6 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 15 Mar 2022 14:07:01 +0100 Subject: [PATCH 056/215] Update 550_rebuild_initramfs.sh Removed useless "unalias ls" because "man bash" reads: "Aliases are not expanded when the shell is not interactive, unless the expand_aliases shell option is set" and 'expand_aliases' appears nowhere in ReaR. --- usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh index 1287dd2abf..6566dbcb55 100644 --- a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh +++ b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh @@ -67,11 +67,6 @@ WITH_INITRD_MODULES=$( printf '%s\n' $INITRD_MODULES | awk '{printf "--with=%s " # kdump images as they are build by kdump # initramfs rescue images (>= Rhel 7), which need all modules and are created by new-kernel-pkg # initrd-plymouth.img (>= Rhel 7), which contains only files needed for graphical boot via plymouth -# The unalias originates from https://github.com/rear/rear/commit/d029e0dbe98b9fed496d8f3857bef132c8f165ad -# This is the only place in ReaR where unalias is called and it unaliases ls also for all subsequent scripts. -# TODO: Explain why this unalias is needed here and why it is also needed in all subsequent scripts: -unalias ls 2>/dev/null - for INITRD_IMG in $( ls $TARGET_FS_ROOT/boot/initramfs-*.img $TARGET_FS_ROOT/boot/initrd-*.img | egrep -v '(kdump|rescue|plymouth)' ) ; do # Do not use KERNEL_VERSION here because that is readonly in the rear main script: kernel_version=$( basename $( echo $INITRD_IMG ) | cut -f2- -d"-" | sed s/"\.img"// ) From d615581577676d2e9201f1278d02e357273115f9 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 16 Mar 2022 09:14:51 +0100 Subject: [PATCH 057/215] Update default.conf In default.conf better describe USB_DEVICE in particular that normally it gets set automatically and if it is set then it must be a partition device like /dev/sdb1 cf. https://github.com/rear/rear/issues/2770#issuecomment-1068831482 --- usr/share/rear/conf/default.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 8f10f80757..368be67a1d 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -939,7 +939,11 @@ ISO_RECOVER_MODE="" # The recommended way to do that is to use the "rear format" workflow. # Call "rear format -- --help" (the '--' is mandatory) to see format workflow options. # -# The device to use, set automatically by BACKUP=NETFS and BACKUP_URL=usb:///dev/sdb1 +# The partition device to use (e.g. the exact partition like /dev/sdb1). +# Normally USB_DEVICE gets set automatically for BACKUP=NETFS +# via a specific BACKUP_URL like BACKUP_URL=usb:///dev/sdb1 +# or via the generic BACKUP_URL=usb:///dev/disk/by-label/REAR-000 +# (see also USB_DEVICE_FILESYSTEM_LABEL below): USB_DEVICE= # # The partition type that is used when formatting a medium for use with ReaR via the format workflow. From a09e5f4a01b46722bf35c0559dd02407f4d56d43 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 16 Mar 2022 11:10:50 +0100 Subject: [PATCH 058/215] Update default.conf In default.conf increase USB_UEFI_PART_SIZE to 1024 MiB cf. https://github.com/rear/rear/pull/1205 in particular to also make things work by default when additional hird-party kernel modules and firmware (e.g. from Nvidia) are used, cf. https://github.com/rear/rear/issues/2770#issuecomment-1068935688 --- usr/share/rear/conf/default.conf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 368be67a1d..f0aec9a1d8 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1007,18 +1007,20 @@ USB_PARTITION_ALIGN_BLOCK_SIZE="8" # in MiB when formatting a medium by the format workflow. # If USB_UEFI_PART_SIZE is empty or invalid (i.e. not an unsigned integer larger than 0) # the user must interactively enter a valid value while running the format workflow. -# The default value of 512 MiB should be sufficiently big and it is in compliance -# with the 8 MiB partition alignment default value ( 512 = 8 * 64 ) -# and also with higher 2^n MiB partition alignment values +# The default value of 1024 MiB should be sufficiently big # cf. https://github.com/rear/rear/pull/1205 -# Furthermore the default value of 512 MiB results that the FAT filesystem of the ESP +# in particular when third-party kernel modules and firmware (e.g. from Nvidia) are used +# cf. https://github.com/rear/rear/issues/2770#issuecomment-1068935688 +# and 1024 MiB is in compliance with the 8 MiB partition alignment value ( 1024 = 8 * 128 ) +# and also with higher 2^n MiB partition alignment values. +# Furthermore the default value of 1024 MiB results that the FAT filesystem of the ESP # will be in compliance with that the ESP should officially use a FAT32 filesystem # because mkfs.vfat automatically makes a FAT32 filesystem starting at 512 MiB # (a FAT16 ESP works in most cases but causes issues with certain UEFI firmware) # cf. https://github.com/rear/rear/issues/2575 # The value of USB_UEFI_PART_SIZE will be rounded to the nearest # USB_PARTITION_ALIGN_BLOCK_SIZE chunk: -USB_UEFI_PART_SIZE="512" +USB_UEFI_PART_SIZE="1024" # # Default boot option (i.e. what gets booted automatically after some timeout) # when EXTLINUX boots the USB stick or USB disk or other disk device on BIOS systems. From 84df1b88ef0000c8906ad6d7e27ff66ddb72acc2 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 16 Mar 2022 11:36:36 +0100 Subject: [PATCH 059/215] Update rear.8 Fixed typo '/dev/disk/by-path/REAR-000' -> '/dev/disk/by-label/REAR-000' --- doc/rear.8 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/rear.8 b/doc/rear.8 index 85cc944349..b899b6b3f5 100644 --- a/doc/rear.8 +++ b/doc/rear.8 @@ -247,8 +247,7 @@ TAPE_DEVICE\&. .PP OUTPUT=\fBUSB\fR .RS 4 -Create a bootable USB disk (using extlinux)\&. Specify the USB storage device by using -USB_DEVICE\&. +Create a bootable USB disk\&. .RE .PP OUTPUT=\fBRAWDISK\fR @@ -638,9 +637,9 @@ domain=_domain_ BACKUP_URL=\fBusb://\fR .RS 4 To backup to USB storage device, use -BACKUP_URL=usb:///dev/disk/by\-path/REAR\-000 -or use a real device node or a specific filesystem label\&. Alternatively, you can specify the device using -USB_DEVICE=/dev/disk/by\-path/REAR\-000\&. +BACKUP_URL=usb:///dev/disk/by\-label/REAR\-000 +or use a partition device node or a specific filesystem label\&. Alternatively, you may specify +the partition device using USB_DEVICE=/dev/disk/by\-label/REAR\-000 if needed\&. .sp If you combine this with OUTPUT=USB From 2e9b1d34afafeb5b305f254bdc954b0be23e97e9 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 16 Mar 2022 11:36:39 +0100 Subject: [PATCH 060/215] Update rear.8.adoc Fixed typo '/dev/disk/by-path/REAR-000' -> '/dev/disk/by-label/REAR-000' --- doc/rear.8.adoc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/rear.8.adoc b/doc/rear.8.adoc index c81948e604..d68c915c62 100644 --- a/doc/rear.8.adoc +++ b/doc/rear.8.adoc @@ -179,8 +179,7 @@ Create a bootable OBDR tape (optionally including the backup archive). Specify the OBDR tape device by using +TAPE_DEVICE+. OUTPUT=*USB*:: -Create a bootable USB disk (using extlinux). Specify the USB storage -device by using +USB_DEVICE+. +Create a bootable USB disk. OUTPUT=*RAWDISK*:: Create a bootable image file named "rear-$(hostname).raw.gz", which @@ -413,9 +412,9 @@ domain=_domain_ ---- BACKUP_URL=*usb://*:: -To backup to USB storage device, use +BACKUP_URL=usb:///dev/disk/by-path/REAR-000+ -or use a real device node or a specific filesystem label. Alternatively, you -can specify the device using +USB_DEVICE=/dev/disk/by-path/REAR-000+. +To backup to USB storage device, use +BACKUP_URL=usb:///dev/disk/by-label/REAR-000+ +or use a partition device node or a specific filesystem label. Alternatively, you +may specify the partition device using +USB_DEVICE=/dev/disk/by-label/REAR-000+ if needed. + If you combine this with +OUTPUT=USB+ you will end up with a bootable USB device. From 55e38dc76df7c1d8e145fd530465ddd8b71cdd73 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 18 Mar 2022 13:56:40 +0100 Subject: [PATCH 061/215] Update 210_raid_layout.sh Again support RAID10 "far" and "offset" layouts. Currently if there is more than one RAID10 layout value only the first one is used because according to https://unix.stackexchange.com/questions/280283/is-it-possible-to-create-a-mdadm-raid10-with-both-near-and-far-layout-options it seems it is not possible (or it does not make sense in practice) to set both "near=..." and "far=..." and we also assume it is not possible (or it does not make sense in practice) to set more than one RAID10 layout value because we I don't know what would be the mdadm command to create them, cf. https://github.com/rear/rear/pull/2768#issuecomment-1072361069 --- .../layout/save/GNU/Linux/210_raid_layout.sh | 96 ++++++++++++------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh index 03c81b89c2..debdfb1c84 100644 --- a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh @@ -17,7 +17,7 @@ local basename local line local metadata level raid_devices uuid name spare_devices layout chunksize component_devices container_size local container array_size -local layout_option layout_option_name layout_near_option_value layout_near_option +local layout_option layout_option_name layout_option_value layout_option_setting local component_device local raid_layout_entry local raid_dev_size raid_dev_label @@ -186,64 +186,90 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d uuid=${line[2]} test $uuid && raid_layout_entry+=" uuid=$uuid" - # A "Layout :" line in the detailed mdadm output looks like + # A "Layout :" line in the detailed mdadm output normally looks like # Layout : near=2 - # or - # Layout : near=2, far=2 - # or - # Layout : near=2, far=1 + # Layout : far=3 + # Layout : offset=4 + # cf. https://github.com/rear/rear/pull/2768#issuecomment-1072362485 + # and https://github.com/rear/rear/pull/2768#issuecomment-1072361069 + # but it might also look like + # Layout : near=2, far=3 + # or it might even look like (regardless that this was never seen in practice) + # Layout : near=2, far=3, offset=4 # cf. https://linux-blog.anracom.com/tag/far-layout/ # and https://unix.stackexchange.com/questions/280283/is-it-possible-to-create-a-mdadm-raid10-with-both-near-and-far-layout-options # and https://ubuntuforums.org/showthread.php?t=1689828&page=4 - # where for those examples the line array becomes ("declare -p line" outputs): + # For the above examples the line array becomes ("declare -p line" outputs): # declare -a line=([0]="Layout" [1]=":" [2]="near=2") - # declare -a line=([0]="Layout" [1]=":" [2]="near=2," [3]="far=2") - # declare -a line=([0]="Layout" [1]=":" [2]="near=2," [3]="far=1") - # so "near=..." is array element number 2 and "far=..." is array element number 3 if it exists: + # declare -a line=([0]="Layout" [1]=":" [2]="far=3") + # declare -a line=([0]="Layout" [1]=":" [2]="offset=4") + # declare -a line=([0]="Layout" [1]=":" [2]="near=2," [3]="far=3") + # declare -a line=([0]="Layout" [1]=":" [2]="near=2," [3]="far=3," [4]="offset=4") line=( $( grep "Layout :" $mdadm_details ) ) - # We use ${line[3]:-} to be safe against "bash: line[3]: unbound variable" in case of 'set -eu' + # We use ${line[3]:-} and ${line[4]:-} to be safe against things like + # "bash: line[3]: unbound variable" in case of 'set -eu' # so for the above examples the layout string becomes: # near=2 - # near=2,far=2 - # near=2,far=1 - layout="${line[2]}${line[3]:-}" + # far=3 + # offest=4 + # near=2,far=3 + # near=2,far=3,offset=4 + layout="${line[2]}${line[3]:-}${line[4]:-}" # For RAID10 have the layout value what the mdadm command needs as --layout option value # so with the above examples the mdadm command option --layout=... value has to become - # near=2 -> n2 - # near=2,far=2 -> n2 - # near=2,far=1 -> n2 - # TODO: Currently the RAID10 layout "far=..." value is ignored because according to + # near=2 -> n2 + # far=3 -> f3 + # offset=4 -> o4 + # near=2,far=3 -> n2 + # near=2,far=3,offset=4 -> n2 + # TODO: Currently if there is more than one RAID10 layout value only the first one is used because according to # https://unix.stackexchange.com/questions/280283/is-it-possible-to-create-a-mdadm-raid10-with-both-near-and-far-layout-options # it seems it is not possible (or it does not make sense in practice) to set both "near=..." and "far=..." - # see also the initial implementation https://github.com/rear/rear/commit/4dbbb288057189076d82cb8e477cd994a4ce1851 - # where no support for the "far=..." value was implemented which had been OK since then (dated May 24, 2011): + # and we also assume it is not possible (or it does not make sense in practice) to set more than one RAID10 layout value. if test "$level" = "raid10" ; then - layout_near_option="" + layout_option_setting="" OIFS=$IFS IFS="," for layout_option in $layout ; do layout_option_name=${layout_option%=*} - # Only use the "near=..." value but at least inform the user: - if ! test "$layout_option_name" = "near" ; then - LogPrintError "RAID10 layout option '$layout_option' is ignored (only 'near=...' support is implemented)" + layout_option_value=${layout_option#*=} + # The RAID10 layout option value must be "a small number" where "2 is normal, 3 can be useful" + # according to "man mdadm" (of mdadm v4.1 in openSUSE Leap 15.3). + # This test also fails when the RAID10 layout option value is not a number: + if ! test $layout_option_value -gt 0 ; then + LogPrintError "Ignoring RAID10 layout '$layout_option' for $raiddevice (the value is not at least 1)" continue fi - layout_near_option_value=${layout_option#*=} - # TODO: I do not understand why layout=n1 is not set by ReaR because - # the default is layout=n2 (according to 'man mdadm' with mdadm v4.1 on openSUSE Leap 15.3) - # so when a user has "Layout : near=1" then ReaR should recreate it as it was before - # regardless if such a RAID10 layout makes sense in practice - # so we tell the user at least in debug mode: - if ! test "$layout_near_option_value" -gt 1 ; then - DebugPrint "RAID10 layout option '$layout_option' is ignored (its value must be greater than 1)" + # Now the RAID10 layout option value is at least a number: + if ! test $layout_option_value -le 9 ; then + LogPrintError "Ignoring RAID10 layout '$layout_option' for $raiddevice (the value is not a small number)" continue fi - layout_near_option="n$layout_near_option_value" + # When a RAID10 layout option is already set for this RAID array an additional one is not supported: + if test $layout_option_setting ; then + LogPrintError "Ignoring additional RAID10 layout '$layout_option' for $raiddevice (only one RAID10 layout setting is supported)" + continue + fi + # Save the RAID10 layout option with the right syntax for the mdadm --layout option value during "rear recover": + case "$layout_option_name" in + (near) + layout_option_setting="n$layout_option_value" + ;; + (far) + layout_option_setting="f$layout_option_value" + ;; + (offset) + layout_option_setting="o$layout_option_value" + ;; + (*) + LogPrintError "Ignoring RAID10 layout '$layout_option' for $raiddevice (only 'near' 'far' and 'offset' are valid)" + ;; + esac done IFS=$OIFS - # Ensure $layout_near_option is a single non empty and non blank word + # Ensure $layout_option_setting is a single non empty and non blank word # (no quoting because test " " returns zero exit code): - test $layout_near_option && layout="$layout_near_option" || layout="" + test $layout_option_setting && layout="$layout_option_setting" || layout="" fi # mdadm can print '-unknown-' for a RAID layout # which got recently (2019-12-02) added to RAID0 (it existed before for RAID5 and RAID6 and RAID10) see From 86e54b543cfb29db86b70942f90d1c1b798b624c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 18 Mar 2022 14:06:58 +0100 Subject: [PATCH 062/215] Update 210_raid_layout.sh Do an initial general test first of all and not somewhere later in between other code. --- .../rear/layout/save/GNU/Linux/210_raid_layout.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh index debdfb1c84..4a70bd5562 100644 --- a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh @@ -231,6 +231,11 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d OIFS=$IFS IFS="," for layout_option in $layout ; do + # When a RAID10 layout option is already set for this RAID array an additional one is not supported: + if test $layout_option_setting ; then + LogPrintError "Ignoring additional RAID10 layout '$layout_option' for $raiddevice (only one RAID10 layout setting is supported)" + continue + fi layout_option_name=${layout_option%=*} layout_option_value=${layout_option#*=} # The RAID10 layout option value must be "a small number" where "2 is normal, 3 can be useful" @@ -245,11 +250,6 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d LogPrintError "Ignoring RAID10 layout '$layout_option' for $raiddevice (the value is not a small number)" continue fi - # When a RAID10 layout option is already set for this RAID array an additional one is not supported: - if test $layout_option_setting ; then - LogPrintError "Ignoring additional RAID10 layout '$layout_option' for $raiddevice (only one RAID10 layout setting is supported)" - continue - fi # Save the RAID10 layout option with the right syntax for the mdadm --layout option value during "rear recover": case "$layout_option_name" in (near) From 2bf11d52078e324b98618731298e4e4698e6c18d Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 21 Mar 2022 12:33:15 +0100 Subject: [PATCH 063/215] Update 310_network_devices.sh In rescue/GNU/Linux/310_network_devices.sh fix SC2091: Remove surrounding $() to avoid executing output cf. https://github.com/rear/rear/commit/15567ede425401b008e5b1680db36a2c62752b8f#r68281832 --- usr/share/rear/rescue/GNU/Linux/310_network_devices.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh index afc08193b9..ceb6960e00 100644 --- a/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh +++ b/usr/share/rear/rescue/GNU/Linux/310_network_devices.sh @@ -239,7 +239,7 @@ function map_network_interface () { local network_interface=$1 local mapped_as=$2 - if $( printf "%s\n" "${MAPPED_NETWORK_INTERFACES[@]}" | grep -qw ^$network_interface ) ; then + if printf "%s\n" "${MAPPED_NETWORK_INTERFACES[@]}" | grep -qw "^$network_interface" ; then # There is an error in the code. This means a handle_* function has # been called on an already mapped interface, which shouldn't happen. BugError "'$network_interface' is already mapped." From 90930fa7168be63fe6d1b668f870b03451c6a141 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 21 Mar 2022 14:36:30 +0100 Subject: [PATCH 064/215] Update 550_rebuild_initramfs.sh Quote ${OLD_INITRD_MODULES[@]} for consistency with places where it actually matters. --- usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh index 6566dbcb55..213c5c2da7 100644 --- a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh +++ b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh @@ -52,7 +52,7 @@ Log "Original OLD_INITRD_MODULES=( ${OLD_INITRD_MODULES[*]} )" # To achieve this, we list the old modules twice in the variable NEW_INITRD_MODULES and then add the new modules. # Then we use "uniq -u" to filter out the modules which only appear once in the list. # The resulting array contains only the new modules: -NEW_INITRD_MODULES=( ${OLD_INITRD_MODULES[@]} ${OLD_INITRD_MODULES[@]} $( cat $TMP_DIR/storage_drivers ) ) +NEW_INITRD_MODULES=( "${OLD_INITRD_MODULES[@]}" "${OLD_INITRD_MODULES[@]}" $( cat $TMP_DIR/storage_drivers ) ) NEW_INITRD_MODULES=( $( tr " " "\n" <<< "${NEW_INITRD_MODULES[*]}" | sort | uniq -u ) ) # Concatenate the old and new modules into a string: From 5d5dfedda3f4d36546618288515428b550543344 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 6 Apr 2022 11:59:46 +0200 Subject: [PATCH 065/215] Update 210_raid_layout.sh Use 'test ... && test ...' instead of 'test ... -a ...' to avoid a bash error message "bash: test: too many arguments" when $layout is blank or empty cf. https://github.com/rear/rear/pull/2768#discussion_r843740413 --- usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh index 4a70bd5562..7dafac4e1b 100644 --- a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh @@ -279,8 +279,13 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d # during "rear recover" with something like "mdadm: layout -unknown- not understood for raid0" # see https://github.com/rear/rear/issues/2616 # and ensure $layout is a single non empty and non blank word - # (no quoting because test " " returns zero exit code): - test $layout -a '-unknown-' != "$layout" && raid_layout_entry+=" layout=$layout" + # (no quoting because test " " returns zero exit code) + # and 'test ... && test ...' instead of 'test ... -a ...' to avoid a bash error message + # because when $layout is blank or empty test $layout -a '-unknown-' != "$layout" + # becomes test -a '-unknown-' != "" + # which results "bash: test: too many arguments" + # cf. https://github.com/rear/rear/pull/2768#discussion_r843740413 + test $layout && test '-unknown-' != "$layout" && raid_layout_entry+=" layout=$layout" chunksize=$( grep "Chunk Size" $mdadm_details | tr -d " " | cut -d ":" -f "2" | sed -r 's/^([0-9]+).+/\1/') test $chunksize && raid_layout_entry+=" chunk=$chunksize" From 87ed7f8b513273de378a50dea37ca4829d0bcc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Pannenb=C3=A4cker?= Date: Wed, 13 Apr 2022 10:18:20 +0200 Subject: [PATCH 066/215] Update .travis.yml Travis is currently not in use for this project thus its configuration file is commented out. --- .travis.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index a9409ae520..a07a39b1b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,13 @@ -language: shell - -arch: - - amd64 - - ppc64le - -os: linux - -script: - - make validate - - sudo make install +# Since June 15th, 2021, the building on travis-ci.org is ceased. Please use travis-ci.com from now on. +# +# language: shell +# +# arch: +# - amd64 +# - ppc64le +# +# os: linux +# +# script: +# - make validate +# - sudo make install From 8563989dce2cdf8983528a90e74b2d0a14fc80ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Pannenb=C3=A4cker?= Date: Wed, 13 Apr 2022 10:19:47 +0200 Subject: [PATCH 067/215] Update README.adoc Travis is not in use for this project (any more). The link to to the state has been removed from the README: --- README.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.adoc b/README.adoc index 1a9c267e46..cea4c54cb1 100644 --- a/README.adoc +++ b/README.adoc @@ -1,8 +1,6 @@ Relax-and-Recover ================= -image:https://travis-ci.org/rear/rear.svg?branch=master["Build Status", link="https://travis-ci.org/rear/rear"] - Relax-and-Recover is the leading Open Source bare metal disaster recovery and system migration solution. It is a modular framework with many ready-to-go workflows for common situations. From 9a5475e3e3ee4e6a070897d331a04b207301cedb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Aragon=C3=A9s=20Muniesa?= Date: Thu, 21 Apr 2022 17:20:24 +0200 Subject: [PATCH 068/215] recovering the original changes. adding suggested kernel parameters --- usr/share/rear/output/default/940_grub2_rescue.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/output/default/940_grub2_rescue.sh b/usr/share/rear/output/default/940_grub2_rescue.sh index f0650af547..7847d63b51 100644 --- a/usr/share/rear/output/default/940_grub2_rescue.sh +++ b/usr/share/rear/output/default/940_grub2_rescue.sh @@ -229,7 +229,7 @@ else fi ( echo " search --no-floppy --fs-uuid --set=root $grub_boot_uuid" echo " echo 'Loading kernel $boot_kernel_file ...'" - echo " linux $grub_boot_dir/$boot_kernel_name $KERNEL_CMDLINE" + echo " linux $grub_boot_dir/$boot_kernel_name root=/dev/ram0 vga=normal rw $KERNEL_CMDLINE" echo " echo 'Loading initrd $boot_initrd_file (may take a while) ...'" echo " initrd $grub_boot_dir/$boot_initrd_name" echo "}" From 5f8094cb7daa00d1d71ba6aaa119c6bc285ab19d Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 22 Apr 2022 14:17:46 +0200 Subject: [PATCH 069/215] Update 300_create_extlinux.sh In output/USB/Linux-i386/300_create_extlinux.sh add an explanatory comment how that code works, cf. https://github.com/rear/rear/pull/2794#issuecomment-1106286485 --- .../rear/output/USB/Linux-i386/300_create_extlinux.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh b/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh index 21ccefce80..8fba74d3b5 100644 --- a/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh +++ b/usr/share/rear/output/USB/Linux-i386/300_create_extlinux.sh @@ -143,6 +143,13 @@ EOF # Clean up older images of a given system, but keep USB_RETAIN_BACKUP_NR # entries for backup and rescue when backup on USB works in default mode. +# It keeps by default the two (by default USB_RETAIN_BACKUP_NR=2) +# topmost directories of the "ls -dt $BUILD_DIR/outputfs/rear/$HOSTNAME/*" +# output ("ls -t" sorts by time, newest first) +# regardless whether or not a new backup or a new rescue system was created +# so when there is no new backup or no new rescue system +# it keeps the existing directories as is, +# cf. https://github.com/rear/rear/pull/2794#issuecomment-1106286485 # When USB_SUFFIX is set the compliance mode is used where # backup on USB works in compliance with backup on NFS which means # a fixed backup directory and no automated removal of backups or other stuff From 199f4af47b9773103115bce78342735f02c8d104 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 22 Apr 2022 14:27:30 +0200 Subject: [PATCH 070/215] Update default.conf In default.conf tell that USB_RETAIN_BACKUP_NR is only supported when EXTLINUX is used as bootloader for USB (triggered by https://github.com/rear/rear/pull/2794) --- usr/share/rear/conf/default.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index f0aec9a1d8..dde6945285 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1056,7 +1056,9 @@ USB_SUFFIX="" # # Number of older rescue environments or backups to retain on USB. # What is more than USB_RETAIN_BACKUP_NR gets automatically removed. -# This setting is ignored when USB_SUFFIX is set (see above). +# This is only supported when EXTLINUX is used as bootloader for USB +# see USB_BOOTLOADER and output/USB/Linux-i386/300_create_extlinux.sh +# and this setting is ignored when USB_SUFFIX is set (see above). USB_RETAIN_BACKUP_NR=2 # # Booloader used for the USB medium. From 1d84f11397b3c361cdf216388ede897c715ca207 Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Sat, 23 Apr 2022 15:09:55 +0200 Subject: [PATCH 071/215] Update stale.yml Add bug and minor bug to the list of issues not to touch --- .github/stale.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/stale.yml b/.github/stale.yml index 261d5fac66..59e0528e81 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -10,6 +10,8 @@ exemptLabels: - critical/security bug - severe improvement - blocker + - bug + - minor bug # Label to use when marking an issue as stale staleLabel: ready-to-close? # Comment to post when marking an issue as stale. Set to `false` to disable From 09560933225891f9ae6a93fb46a9a691f2d03b51 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 13:03:28 +0200 Subject: [PATCH 072/215] Update 230_filesystem_layout.sh In layout/save/GNU/Linux/230_filesystem_layout.sh aviod SC1087 by using ${subvolume_mountpoint} with curly brackets because we need the subsequent square brackets literally (subvolume_mountpoint is a string, not an array) cf. https://github.com/rear/rear/issues/1040 --- .../rear/layout/save/GNU/Linux/230_filesystem_layout.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh b/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh index fb627a19f4..6aded348ee 100644 --- a/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh @@ -452,8 +452,10 @@ fi # try to find the mountpoint in /etc/fstab and try to read the subvol=... option value if exists # (using subvolid=... can fail because the subvolume ID can be different during system recovery). # Because both "mount ... -o subvol=/path/to/subvolume" and "mount ... -o subvol=path/to/subvolume" work - # the subvolume path can be specified with or without leading '/': - btrfs_subvolume_path=$( egrep "[[:space:]]$subvolume_mountpoint[[:space:]]+btrfs[[:space:]]" /etc/fstab \ + # the subvolume path can be specified with or without leading '/'. + # Aviod SC1087 by using ${subvolume_mountpoint} with curly brackets because + # we need the subsequent square brackets literally (subvolume_mountpoint is a string, not an array): + btrfs_subvolume_path=$( egrep "[[:space:]]${subvolume_mountpoint}[[:space:]]+btrfs[[:space:]]" /etc/fstab \ | egrep -v '^[[:space:]]*#' \ | grep -o 'subvol=[^ ]*' | cut -s -d '=' -f 2 ) fi From 44ee42e06f5c67507cfb1a7ac0f8066f6daa8daf Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 13:29:59 +0200 Subject: [PATCH 073/215] Update _input-output-functions.sh In lib/_input-output-functions.sh disable SC2218 "This function is only defined later. Move the definition up." because it seems ShellCheck falsely thinks 'trap' is the below defined function (i.e. it seems ShellCheck does not recognize 'builtin') cf. https://github.com/rear/rear/issues/1040 --- usr/share/rear/lib/_input-output-functions.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/lib/_input-output-functions.sh b/usr/share/rear/lib/_input-output-functions.sh index d4100bf6fc..3b2be0d3cf 100644 --- a/usr/share/rear/lib/_input-output-functions.sh +++ b/usr/share/rear/lib/_input-output-functions.sh @@ -322,7 +322,11 @@ function DoExitTasks () { done } -# The command (actually the function) DoExitTasks is executed on exit from the shell: +# The command (actually the function) DoExitTasks is executed on exit from the shell. +# Avoid SC2218 "This function is only defined later. Move the definition up." +# because it seems ShellCheck falsely thinks 'trap' is the below defined function +# (i.e. it seems ShellCheck does not recognize 'builtin') +# shellcheck disable=SC2218 builtin trap "DoExitTasks" EXIT # Prepare that STDIN STDOUT and STDERR can be later redirected to anywhere @@ -360,7 +364,11 @@ QuietAddExitTask "(( EXIT_FAIL_MESSAGE )) && echo '${MESSAGE_PREFIX}$PROGRAM $WO # USR1 is used to abort on errors. # It is not using PrintError but does direct output to the original STDERR. -# Set EXIT_FAIL_MESSAGE to 0 to avoid an additional failed message via the QuietAddExitTask above: +# Set EXIT_FAIL_MESSAGE to 0 to avoid an additional failed message via the QuietAddExitTask above. +# Avoid SC2218 "This function is only defined later. Move the definition up." +# because it seems ShellCheck falsely thinks 'trap' is the below defined function +# (i.e. it seems ShellCheck does not recognize 'builtin') +# shellcheck disable=SC2218 builtin trap "EXIT_FAIL_MESSAGE=0 ; echo '${MESSAGE_PREFIX}Aborting due to an error, check $RUNTIME_LOGFILE for details' 1>&8 ; kill $MASTER_PID" USR1 # Make sure nobody else can use trap: From c1da3dcb1a4306d3f670226d63151922294718ee Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 13:42:02 +0200 Subject: [PATCH 074/215] Update 240_swaps_layout.sh In layout/save/GNU/Linux/240_swaps_layout.s fixed SC2045 "Iterating over ls output is fragile. Use globs." by using plain "/dev/mapper/*" instead of "$(ls /dev/mapper/*)" cf. https://github.com/rear/rear/issues/1040 --- usr/share/rear/layout/save/GNU/Linux/240_swaps_layout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/240_swaps_layout.sh b/usr/share/rear/layout/save/GNU/Linux/240_swaps_layout.sh index b9a14e005e..e8f781e0d5 100644 --- a/usr/share/rear/layout/save/GNU/Linux/240_swaps_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/240_swaps_layout.sh @@ -11,7 +11,7 @@ Log "Saving Swap information." continue fi # if filename is on a lv, try to find the DM name - for dlink in $(ls /dev/mapper/*) ; do + for dlink in /dev/mapper/* ; do target=$(readlink -f $dlink) if [ "$target" = "$filename" ] ; then filename=$dlink From 9f9ca5d7e2f8953591547d7aece426394f77015d Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 13:55:55 +0200 Subject: [PATCH 075/215] Update 850_save_sysfs_uefi_vars.sh In rescue/default/850_save_sysfs_uefi_vars.sh disable SC2045: "Iterating over ls output is fragile. Use globs." because SYSFS_DIR_EFI_VARS is a directory where we want to 'ls' its contents cf. https://github.com/rear/rear/issues/1040 --- usr/share/rear/rescue/default/850_save_sysfs_uefi_vars.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/share/rear/rescue/default/850_save_sysfs_uefi_vars.sh b/usr/share/rear/rescue/default/850_save_sysfs_uefi_vars.sh index f2c73811c8..a1af17fa90 100644 --- a/usr/share/rear/rescue/default/850_save_sysfs_uefi_vars.sh +++ b/usr/share/rear/rescue/default/850_save_sysfs_uefi_vars.sh @@ -85,6 +85,9 @@ for dummy in "once" ; do # See prep/default/320_include_uefi_env.sh how SYSFS_DIR_EFI_VARS is set: case "$SYSFS_DIR_EFI_VARS" in (/sys/firmware/efi/vars) + # Disable SC2045: "Iterating over ls output is fragile. Use globs." + # because SYSFS_DIR_EFI_VARS is a directory where we want to 'ls' its contents + # shellcheck disable=SC2045 for uefi_dir in $( ls $SYSFS_DIR_EFI_VARS ) ; do uefi_var=$( echo $uefi_dir | cut -d- -f 1 ) [[ "$uefi_var" = "new_var" ]] && continue @@ -99,6 +102,9 @@ for dummy in "once" ; do uefi_bootloader_DOS_path=$( uefi_extract_bootloader $SYSFS_DIR_EFI_VARS/Boot${boot_current}-*/data ) ;; (/sys/firmware/efi/efivars) + # Disable SC2045: "Iterating over ls output is fragile. Use globs." + # because SYSFS_DIR_EFI_VARS is a directory where we want to 'ls' its contents + # shellcheck disable=SC2045 for uefi_file in $( ls $SYSFS_DIR_EFI_VARS ) ; do uefi_var=$( echo $uefi_file | cut -d- -f 1 ) efi_data="$( efibootmgr_read_var $uefi_var $efibootmgr_output )" From 5c366551b8ce8dcb212acca85514064588f0a2b0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 14:09:45 +0200 Subject: [PATCH 076/215] Update 500_make_backup.sh In backup/YUM/default/500_make_backup.sh fixed SC2068: "Double quote array expansions to avoid re-splitting elements." by using double quotes also for Log ... "${BACKUP_PROG_OPTIONS[@]}" (so it is now same as in backup/NETFS/default/500_make_backup.sh) cf. https://github.com/rear/rear/issues/1040 --- usr/share/rear/backup/YUM/default/500_make_backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/backup/YUM/default/500_make_backup.sh b/usr/share/rear/backup/YUM/default/500_make_backup.sh index 1687a0223f..2a77e04417 100644 --- a/usr/share/rear/backup/YUM/default/500_make_backup.sh +++ b/usr/share/rear/backup/YUM/default/500_make_backup.sh @@ -103,7 +103,7 @@ if ! is_true "$BACKUP_SELINUX_DISABLE" ; then fi # Generate the actual backup archive, excluding all of the RPM-provided files which have NOT been modified -Log tar --preserve-permissions --same-owner --warning=no-xdev --sparse --block-number --totals --no-wildcards-match-slash --one-file-system --ignore-failed-read ${BACKUP_PROG_OPTIONS[@]} --gzip -C / -c -f $backuparchive --exclude-from=$yum_backup_dir/rpm_backup_exclude_files.dat -X $TMP_DIR/backup-exclude.txt $(cat $TMP_DIR/backup-include.txt) $RUNTIME_LOGFILE +Log tar --preserve-permissions --same-owner --warning=no-xdev --sparse --block-number --totals --no-wildcards-match-slash --one-file-system --ignore-failed-read "${BACKUP_PROG_OPTIONS[@]}" --gzip -C / -c -f $backuparchive --exclude-from=$yum_backup_dir/rpm_backup_exclude_files.dat -X $TMP_DIR/backup-exclude.txt $(cat $TMP_DIR/backup-include.txt) $RUNTIME_LOGFILE tar --preserve-permissions --same-owner --warning=no-xdev --sparse --block-number --totals --no-wildcards-match-slash --one-file-system --ignore-failed-read "${BACKUP_PROG_OPTIONS[@]}" --gzip -C / -c -f $backuparchive --exclude-from=$yum_backup_dir/rpm_backup_exclude_files.dat -X $TMP_DIR/backup-exclude.txt $(cat $TMP_DIR/backup-include.txt) $RUNTIME_LOGFILE # Restore the ReaR default bash flags and options (see usr/sbin/rear): From cc3b6f63095154c78661adf77f008b8c5904ccea Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 14:29:10 +0200 Subject: [PATCH 077/215] Update 500_make_backup.sh In backup/NETFS/default/500_make_backup.sh fixed SC2068: "Double quote array expansions to avoid re-splitting elements." by using double quotes also for "${!backup_prog_shortnames[@]}" cf. https://github.com/rear/rear/issues/1040 --- usr/share/rear/backup/NETFS/default/500_make_backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/backup/NETFS/default/500_make_backup.sh b/usr/share/rear/backup/NETFS/default/500_make_backup.sh index 8efeeee0a5..229300c625 100644 --- a/usr/share/rear/backup/NETFS/default/500_make_backup.sh +++ b/usr/share/rear/backup/NETFS/default/500_make_backup.sh @@ -174,7 +174,7 @@ case "$(basename ${BACKUP_PROG})" in # Variable used to record the short name of piped commands in case of # error, e.g. ( "tar" "cat" "dd" ) in case of unencrypted and unsplit backup. - for index in ${!backup_prog_shortnames[@]} ; do + for index in "${!backup_prog_shortnames[@]}" ; do [ -n "${backup_prog_shortnames[$index]}" ] || BugError "No computed shortname for pipe component $index" done From 8edacc1ca03818416c35e9be65ee57dad97b94f3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 14:43:21 +0200 Subject: [PATCH 078/215] Update 500_make_duplicity_backup.sh In backup/DUPLICITY/default/500_make_duplicity_backup.sh disable SC2068: "Double quote array expansions to avoid re-splitting elements." because whether or not "Evaluation of Patterns By the Shell" (globbing) happens is controlled by BACKUP_DUPLICITY_EXCLUDE_EVALUATE_BY_SHELL cf. https://github.com/rear/rear/issues/1040 --- .../backup/DUPLICITY/default/500_make_duplicity_backup.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/backup/DUPLICITY/default/500_make_duplicity_backup.sh b/usr/share/rear/backup/DUPLICITY/default/500_make_duplicity_backup.sh index 53dff408e0..269873632a 100644 --- a/usr/share/rear/backup/DUPLICITY/default/500_make_duplicity_backup.sh +++ b/usr/share/rear/backup/DUPLICITY/default/500_make_duplicity_backup.sh @@ -43,11 +43,13 @@ if [ "$BACKUP_PROG" = "duplicity" ] ; then if ! is_true "$BACKUP_DUPLICITY_EXCLUDE_EVALUATE_BY_SHELL"; then set -f # Temporarily Stop Evaluation of Patterns By the Shell fi - + # Disable SC2068: "Double quote array expansions to avoid re-splitting elements." + # because whether or not "Evaluation of Patterns By the Shell" (globbing) happens + # is controlled by BACKUP_DUPLICITY_EXCLUDE_EVALUATE_BY_SHELL + # shellcheck disable=SC2068 for EXDIR in ${BACKUP_DUPLICITY_EXCLUDE[@]} ; do - EXCLUDES="$EXCLUDES --exclude $EXDIR" + EXCLUDES+=" --exclude $EXDIR" done - if ! is_true "$BACKUP_DUPLICITY_EXCLUDE_EVALUATE_BY_SHELL"; then set +f # Reenable Evaluation of Patterns By the Shell fi From 5f4c7383a55785127f487b9263c2bac09a2f0d44 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 14:51:53 +0200 Subject: [PATCH 079/215] Update 070_set_backup_archive.sh In prep/NETFS/default/070_set_backup_archive.sh fixed SC2068: "Double quote array expansions to avoid re-splitting elements." by using double quotes also for IsInArray "$WORKFLOW" "${..._workflows[@]}" cf. https://github.com/rear/rear/issues/1040 --- .../rear/prep/NETFS/default/070_set_backup_archive.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh b/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh index 4961bcee60..31f555a4c8 100644 --- a/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh +++ b/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh @@ -51,7 +51,7 @@ if ! test "incremental" = "$BACKUP_TYPE" -o "differential" = "$BACKUP_TYPE" ; th LogPrint "Using backup archive '$backuparchive'" # This script is also run during "rear recover/restoreonly" where RESTORE_ARCHIVES must be set. local backup_restore_workflows=( "recover" "restoreonly" ) - if IsInArray $WORKFLOW ${backup_restore_workflows[@]} ; then + if IsInArray "$WORKFLOW" "${backup_restore_workflows[@]}" ; then # Only set RESTORE_ARCHIVES when the backup archive is actually accessible # cf. https://github.com/rear/rear/issues/1166 if test -r "$backuparchive" ; then @@ -163,7 +163,7 @@ local create_backup_type="" # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": local recovery_workflows=( "recover" "layoutonly" "restoreonly" ) -if ! IsInArray $WORKFLOW ${recovery_workflows[@]} ; then +if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # When today is a specified full backup day, do a full backup in any case # (regardless if there is already a full backup of this day): if IsInArray "$current_weekday" "${FULLBACKUPDAY[@]}" ; then @@ -187,7 +187,7 @@ if test "$latest_full_backup" ; then local full_or_incremental_backup_glob_regex="$date_time_glob_regex-[$full_backup_marker$incremental_backup_marker]$backup_file_suffix" # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": - if ! IsInArray $WORKFLOW ${recovery_workflows[@]} ; then + if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # There is nothing to do here if it is already decided that # a full backup must be created (see "full backup day" above"): if ! test "full" = "$create_backup_type" ; then @@ -241,7 +241,7 @@ if test "$latest_full_backup" ; then else # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": - if ! IsInArray $WORKFLOW ${recovery_workflows[@]} ; then + if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # If no latest full backup is found create one during "rear mkbackup": create_backup_type="full" LogPrint "No full backup found (YYYY-MM-DD-HHMM-F.tar.gz) triggers full backup" From 1e39593335d990988778b05da5135a31152e46a8 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 14:59:50 +0200 Subject: [PATCH 080/215] Update output-functions.sh In lib/output-functions.sh fixed SC2045 "Iterating over ls output is fragile. Use globs." by using plain "/sys/block/*/device/model" instead of "$( ls /sys/block/*/device/model )" cf. https://github.com/rear/rear/issues/1040 --- usr/share/rear/lib/output-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/output-functions.sh b/usr/share/rear/lib/output-functions.sh index 923fb065a2..d71cb23b37 100644 --- a/usr/share/rear/lib/output-functions.sh +++ b/usr/share/rear/lib/output-functions.sh @@ -5,7 +5,7 @@ function FindUsbDevices () { local d sysfspath device # we use the model to find USB devices - for d in $( ls /sys/block/*/device/model ) ; do + for d in /sys/block/*/device/model ; do grep -q -i -E 'usb|FlashDisk' $d || continue # analyzing $d # /sys/block/sdb From cae8eae6b344a25b48e9f5fe1d1d14aad34261fb Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 26 Apr 2022 15:03:18 +0200 Subject: [PATCH 081/215] Update 070_set_backup_archive.sh In prep/NETFS/default/070_set_backup_archive.sh fixed a missed SC2068: "Double quote array expansions to avoid re-splitting elements." by using double quotes also for IsInArray "$WORKFLOW" "${..._workflows[@]}" cf. https://github.com/rear/rear/commit/5f4c7383a55785127f487b9263c2bac09a2f0d44 and https://github.com/rear/rear/issues/1040 --- usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh b/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh index 31f555a4c8..4644b79bb0 100644 --- a/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh +++ b/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh @@ -261,7 +261,7 @@ else fi # Code regarding creating a backup is useless during "rear recover" and # messages about creating a backup are misleading during "rear recover": -if ! IsInArray $WORKFLOW ${recovery_workflows[@]} ; then +if ! IsInArray "$WORKFLOW" "${recovery_workflows[@]}" ; then # Set the right variables for creating a backup (but do not actually do anything at this point): case "$create_backup_type" in (full) From 1334414ecfb1577ee080ad9eb94b045c491c3223 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 27 Apr 2022 12:54:50 +0200 Subject: [PATCH 082/215] Update _input-output-functions.sh In the LogPrintError function comment in rear/lib/_input-output-functions.sh tell that real error messages should be prefixed with 'Error: ' in the LogPrintError message but messages that are no real error messages should not be prefixed with 'Warning: ' see https://blog.schlomo.schapiro.org/2015/04/warning-is-waste-of-my-time.html cf. https://github.com/rear/rear/pull/2795#issuecomment-1110798744 --- usr/share/rear/lib/_input-output-functions.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/lib/_input-output-functions.sh b/usr/share/rear/lib/_input-output-functions.sh index 3b2be0d3cf..4e04ca21b5 100644 --- a/usr/share/rear/lib/_input-output-functions.sh +++ b/usr/share/rear/lib/_input-output-functions.sh @@ -489,9 +489,11 @@ function LogUserOutput () { # LogPrintError does not error out (the Error function is meant to error out). # LogPrintError is meant to show error messages when we do not want to error out, # (for example when at the end of "rear recover" it failed to install a bootloader). +# Real error messages should be prefixed with 'Error: ' in the LogPrintError message. # LogPrintError is also meant to show important "error-like" messages to the user # (for example when the user must decide if that means a real error in his case) -# and other important messages that must appear on the user's terminal +# and other important messages that must appear on the user's terminal. +# Messages that are no real error messages should not be prefixed with 'Warning: ' # cf. https://blog.schlomo.schapiro.org/2015/04/warning-is-waste-of-my-time.html function LogPrintError () { Log "$@" From 17ee5f87d742a57f82960b08795b07896969ac8b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 4 May 2022 13:08:31 +0200 Subject: [PATCH 083/215] Update 450_check_bootloader_files.sh In layout/save/default/450_check_bootloader_files.sh added bash globbing characters [] around the first letter to ensure that with 'shopt -s nullglob' files that do not exist will not appear so nonexistent files are not appended to CHECK_CONFIG_FILES cf. https://github.com/rear/rear/pull/2796#issuecomment-1117171070 --- .../save/default/450_check_bootloader_files.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/usr/share/rear/layout/save/default/450_check_bootloader_files.sh b/usr/share/rear/layout/save/default/450_check_bootloader_files.sh index d0a6b6292c..1324220ede 100644 --- a/usr/share/rear/layout/save/default/450_check_bootloader_files.sh +++ b/usr/share/rear/layout/save/default/450_check_bootloader_files.sh @@ -7,33 +7,38 @@ # An artificial bash array is used so that the first array element $used_bootloader is the used bootloader: used_bootloader=( $( cat $VAR_DIR/recovery/bootloader ) ) +# No quoting of the elements that are appended to the CHECK_CONFIG_FILES array together with +# the bash globbing characters like '*' or the [] around the first letter make sure +# that with 'shopt -s nullglob' files that do not exist will not appear +# so nonexistent files are not appended to CHECK_CONFIG_FILES +# cf. https://github.com/rear/rear/pull/2796#issuecomment-1117171070 case $used_bootloader in (EFI|GRUB2-EFI) CHECK_CONFIG_FILES+=( /boot/efi/EFI/*/grub*.cfg ) ;; (GRUB|GRUB2) - CHECK_CONFIG_FILES+=( /etc/grub.cfg /etc/grub2.cfg /boot/grub2/grub2.cfg /boot/grub/grub.cfg ) + CHECK_CONFIG_FILES+=( /[e]tc/grub.cfg /[e]tc/grub2.cfg /[b]oot/grub2/grub2.cfg /[b]oot/grub/grub.cfg ) ;; (LILO) - CHECK_CONFIG_FILES+=( /etc/lilo.conf ) + CHECK_CONFIG_FILES+=( /[e]tc/lilo.conf ) ;; (ELILO) - CHECK_CONFIG_FILES+=( /etc/elilo.conf ) + CHECK_CONFIG_FILES+=( /[e]tc/elilo.conf ) ;; (PPC) # PPC arch bootloader can be : # - LILO : SLES < 12 # - YABOOT : RHEL < 7 # - GRUB2 : SLES >= 12, RHEL >= 7, Ubuntu and other new Linux on POWER distro. - CHECK_CONFIG_FILES+=( /etc/lilo.conf /etc/yaboot.conf /etc/grub.cfg /etc/grub2.cfg /boot/grub2/grub2.cfg /boot/grub/grub.cfg) + CHECK_CONFIG_FILES+=( /[e]tc/lilo.conf /[e]tc/yaboot.conf /[e]tc/grub.cfg /[e]tc/grub2.cfg /[b]oot/grub2/grub2.cfg /[b]oot/grub/grub.cfg) ;; (ARM|ARM-ALLWINNER) - CHECK_CONFIG_FILES+=( /boot/boot.scr ) + CHECK_CONFIG_FILES+=( /[b]oot/boot.scr ) ;; (ZIPL) # cf. https://github.com/rear/rear/issues/2137 # s390 - for rhel, ubuntu zipl config must be exist for restore. sles > 11 does not use zipl directly - CHECK_CONFIG_FILES+=( /etc/zipl.conf ) + CHECK_CONFIG_FILES+=( /[e]tc/zipl.conf ) ;; (*) BugError "Unknown bootloader ($used_bootloader) - ask for sponsoring to get this fixed" From 8c847f7b17d905d769acdf44dcb6d914cbe49c4a Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 4 May 2022 13:12:50 +0200 Subject: [PATCH 084/215] Update 450_check_network_files.sh In layout/save/default/450_check_network_files.sh added bash globbing characters [] around the first letter to ensure that with 'shopt -s nullglob' files that do not exist will not appear so nonexistent files are not appended to CHECK_CONFIG_FILES cf. https://github.com/rear/rear/pull/2796#issuecomment-1117171070 --- .../rear/layout/save/default/450_check_network_files.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/default/450_check_network_files.sh b/usr/share/rear/layout/save/default/450_check_network_files.sh index c79ee748c1..e926ce3926 100644 --- a/usr/share/rear/layout/save/default/450_check_network_files.sh +++ b/usr/share/rear/layout/save/default/450_check_network_files.sh @@ -1,6 +1,12 @@ # This file is part of Relax-and-Recover, licensed under the GNU General # Public License. Refer to the included COPYING for full text of license. + +# No quoting of the elements that are appended to the CHECK_CONFIG_FILES array together with +# the bash globbing characters like '*' or the [] around the first letter make sure +# that with 'shopt -s nullglob' files that do not exist will not appear +# so nonexistent files are not appended to CHECK_CONFIG_FILES +# cf. https://github.com/rear/rear/pull/2796#issuecomment-1117171070 if [[ -d /etc/sysconfig/network ]] ; then # SUSE CHECK_CONFIG_FILES+=( /etc/sysconfig/network/ifcfg-* ) @@ -9,6 +15,6 @@ elif [[ -d /etc/sysconfig/network-scripts ]] ; then CHECK_CONFIG_FILES+=( /etc/sysconfig/network-scripts/ifcfg-* ) elif [[ -d /etc/network ]] ; then # Debian - CHECK_CONFIG_FILES+=( /etc/network/interfaces ) + CHECK_CONFIG_FILES+=( /[e]tc/network/interfaces ) fi From b4d537c53fbee16f0328fdace52f75ea95165a2a Mon Sep 17 00:00:00 2001 From: Gratien D'haese Date: Thu, 5 May 2022 11:46:22 +0200 Subject: [PATCH 085/215] Comment out remote space check with rsync #2760 Signed-off-by: Gratien D'haese --- .../RSYNC/default/450_calculate_req_space.sh | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh index eb99dbf6ba..4eccf25313 100644 --- a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh +++ b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh @@ -13,21 +13,22 @@ while read -r ; do done < $TMP_DIR/backup-include.txt LogPrint "Estimated size of local file systems is $(( _local_size / 1024 )) MB" -case $RSYNC_PROTO in - (ssh) - LogPrint "Calculating size of $RSYNC_HOST:$RSYNC_PATH" - ssh -l $RSYNC_USER $RSYNC_HOST "df -P $RSYNC_PATH" >$TMP_DIR/rs_size - StopIfError "Failed to determine size of $RSYNC_PATH" - _div=1 # 1024-blocks - grep -q "512-blocks" $TMP_DIR/rs_size && _div=2 # HPUX: divide with 2 to get kB size - _remote_size=$( tail -n 1 $TMP_DIR/rs_size | awk '{print $2}' ) - _remote_size=$(( _remote_size / _div )) - [[ $_remote_size -gt $_local_size ]] - StopIfError "Not enough disk space available on $RSYNC_HOST:$RSYNC_PATH ($_remote_size < $_local_size)" - ;; - (rsync) - # TODO: how can we calculate the free size on remote system via rsync protocol?? - : - ;; -esac +# Commenting out next block according decision of issue #2760 +#case $RSYNC_PROTO in +# (ssh) +# LogPrint "Calculating size of $RSYNC_HOST:$RSYNC_PATH" +# ssh -l $RSYNC_USER $RSYNC_HOST "df -P $RSYNC_PATH" >$TMP_DIR/rs_size +# StopIfError "Failed to determine size of $RSYNC_PATH" +# _div=1 # 1024-blocks +# grep -q "512-blocks" $TMP_DIR/rs_size && _div=2 # HPUX: divide with 2 to get kB size +# _remote_size=$( tail -n 1 $TMP_DIR/rs_size | awk '{print $2}' ) +# _remote_size=$(( _remote_size / _div )) +# [[ $_remote_size -gt $_local_size ]] +# StopIfError "Not enough disk space available on $RSYNC_HOST:$RSYNC_PATH ($_remote_size < $_local_size)" +# ;; +# (rsync) +# # TODO: how can we calculate the free size on remote system via rsync protocol?? +# : +# ;; +#esac From f7ea701da2aeb0d46c75a295a2e4a68ac02a9474 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Mon, 25 Apr 2022 17:48:45 +0200 Subject: [PATCH 086/215] Verify checksums after file restore from backup Configuration files can get out-of-sync with the recreated layout. This leads to inconsistent configuration, e.g. filesystem not mountable at boot, if they are mounted by UUIDs, and failures to migrate UUIDs in files (if needed). To detect this problem, use the saved md5sums of configuration files (originally used for "checklayout") to verify the restored files and inform the user in the case of a mismatch. No attempt is made to repair the problem; the user is informed, but they must fix the problem manually. --- usr/share/rear/finalize/default/060_compare_files.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 usr/share/rear/finalize/default/060_compare_files.sh diff --git a/usr/share/rear/finalize/default/060_compare_files.sh b/usr/share/rear/finalize/default/060_compare_files.sh new file mode 100644 index 0000000000..5d7b061335 --- /dev/null +++ b/usr/share/rear/finalize/default/060_compare_files.sh @@ -0,0 +1,6 @@ +if [ -e $VAR_DIR/layout/config/files.md5sum ] ; then + if ! chroot $TARGET_FS_ROOT md5sum -c --quiet < $VAR_DIR/layout/config/files.md5sum ; then + LogPrintError "Some configuration files in the restored system do not match the saved layout!" + return 1 + fi +fi From 7f61a4f4283e403ba3e44db9ee6bd52bd45d9519 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Mon, 25 Apr 2022 17:55:43 +0200 Subject: [PATCH 087/215] Save hashes of all files that need ID migration If a file may need a migration of UUIDs, then it is very likely prone to get out-of-sync with the layout if it gets modified. Add all such files to the list of files whose hashes are recorded in the saved layout to detect such problems. Introduce a variable called FILES_TO_PATCH_PATTERNS to hold the list of these files (shell glob patterns, actually), as it is now needed in multiple places. Would it make sense to just remove [e]tc/mtab from FILES_TO_PATCH_PATTERNS? The file used to be maintaned dynamically by mount(8) even before it became a symlink. Why has it been ever useful to patch the IDs or device names in it, if it gets regenerated on every boot anyway? --- usr/share/rear/conf/default.conf | 21 +++++++++ .../GNU/Linux/280_migrate_uuid_tags.sh | 15 ++----- .../save/default/490_check_files_to_patch.sh | 43 +++++++++++++++++++ 3 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 usr/share/rear/layout/save/default/490_check_files_to_patch.sh diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index dde6945285..2739a9f4bf 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -332,8 +332,29 @@ CDROM_SIZE=20 # which exits with non-zero exit code when the disk layout or those files changed # (cf. https://github.com/rear/rear/issues/1134) but the checklayout workflow # does not automatically recreate the rescue/recovery system. +# Files matching FILES_TO_PATCH_PATTERNS are added to this list automatically. CHECK_CONFIG_FILES=( '/etc/drbd/' '/etc/drbd.conf' '/etc/lvm/lvm.conf' '/etc/multipath.conf' '/etc/rear/' '/etc/udev/udev.conf' ) +# FILES_TO_PATCH_PATTERNS is a space-separated list of shell glob patterns. +# Files that match are eligible for a final migration of UUIDs and other +# identifiers after recovery (if the layout recreation process has led +# to a change of an UUID or a device name and a corresponding change needs +# to be performed on restored configuration files ). +# See finalize/GNU/Linux/280_migrate_uuid_tags.sh +# The [] around the first letter make sure that shopt -s nullglob removes this file from the list if it does not exist + +FILES_TO_PATCH_PATTERNS="[b]oot/{grub.conf,menu.lst,device.map} [e]tc/grub.* \ + [b]oot/grub/{grub.conf,grub.cfg,menu.lst,device.map} \ + [b]oot/grub2/{grub.conf,grub.cfg,menu.lst,device.map} \ + [e]tc/sysconfig/grub [e]tc/sysconfig/bootloader \ + [e]tc/lilo.conf [e]tc/elilo.conf \ + [e]tc/mtab [e]tc/fstab \ + [e]tc/mtools.conf \ + [e]tc/smartd.conf [e]tc/sysconfig/smartmontools \ + [e]tc/sysconfig/rawdevices \ + [e]tc/security/pam_mount.conf.xml \ + [b]oot/efi/*/*/grub.cfg" + ## # Relax-and-Recover recovery system update during "rear recover" # diff --git a/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh b/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh index 074689a110..d994ce8ef2 100644 --- a/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh +++ b/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh @@ -23,18 +23,9 @@ LogPrint "Migrating filesystem UUIDs in certain restored files in $TARGET_FS_ROO local symlink_target="" local restored_file="" -# the funny [] around the first letter make sure that shopt -s nullglob removes this file from the list if it does not exist -# the files without a [] are mandatory, like fstab FIXME: but below there is [e]tc/fstab not etc/fstab - why? -for restored_file in [b]oot/{grub.conf,menu.lst,device.map} [e]tc/grub.* \ - [b]oot/grub/{grub.conf,grub.cfg,menu.lst,device.map} \ - [b]oot/grub2/{grub.conf,grub.cfg,menu.lst,device.map} \ - [e]tc/sysconfig/grub [e]tc/sysconfig/bootloader \ - [e]tc/lilo.conf [e]tc/elilo.conf \ - [e]tc/mtab [e]tc/fstab \ - [e]tc/mtools.conf \ - [e]tc/smartd.conf [e]tc/sysconfig/smartmontools \ - [e]tc/sysconfig/rawdevices \ - [e]tc/security/pam_mount.conf.xml [b]oot/efi/*/*/grub.cfg +# The variable expansion is deliberately not quoted in order to perform +# pathname expansion on the variable value. +for restored_file in $FILES_TO_PATCH_PATTERNS do # Silently skip directories and file not found: test -f "$restored_file" || continue diff --git a/usr/share/rear/layout/save/default/490_check_files_to_patch.sh b/usr/share/rear/layout/save/default/490_check_files_to_patch.sh new file mode 100644 index 0000000000..ee717063a1 --- /dev/null +++ b/usr/share/rear/layout/save/default/490_check_files_to_patch.sh @@ -0,0 +1,43 @@ +# FILES_TO_PATCH_PATTERNS is a space-separated list of shell glob patterns. +# Files that match are eligible for a final migration of UUIDs and other +# identifiers after recovery (if the layout recreation process has led +# to a change of an UUID or a device name and a corresponding change needs +# to be performed on restored configuration files ). +# See finalize/GNU/Linux/280_migrate_uuid_tags.sh +# We should add all such files to CHECK_CONFIG_FILES - if they change, +# we risk inconsistencies between the restored files and recreated layout, +# or failures of UUID migration. + +local file final_file symlink_target + +# The patterns are relative to /, change directory there +# so that the shell finds the files during pathname expansion +pushd / >/dev/null +# The variable expansion is deliberately not quoted in order to perform +# pathname expansion on the variable value. +for file in $FILES_TO_PATCH_PATTERNS ; do + final_file="/$file" + IsInArray "$final_file" "${CHECK_CONFIG_FILES[@]}" && continue + # Symlink handling (partially from 280_migrate_uuid_tags.sh): + # avoid dead symlinks, and symlinks to files on dynamic filesystems + # ( /proc etc.) - they are expected to change and validating + # their checksums has no sense + if test -L "$final_file" ; then + if symlink_target="$( readlink -e "$final_file" )" ; then + # If the symlink target contains /proc/ /sys/ /dev/ or /run/ we skip it because then + # the symlink target is considered to not be a restored file that needs to be patched + # and thus we don't need to generate and check its hash, either + # cf. https://github.com/rear/rear/pull/2047#issuecomment-464846777 + if echo $symlink_target | egrep -q '/proc/|/sys/|/dev/|/run/' ; then + Log "Skip adding symlink $final_file target $symlink_target on /proc/ /sys/ /dev/ or /run/ to CHECK_CONFIG_FILES" + continue + fi + Debug "Adding symlink $final_file with target $symlink_target to CHECK_CONFIG_FILES" + else + LogPrint "Skip adding dead symlink $final_file to CHECK_CONFIG_FILES" + continue + fi + fi + CHECK_CONFIG_FILES+=( "$final_file" ) +done +popd >/dev/null From b3f1ffcd70b45b4d4e0931b1a93a979538dcc45c Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Tue, 26 Apr 2022 15:35:43 +0200 Subject: [PATCH 088/215] Reuse patterns of files to transform finalize/GNU/Linux/250_migrate_disk_devices_layout.sh and finalize/GNU/Linux/280_migrate_uuid_tags.sh used to have the same list of file patterns to transform (migrate disk names and IDs), but the list have diverged. Unify them again by using the variable FILES_TO_PATCH_PATTERNS for both (with merged content). XXX finalize/GNU/Linux/260_rename_diskbyid.sh seems to be doing a similar thing, but the list of files that it transforms is different. Should we unify it with the other two as well? XXX finalize/GNU/Linux/260_rename_diskbyid.sh prints this "WARNING ! You are mounting some devices by ID. Please be aware that the IDs are hardware dependent and that you might have to adjust your fstab to match the new IDs. Currently your system has the following disks with LUN IDs:" but why, if finalize/GNU/Linux/260_rename_diskbyid.sh is supposed to fix them? --- usr/share/rear/conf/default.conf | 1 + .../GNU/Linux/250_migrate_disk_devices_layout.sh | 16 +++------------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 2739a9f4bf..3406e93b1d 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -348,6 +348,7 @@ FILES_TO_PATCH_PATTERNS="[b]oot/{grub.conf,menu.lst,device.map} [e]tc/grub.* \ [b]oot/grub2/{grub.conf,grub.cfg,menu.lst,device.map} \ [e]tc/sysconfig/grub [e]tc/sysconfig/bootloader \ [e]tc/lilo.conf [e]tc/elilo.conf \ + [e]tc/yaboot.conf \ [e]tc/mtab [e]tc/fstab \ [e]tc/mtools.conf \ [e]tc/smartd.conf [e]tc/sysconfig/smartmontools \ diff --git a/usr/share/rear/finalize/GNU/Linux/250_migrate_disk_devices_layout.sh b/usr/share/rear/finalize/GNU/Linux/250_migrate_disk_devices_layout.sh index 1a91a0e309..e869e5e97a 100644 --- a/usr/share/rear/finalize/GNU/Linux/250_migrate_disk_devices_layout.sh +++ b/usr/share/rear/finalize/GNU/Linux/250_migrate_disk_devices_layout.sh @@ -29,19 +29,9 @@ LogPrint "The original restored files get saved in $save_original_file_dir (in $ local symlink_target="" local restored_file="" -# the funny [] around the first letter make sure that shopt -s nullglob removes this file from the list if it does not exist -# the files without a [] are mandatory, like fstab FIXME: but below there is [e]tc/fstab not etc/fstab - why? - -for restored_file in [b]oot/{grub.conf,menu.lst,device.map} [e]tc/grub.* [b]oot/grub/{grub.conf,menu.lst,device.map} \ - [b]oot/grub2/{grub.conf,grub.cfg,menu.lst,device.map} \ - [e]tc/sysconfig/grub [e]tc/sysconfig/bootloader \ - [e]tc/lilo.conf \ - [e]tc/yaboot.conf \ - [e]tc/mtab [e]tc/fstab \ - [e]tc/mtools.conf \ - [e]tc/smartd.conf [e]tc/sysconfig/smartmontools \ - [e]tc/sysconfig/rawdevices \ - [e]tc/security/pam_mount.conf.xml [b]oot/efi/*/*/grub.cfg +# The variable expansion is deliberately not quoted in order to perform +# pathname expansion on the variable value. +for restored_file in $FILES_TO_PATCH_PATTERNS do # Silently skip directories and file not found: test -f "$restored_file" || continue From 0d76d6b4da44a7ec21f4b8e47a6cc831af1ac5f5 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Thu, 28 Apr 2022 15:37:11 +0200 Subject: [PATCH 089/215] Improve the visibility of checksum errors Prefix the message about mismatch between layout and restored files by "Error:", as it is probably a serious problem that may require manual fixup. Show the list of changed files displayed by md5sum and also save it in the log. --- usr/share/rear/finalize/default/060_compare_files.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/finalize/default/060_compare_files.sh b/usr/share/rear/finalize/default/060_compare_files.sh index 5d7b061335..6947fda9f6 100644 --- a/usr/share/rear/finalize/default/060_compare_files.sh +++ b/usr/share/rear/finalize/default/060_compare_files.sh @@ -1,6 +1,6 @@ if [ -e $VAR_DIR/layout/config/files.md5sum ] ; then - if ! chroot $TARGET_FS_ROOT md5sum -c --quiet < $VAR_DIR/layout/config/files.md5sum ; then - LogPrintError "Some configuration files in the restored system do not match the saved layout!" + if ! chroot $TARGET_FS_ROOT md5sum -c --quiet < $VAR_DIR/layout/config/files.md5sum 1>> >( tee -a "$RUNTIME_LOGFILE" 1>&7 ) 2>> >( tee -a "$RUNTIME_LOGFILE" 1>&8 ) ; then + LogPrintError "Error: Restored files do not match the recreated system in $TARGET_FS_ROOT" return 1 fi fi From b1b1706aed3969a334b7ea6bc73f89481ff1e687 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Wed, 4 May 2022 14:38:46 +0200 Subject: [PATCH 090/215] Declare local variables --- usr/share/rear/layout/save/default/600_snapshot_files.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/default/600_snapshot_files.sh b/usr/share/rear/layout/save/default/600_snapshot_files.sh index 0ebf197cc2..3ac6b07ef4 100644 --- a/usr/share/rear/layout/save/default/600_snapshot_files.sh +++ b/usr/share/rear/layout/save/default/600_snapshot_files.sh @@ -3,7 +3,8 @@ if [ "$WORKFLOW" = "checklayout" ] ; then return 0 fi -config_files=() +local obj +local config_files=() for obj in "${CHECK_CONFIG_FILES[@]}" ; do if [ -d "$obj" ] ; then config_files+=( $( find "$obj" -type f ) ) From 5ea6f988b78e687df271413dd735898dd895c693 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 5 May 2022 12:24:33 +0200 Subject: [PATCH 091/215] Update 450_check_bootloader_files.sh In layout/save/default/450_check_bootloader_files.sh use /[e]tc/grub*.cfg and /[b]oot/*/grub*.cfg with '*' globbing patterns (as in the EFI|GRUB2-EFI case) to find any of grub.cgf or grub2.cfg in /etc/ or in /boot/ (e.g. in openSUSE Leap 15.3 there is /boot/grub2/grub.cfg), cf. https://github.com/rear/rear/pull/2796#issuecomment-1118387393 --- .../rear/layout/save/default/450_check_bootloader_files.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/layout/save/default/450_check_bootloader_files.sh b/usr/share/rear/layout/save/default/450_check_bootloader_files.sh index 1324220ede..1827298959 100644 --- a/usr/share/rear/layout/save/default/450_check_bootloader_files.sh +++ b/usr/share/rear/layout/save/default/450_check_bootloader_files.sh @@ -17,7 +17,7 @@ case $used_bootloader in CHECK_CONFIG_FILES+=( /boot/efi/EFI/*/grub*.cfg ) ;; (GRUB|GRUB2) - CHECK_CONFIG_FILES+=( /[e]tc/grub.cfg /[e]tc/grub2.cfg /[b]oot/grub2/grub2.cfg /[b]oot/grub/grub.cfg ) + CHECK_CONFIG_FILES+=( /[e]tc/grub*.cfg /[b]oot/*/grub*.cfg ) ;; (LILO) CHECK_CONFIG_FILES+=( /[e]tc/lilo.conf ) @@ -30,7 +30,7 @@ case $used_bootloader in # - LILO : SLES < 12 # - YABOOT : RHEL < 7 # - GRUB2 : SLES >= 12, RHEL >= 7, Ubuntu and other new Linux on POWER distro. - CHECK_CONFIG_FILES+=( /[e]tc/lilo.conf /[e]tc/yaboot.conf /[e]tc/grub.cfg /[e]tc/grub2.cfg /[b]oot/grub2/grub2.cfg /[b]oot/grub/grub.cfg) + CHECK_CONFIG_FILES+=( /[e]tc/lilo.conf /[e]tc/yaboot.conf /[e]tc/grub*.cfg /[b]oot/*/grub*.cfg ) ;; (ARM|ARM-ALLWINNER) CHECK_CONFIG_FILES+=( /[b]oot/boot.scr ) From 58ddf3e094456df3c41f0b2089ee962d34836da6 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 5 May 2022 15:50:19 +0200 Subject: [PATCH 092/215] Update default.conf In default.conf added '[e]tc/crypttab' to FILES_TO_PATCH_PATTERNS, cf. https://github.com/rear/rear/pull/2795#discussion_r859670066 --- usr/share/rear/conf/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 3406e93b1d..fdbf489430 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -349,7 +349,7 @@ FILES_TO_PATCH_PATTERNS="[b]oot/{grub.conf,menu.lst,device.map} [e]tc/grub.* \ [e]tc/sysconfig/grub [e]tc/sysconfig/bootloader \ [e]tc/lilo.conf [e]tc/elilo.conf \ [e]tc/yaboot.conf \ - [e]tc/mtab [e]tc/fstab \ + [e]tc/mtab [e]tc/fstab [e]tc/crypttab \ [e]tc/mtools.conf \ [e]tc/smartd.conf [e]tc/sysconfig/smartmontools \ [e]tc/sysconfig/rawdevices \ From 3803132de6ac2bd31b56e1dc2db29e771703a3c1 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 14:18:42 +0200 Subject: [PATCH 093/215] Update 120_confirm_wipedisk_disks.sh In layout/recreate/default/120_confirm_wipedisk_disks.sh rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 plus some minor code simplifcation and more explanatory comments and user messages. --- .../default/120_confirm_wipedisk_disks.sh | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh b/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh index c400b79382..ac18ea95b5 100644 --- a/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh +++ b/usr/share/rear/layout/recreate/default/120_confirm_wipedisk_disks.sh @@ -62,15 +62,19 @@ else # but at this point here the devices in disklayout.conf are already migrated to what they are on the recovery system # so we can check disklayout.conf what the parent disk of the RAID device is on the current recovery system, # cf. the code in layout/prepare/GNU/Linux/120_include_raid_code.sh - local raid raiddevice options - read raid raiddevice options < <(grep "^raid $disk_to_be_wiped " "$LAYOUT_FILE") + local keyword raiddevice options + # The disklayout.conf keyword for a RAID array is 'raidarray' and $raiddevice is e.g. /dev/md127 + # and $options is a string that should contain a word like devices=/dev/sda,/dev/sdb,/dev/sdc + read keyword raiddevice options < <( grep "^raidarray $disk_to_be_wiped " "$LAYOUT_FILE" ) if ! test "$raiddevice" = "$disk_to_be_wiped" ; then - # Continue with the next disk_to_be_wiped when the current one is no RAID device: + # Continue with the next disk_to_be_wiped when the current one is no RAID device. + # We are in the 'else' clause of the outer 'if' so disk_to_be_wiped does not exist as block device: DebugPrint "Skipping $disk_to_be_wiped to be wiped ($disk_to_be_wiped does not exist as block device)" continue - else - DebugPrint "RAID device $raiddevice does not exist - trying to determine its parent disks" fi + # The current disk_to_be_wiped is a RAID device like /dev/md127 that does not (yet) exist + # as block device in the currently running ReaR recovery system: + DebugPrint "RAID device $raiddevice does not exist - trying to determine the parent disks of its component devices" local component_devices=() local option for option in $options ; do @@ -103,14 +107,13 @@ else # `-/dev/sdc1 /dev/sdc1 /dev/sdc part # There is no PKNAME for disks so we use KNAME (so the parent of a disk is the disk itself) # and we also use KNAME as fallback when lsblk does not support PKNAME and proceed bona fide - # (so we wipe only KNAME of a partition but not its parent disk when PKNAME is not supported) - # if parent_device is not one single word (valid device names are single words): + # (so we wipe only KNAME of a partition but not its parent disk when PKNAME is not supported): parent_device="$( lsblk -inpo PKNAME "$component_device" 2>/dev/null | awk NF | head -n1 )" test $parent_device || parent_device="$( lsblk -inpo KNAME "$component_device" 2>/dev/null | awk NF | head -n1 )" # Without quoting an empty parent_device would result plain "test -b" which would (falsely) succeed: if test -b "$parent_device" ; then # parent_device is usually a disk but in the KNAME fallback case it could be a partition: - DebugPrint "$parent_device is a parent of $raiddevice that should be wiped" + DebugPrint "$parent_device is a parent of component device $component_device of $raiddevice that should be wiped" # Write-protection for the disks in DISKS_TO_BE_WIPED (see above). # When parent_device is a partition the function write_protection_ids() in lib/write-protect-functions.sh # also tries to determine its parent disk if possible to check the disk device in DISKS_TO_BE_WIPED: @@ -127,7 +130,7 @@ else fi done if is_false $added_parent_device ; then - DebugPrint "Skipping RAID device $raiddevice to be wiped (no parent disk found for it)" + DebugPrint "Skipping RAID device $raiddevice to be wiped (no parent disk found for its component devices)" fi fi done From 8d8b36bb75854efb3b084c959ea4cb5b6645fb95 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 14:37:53 +0200 Subject: [PATCH 094/215] Update 220_verify_layout.sh In layout/recreate/default/220_verify_layout.sh rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 --- .../recreate/default/220_verify_layout.sh | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/usr/share/rear/layout/recreate/default/220_verify_layout.sh b/usr/share/rear/layout/recreate/default/220_verify_layout.sh index dc59cb1452..4c50e8ad1f 100644 --- a/usr/share/rear/layout/recreate/default/220_verify_layout.sh +++ b/usr/share/rear/layout/recreate/default/220_verify_layout.sh @@ -18,38 +18,38 @@ lsblk -nrpo NAME,KNAME,FSTYPE,UUID | grep '_raid_member' | while read name kname # Check recreated RAID device UUID: if test $uuid ; then # When there is a recreated RAID device with UUID - # we grep for 'raid' entries in disklayout.conf that contain this UUID and + # we grep for 'raidarray' entries in disklayout.conf that contain this UUID and # if found things are considered to be OK because UUIDs must be unique # so it cannot happen that a different device also has this UUID - # i.e. we omit to check if the found UUID matches this RAID device - # 'raid' entries in disklayout.conf contain RAID UUIDs like (excerpts) - # raid /dev/md127 uuid=8d05eb84:2de831d1:dfed54b2:ad592118 devices=/dev/sda,/dev/sdb + # i.e. we omit to check if the found UUID matches this RAID device. + # 'raidarray' entries in disklayout.conf contain RAID UUIDs like (excerpts) + # raidarray /dev/md127 uuid=8d05eb84:2de831d1:dfed54b2:ad592118 devices=/dev/sda,/dev/sdb # but lsblk shows that UUID as 8d05eb84-2de8-31d1-dfed-54b2ad592118 # so we make uuid_alnum_lowercase=8d05eb842de831d1dfed54b2ad592118 - # and make the 'raid' entries alphanumeric lowercase characters plus spaces and '=' characters - # raid devmd127 uuid=8d05eb842de831d1dfed54b2ad592118 devices=devsdadevsdb + # and make the 'raidarray' entries alphanumeric lowercase characters plus spaces and '=' characters + # raidarray devmd127 uuid=8d05eb842de831d1dfed54b2ad592118 devices=devsdadevsdb # where we can grep for "uuid=$uuid_alnum_lowercase": uuid_alnum_lowercase="$( echo "$uuid" | tr -d -c '[:alnum:]' | tr '[:upper:]' '[:lower:]' )" - if ! grep "^raid " $LAYOUT_FILE | tr -d -c '[:alnum:] =' | tr '[:upper:]' '[:lower:]' | grep "uuid=$uuid_alnum_lowercase" ; then + if ! grep "^raidarray " $LAYOUT_FILE | tr -d -c '[:alnum:] =' | tr '[:upper:]' '[:lower:]' | grep "uuid=$uuid_alnum_lowercase" ; then LogPrintError "RAID device $name ($kname) recreated with UUID $uuid that is not in $LAYOUT_FILE" fi else # When there is a recreated RAID device without UUID - # we grep for 'raid' entries in disklayout.conf that contain its NAME or KNAME - # and check if such 'raid' entries have a 'uuid=...' option set and + # we grep for 'raidarray' entries in disklayout.conf that contain its NAME or KNAME + # and check if such 'raidarray' entries have a 'uuid=...' option set and # if yes we assume the recreated RAID device was falsely recreated without UUID: - # 'raid' entries can contain RAID devices like - # raid /dev/md127 ... devices=/dev/sda,/dev/sdb ... uuid=... - # raid /dev/md127 ... uuid=... devices=/dev/sda,/dev/sdb + # 'raidarray' entries can contain RAID devices like + # raidarray /dev/md127 ... devices=/dev/sda,/dev/sdb ... uuid=... + # raidarray /dev/md127 ... uuid=... devices=/dev/sda,/dev/sdb # so we use end of word delimiter '\>' that matches space comma and end of line. # The first grep may find more than one line and the second grep may find more that one UUID. # Check RAID devices named NAME: - if grep "^raid .*$name\>" $LAYOUT_FILE | grep 'uuid=' ; then + if grep "^raidarray .*$name\>" $LAYOUT_FILE | grep 'uuid=' ; then LogPrintError "RAID device $name ($kname) recreated without UUID but there is a UUID for $name in $LAYOUT_FILE" fi # Check RAID devices named KNAME if different than NAME: if test "$kname" != "$name" ; then - if grep "^raid .*$kname\>" $LAYOUT_FILE | grep 'uuid=' ; then + if grep "^raidarray .*$kname\>" $LAYOUT_FILE | grep 'uuid=' ; then LogPrintError "RAID device $kname ($name) recreated without UUID but there is a UUID for $kname in $LAYOUT_FILE" fi fi From c8196f1d08d68d1c3f0f019f9916dcabc0d071d0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 15:10:08 +0200 Subject: [PATCH 095/215] Update 420_autoresize_last_partitions.sh In layout/prepare/default/420_autoresize_last_partitions.sh rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 --- .../default/420_autoresize_last_partitions.sh | 63 ++++++++++--------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/usr/share/rear/layout/prepare/default/420_autoresize_last_partitions.sh b/usr/share/rear/layout/prepare/default/420_autoresize_last_partitions.sh index 4a33168dc0..7acd3a675a 100644 --- a/usr/share/rear/layout/prepare/default/420_autoresize_last_partitions.sh +++ b/usr/share/rear/layout/prepare/default/420_autoresize_last_partitions.sh @@ -134,7 +134,7 @@ function autoresize_last_partition () { # because the partitions exist on the RAID device like # disk /dev/sda 12884901888 gpt # disk /dev/sdc 12884901888 gpt - # raid /dev/md127 level=raid1 raid-devices=2 devices=/dev/sda,/dev/sdc name=raid1sdab ... + # raidarray /dev/md127 level=raid1 raid-devices=2 devices=/dev/sda,/dev/sdc name=raid1sdab ... # part /dev/md127 10485760 1048576 rear-noname bios_grub /dev/md127p1 # part /dev/md127 12739067392 11534336 rear-noname none /dev/md127p2 # so this function can be called for all 'disk' entries @@ -566,16 +566,16 @@ while read layout_type disk_device old_disk_size disk_label junk ; do autoresize_last_partition $disk_device $old_disk_size $disk_label $new_disk_size $new_disk_block_size done < <( grep "^disk " "$LAYOUT_FILE" ) -# Autoresize the last partition for 'raid' entries in disklayout.conf +# Autoresize the last partition for 'raidarray' entries in disklayout.conf # -# Example 'raid' related entries in disklayout.conf (excerpts) for a RAID1 array: +# Example 'raidarray' related entries in disklayout.conf (excerpts) for a RAID1 array: # # # Format: disk # disk /dev/sda 12884901888 gpt # disk /dev/sdc 12884901888 gpt # -# # Format: raid /dev/ level= raid-devices= devices= ... -# raid /dev/md127 level=raid1 raid-devices=2 devices=/dev/sda,/dev/sdc ... +# # Format: raidarray /dev/ level= raid-devices= devices= ... +# raidarray /dev/md127 level=raid1 raid-devices=2 devices=/dev/sda,/dev/sdc ... # # Partitions on /dev/md127 # # Format: part /dev/ # part /dev/md127 10485760 1048576 rear-noname bios_grub /dev/md127p1 @@ -586,7 +586,7 @@ done < <( grep "^disk " "$LAYOUT_FILE" ) # # crypt /dev/mapper/cr_root /dev/md127p2 type=luks1 ... # -# Example 'raid' related entries in disklayout.conf (excerpts) for a RAID0 array +# Example 'raidarray' related entries in disklayout.conf (excerpts) for a RAID0 array # that consists of the partitions /dev/sda3 and /dev/sdb2 and the raw disk /dev/sdc # # # Format: disk @@ -600,8 +600,8 @@ done < <( grep "^disk " "$LAYOUT_FILE" ) # part /dev/sdb 1073741824 1048576 rear-noname swap /dev/sdb1 # part /dev/sdb 7504658432 1074790400 rear-noname raid /dev/sdb2 # -# # Format: raid /dev/ level= raid-devices= devices= ... -# raid /dev/md127 level=raid0 raid-devices=3 devices=/dev/sda3,/dev/sdb2,/dev/sdc ... +# # Format: raidarray /dev/ level= raid-devices= devices= ... +# raidarray /dev/md127 level=raid0 raid-devices=3 devices=/dev/sda3,/dev/sdb2,/dev/sdc ... # # Partitions on /dev/md127 # # Format: part /dev/ # part /dev/md127 11810635776 1572864 rear-noname none /dev/md127p1 @@ -615,15 +615,15 @@ done < <( grep "^disk " "$LAYOUT_FILE" ) # while read layout_type raid_device junk ; do if ! test "$raid_device" ; then - LogPrintError "Cannot autoresize RAID ('raid' entry without RAID device in $LAYOUT_FILE)" - # Continue with the next 'raid' entry in disklayout.conf + LogPrintError "Cannot autoresize RAID ('raidarray' entry without RAID device in $LAYOUT_FILE)" + # Continue with the next 'raidarray' entry in disklayout.conf continue fi message_prefix="Cannot autoresize RAID $raid_device" - # For each 'raid' entry get its raid_component_devs as a string + # For each 'raidarray' entry get its raid_component_devs as a string # cf. the code in layout/prepare/GNU/Linux/120_include_raid_code.sh - read layout_type raid_device raid_options < <(grep "^raid $raid_device " "$LAYOUT_FILE") + read layout_type raid_device raid_options < <(grep "^raidarray $raid_device " "$LAYOUT_FILE") for raid_option in $raid_options ; do case "$raid_option" in (level=*) @@ -639,8 +639,8 @@ while read layout_type raid_device junk ; do esac done if ! test "$raid_component_devs" ; then - LogPrintError "$message_prefix ('raid' entry without RAID component devices in $LAYOUT_FILE)" - # Continue with the next 'raid' entry in disklayout.conf + LogPrintError "$message_prefix ('raidarray' entry without RAID component devices in $LAYOUT_FILE)" + # Continue with the next 'raidarray' entry in disklayout.conf continue fi @@ -670,7 +670,7 @@ while read layout_type raid_device junk ; do disklayout_entry=( $( grep "^part .* $raid_component_dev\$" "$LAYOUT_FILE" ) ) if ! test "$disklayout_entry" ; then LogPrintError "$message_prefix (neither 'disk' nor 'part' entry found $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi # Get the disk where the partition is: @@ -678,7 +678,7 @@ while read layout_type raid_device junk ; do disklayout_entry=( $( grep "^disk $raid_component_dev_disk " "$LAYOUT_FILE" ) ) if ! test "$disklayout_entry" ; then LogPrintError "$message_prefix (no 'disk' found for 'part' entry $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi fi @@ -694,18 +694,18 @@ while read layout_type raid_device junk ; do sysfsname=$( get_sysfs_name $raid_component_dev_disk ) if ! test "$sysfsname" ; then LogPrintError "$message_prefix (get_sysfs_name failed $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi if ! test -d "/sys/block/$sysfsname" ; then LogPrintError "$message_prefix (no '/sys/block/$sysfsname' directory $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi new_disk_size=$( get_disk_size "$sysfsname" ) if ! is_positive_integer $new_disk_size ; then LogPrintError "$message_prefix (get_disk_size failed $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi (( new_disk_sizes_sum += new_disk_size )) @@ -716,7 +716,7 @@ while read layout_type raid_device junk ; do new_disk_block_size=$( get_block_size "$sysfsname" ) if ! is_positive_integer $new_disk_block_size ; then LogPrintError "$message_prefix (get_block_size failed $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi done @@ -727,7 +727,7 @@ while read layout_type raid_device junk ; do old_raid_device_size="${disklayout_entry[2]}" else LogPrintError "$message_prefix (no 'raiddisk' found $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue fi # The new RAID device size differs from the old RAID device size @@ -740,7 +740,7 @@ while read layout_type raid_device junk ; do # Autoresize the last partition on the RAID0 device like /dev/md127 # but not on each component device of the array like /dev/sda3 and /dev/sdb2 and /dev/sdc autoresize_last_partition $raid_device $old_raid_device_size $disk_label $new_raid_device_size $new_disk_block_size - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue ;; (raid1) @@ -761,7 +761,7 @@ while read layout_type raid_device junk ; do disklayout_entry=( $( grep "^part .* $raid_component_dev\$" "$LAYOUT_FILE" ) ) if ! test "$disklayout_entry" ; then LogPrintError "$message_prefix (neither 'disk' nor 'part' entry found $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi # Get the disk where the partition is: @@ -769,7 +769,7 @@ while read layout_type raid_device junk ; do disklayout_entry=( $( grep "^disk $raid_component_dev_disk " "$LAYOUT_FILE" ) ) if ! test "$disklayout_entry" ; then LogPrintError "$message_prefix (no 'disk' found for 'part' entry $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi fi @@ -784,18 +784,18 @@ while read layout_type raid_device junk ; do sysfsname=$( get_sysfs_name $raid_component_dev_disk ) if ! test "$sysfsname" ; then LogPrintError "$message_prefix (get_sysfs_name failed $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi if ! test -d "/sys/block/$sysfsname" ; then LogPrintError "$message_prefix (no '/sys/block/$sysfsname' directory $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi new_disk_size=$( get_disk_size "$sysfsname" ) if ! is_positive_integer $new_disk_size ; then LogPrintError "$message_prefix (get_disk_size failed $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi # Set the new smallest disk size and use its block size in the autoresize_last_partition() call: @@ -808,7 +808,7 @@ while read layout_type raid_device junk ; do new_disk_block_size=$( get_block_size "$sysfsname" ) if ! is_positive_integer $new_disk_block_size ; then LogPrintError "$message_prefix (get_block_size failed $message_suffix)" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue 2 fi fi @@ -824,23 +824,24 @@ while read layout_type raid_device junk ; do # while in practice a RAID0 array of thousands of disks probably will not work reliably: if ! test $old_smallest_size -lt $bash_int_max -a $new_smallest_size -lt $bash_int_max ; then LogPrintError "$message_prefix (no disk size found or size not less than 2^63 - 1)" + # Continue with the next 'raidarray' entry in disklayout.conf continue fi # Autoresize the last partition on the RAID1 device like /dev/md127 # but not on each component device of the array like /dev/sda and /dev/sdc autoresize_last_partition $raid_device $old_smallest_size $disk_label $new_smallest_size $new_disk_block_size - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue ;; (*) # Currently only RAID1 and RAID0 are supported for autoresize: LogPrintError "$message_prefix (autoresizing is not supported for RAID level '$raid_level')" - # Continue with the next 'raid' entry in disklayout.conf + # Continue with the next 'raidarray' entry in disklayout.conf continue ;; esac -done < <( grep "^raid " "$LAYOUT_FILE" ) +done < <( grep "^raidarray " "$LAYOUT_FILE" ) # Use the new LAYOUT_FILE.resized_last_partition with the resized partitions: mv "$disklayout_resized_last_partition" "$LAYOUT_FILE" From 6d94c9a8ba62201773e8814f44ac575f596c122a Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 15:48:06 +0200 Subject: [PATCH 096/215] Update 120_include_raid_code.sh In layout/prepare/GNU/Linux/120_include_raid_code.sh rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 In particular the create_raid() function name is replaced by create_raidarray() see the create_device() function in lib/layout-functions.sh that calls create_$type "$device" where $type is a disklayout.conf keyword The create_device() function is called in layout/prepare/default/540_generate_device_code.sh --- .../GNU/Linux/120_include_raid_code.sh | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/usr/share/rear/layout/prepare/GNU/Linux/120_include_raid_code.sh b/usr/share/rear/layout/prepare/GNU/Linux/120_include_raid_code.sh index d2db6ac4d9..137068178d 100644 --- a/usr/share/rear/layout/prepare/GNU/Linux/120_include_raid_code.sh +++ b/usr/share/rear/layout/prepare/GNU/Linux/120_include_raid_code.sh @@ -10,11 +10,11 @@ has_binary mdadm || return 0 # ARRAY /dev/md/imsm0 metadata=imsm UUID=4d5cf215:80024c95:e19fdff4:2fba35a8 # ARRAY /dev/md/Volume0_0 container=/dev/md/imsm0 member=0 UUID=ffb80762:127807b3:3d7e4f4d:4532022f # -# For the 'mdadm --detail --scan --config=partitions' examples above 'raid' entries in disklayout.conf look like -# raid /dev/md127 level=raid1 raid-devices=2 devices=/dev/sda,/dev/sdb name=raid1sdab metadata=1.0 uuid=8d05eb84:2de831d1:dfed54b2:ad592118 +# For the 'mdadm --detail --scan --config=partitions' examples above 'raidarray' entries in disklayout.conf look like +# raidarray /dev/md127 level=raid1 raid-devices=2 devices=/dev/sda,/dev/sdb name=raid1sdab metadata=1.0 uuid=8d05eb84:2de831d1:dfed54b2:ad592118 # for a RAID1 array and for a RAID CONTAINER with IMSM metadata it looks like -# raid /dev/md127 metadata=imsm level=container raid-devices=2 uuid=4d5cf215:80024c95:e19fdff4:2fba35a8 name=imsm0 devices=/dev/sda,/dev/sdb -# raid /dev/md126 level=raid1 raid-devices=2 uuid=ffb80762:127807b3:3d7e4f4d:4532022f name=Volume0_0 devices=/dev/md127 size=390706176 +# raidarray /dev/md127 metadata=imsm level=container raid-devices=2 uuid=4d5cf215:80024c95:e19fdff4:2fba35a8 name=imsm0 devices=/dev/sda,/dev/sdb +# raidarray /dev/md126 level=raid1 raid-devices=2 uuid=ffb80762:127807b3:3d7e4f4d:4532022f name=Volume0_0 devices=/dev/md127 size=390706176 # cf. layout/save/GNU/Linux/210_raid_layout.sh # and the matching 'part' entries in disklayout.conf look like # part /dev/md127 10485760 1048576 rear-noname bios_grub /dev/md127p1 @@ -25,16 +25,18 @@ has_binary mdadm || return 0 # part /dev/md126 398378139648 1703936000 md126p3 lvm /dev/md126p3 # cf. https://github.com/rear/rear/pull/2702#issuecomment-968904230 -# List of raid devices for which create_raid was already done -# i.e. those raid devices for which there is already code in diskrestore.sh -# but those raid devices are not yet created (they are created after diskrestore.sh was run). +# List of RAID devices for which create_raid was already done +# i.e. those RAID devices for which there is already code in diskrestore.sh +# but those RAID devices are not yet created (they are created after diskrestore.sh was run). # This global variable is set and used in each call of create_raid(). # This global variable is initialized here only once when this script is run: CREATE_RAID_DEVICES_CODE=() -create_raid() { - local raid raiddevice options - read raid raiddevice options < <(grep "^raid $1 " "$LAYOUT_FILE") +create_raidarray() { + local keyword raiddevice options + # The disklayout.conf keyword for a RAID array is 'raidarray' and $raiddevice is e.g. /dev/md127 + # and $options is a string that should contain a word like devices=/dev/sda,/dev/sdb,/dev/sdc + read keyword raiddevice options < <(grep "^raidarray $1 " "$LAYOUT_FILE") local mdadmcmd="mdadm --create $raiddevice --force" @@ -64,7 +66,7 @@ create_raid() { local component_devices_count=${#component_devices[@]} local missing_devices_count=$(( raid_devices - component_devices_count )) if test $missing_devices_count -gt 0 ; then - # Don't consider raid inside a container (it's expected to have 1 device only: the container) + # Don't consider RAID inside a container (it's expected to have 1 device only: the container) if [ $component_devices_count -ne 1 ] || ! IsInArray ${component_devices[0]} "${CREATE_RAID_DEVICES_CODE[@]}" ; then LogPrint "Software RAID $raiddevice has not enough component devices, adding $missing_devices_count 'missing' devices" # Print as many 'missing' as there are missing devices From 0794522adf1e71a6d25a3b952ce6fd906ea7b0f7 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 16:09:44 +0200 Subject: [PATCH 097/215] Update 210_raid_layout.sh In layout/save/GNU/Linux/210_raid_layout.sh rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 --- .../layout/save/GNU/Linux/210_raid_layout.sh | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh index 7dafac4e1b..bb1e6cfb19 100644 --- a/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/210_raid_layout.sh @@ -41,13 +41,14 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d # Skip if it is not an "ARRAY": test "$array" = "ARRAY" || continue - # 'raid' entries in disklayout.conf look like (cf. the 'mdadm --detail --scan --config=partitions' examples above) - # raid /dev/md127 metadata=1.0 level=raid1 raid-devices=2 uuid=8d05eb84:2de831d1:dfed54b2:ad592118 name=raid1sdab devices=/dev/sda,/dev/sdb + # 'raidarray' entries in disklayout.conf look like (cf. the 'mdadm --detail --scan --config=partitions' examples above) + # raidarray /dev/md127 metadata=1.0 level=raid1 raid-devices=2 uuid=8d05eb84:2de831d1:dfed54b2:ad592118 name=raid1sdab devices=/dev/sda,/dev/sdb # for a RAID1 array and for a RAID CONTAINER with IMSM metadata it looks like - # raid /dev/md127 metadata=imsm level=container raid-devices=2 uuid=4d5cf215:80024c95:e19fdff4:2fba35a8 name=imsm0 devices=/dev/sda,/dev/sdb - # raid /dev/md126 level=raid1 raid-devices=2 uuid=ffb80762:127807b3:3d7e4f4d:4532022f name=Volume0_0 devices=/dev/md127 size=390706176 + # raidarray /dev/md127 metadata=imsm level=container raid-devices=2 uuid=4d5cf215:80024c95:e19fdff4:2fba35a8 name=imsm0 devices=/dev/sda,/dev/sdb + # raidarray /dev/md126 level=raid1 raid-devices=2 uuid=ffb80762:127807b3:3d7e4f4d:4532022f name=Volume0_0 devices=/dev/md127 size=390706176 # cf. https://github.com/rear/rear/pull/2702#issuecomment-968904230 - raid_layout_entry="raid" + # Each 'raidarray' entry in disklayout.conf starts with the keyword 'raidarray': + raid_layout_entry="raidarray" # Do not use an array name from a previous run of the while loop: name="" @@ -126,12 +127,12 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d # doc/user-guide/06-layout-configuration.adoc reads # Disk layout file syntax # Software RAID - # raid /dev/ level= raid-devices= [uuid=] [spare-devices=] [layout=] [chunk=] devices= + # raidarray /dev/ level= raid-devices= devices= [name=] [metadata=] [uuid=] [layout=] [chunk=] [spare-devices=] [size=] # so the mdadm options --level --raid-devices and the component-devices are mandatory: line=( $( grep "Raid Level :" $mdadm_details ) ) level=${line[3]} - # A RAID level that is more than one word would make 'read' fail for this 'raid' entry in disklayout.conf + # A RAID level that is more than one word would make 'read' fail for this 'raidarray' entry in disklayout.conf test $level || Error "RAID $raiddevice level '$level' is not a single word" raid_layout_entry+=" level=$level" @@ -167,7 +168,7 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d for component_device in $( grep -o '/dev/.*' $mdadm_details | grep -v '/dev/md' | tr "\n" " " ) ; do component_device=$( get_device_name $component_device ) test -b "$component_device" || Error "RAID $raiddevice component device '$component_device' is no block device" - # A component device that is more than one word would make 'read' fail for this 'raid' entry in disklayout.conf + # A component device that is more than one word would make 'read' fail for this 'raidarray' entry in disklayout.conf test $component_device || Error "RAID $raiddevice component device '$component_device' is not a single word" # Have the component devices string as "first_component_device,second_component_device,..." test $component_devices && component_devices+=",$component_device" || component_devices="$component_device" @@ -297,7 +298,7 @@ mdadm --detail --scan --config=partitions | while read array raiddevice junk ; d test $container_size -gt 0 && raid_layout_entry+=" size=$container_size" echo "# RAID device $raiddevice" >>$DISKLAYOUT_FILE - echo "# Format: raid /dev/ level= raid-devices= devices= [name=] [metadata=] [uuid=] [layout=] [chunk=] [spare-devices=] [size=]" >>$DISKLAYOUT_FILE + echo "# Format: raidarray /dev/ level= raid-devices= devices= [name=] [metadata=] [uuid=] [layout=] [chunk=] [spare-devices=] [size=]" >>$DISKLAYOUT_FILE echo "$raid_layout_entry" >>$DISKLAYOUT_FILE # cf. the code in layout/save/GNU/Linux/200_partition_layout.sh @@ -324,11 +325,11 @@ done mdadm_exit_code=${PIPESTATUS[0]} test $mdadm_exit_code -eq 0 || Error "'mdadm --detail --scan --config=partitions' failed with exit code $mdadm_exit_code" -# mdadm is required in the recovery system if disklayout.conf contains at least one 'raid' entry -# see the create_raid function in layout/prepare/GNU/Linux/120_include_raid_code.sh +# mdadm is required in the recovery system if disklayout.conf contains at least one 'raidarray' entry +# see the create_raidarray function in layout/prepare/GNU/Linux/120_include_raid_code.sh # what program calls are written to diskrestore.sh # cf. https://github.com/rear/rear/issues/1963 -if grep -q '^raid ' $DISKLAYOUT_FILE ; then +if grep -q '^raidarray ' $DISKLAYOUT_FILE ; then REQUIRED_PROGS+=( mdadm ) # mdmon was added via https://github.com/rear/rear/pull/2702 PROGS+=( mdmon ) From 89d99822a7a0d0a24ebe76ec8c3ebf8f7fcd9aa9 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 16:16:24 +0200 Subject: [PATCH 098/215] Update layout-functions.sh In lib/layout-functions.sh rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 --- usr/share/rear/lib/layout-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/lib/layout-functions.sh b/usr/share/rear/lib/layout-functions.sh index 0c633663e0..cb33ac28a2 100644 --- a/usr/share/rear/lib/layout-functions.sh +++ b/usr/share/rear/lib/layout-functions.sh @@ -162,13 +162,13 @@ generate_layout_dependencies() { [ -z "$pool" ] || add_dependency "$dm_prefix-$dm_lvol" "$dm_prefix-$dm_pool" add_component "$dm_prefix-$dm_lvol" "lvmvol" ;; - raid) + raidarray) name=$(echo "$remainder" | cut -d " " -f "1") disks=$(echo "$remainder" | sed -r "s/.*devices=([^ ]+).*/\1/" | tr ',' ' ') for disk in $disks ; do add_dependency "$name" "$disk" done - add_component "$name" "raid" + add_component "$name" "raidarray" ;; fs|btrfsmountedsubvol) dev=$(echo "$remainder" | cut -d " " -f "1") From 4aa743b2ab1ac84fc887f491270665cf5e41045a Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 16:30:48 +0200 Subject: [PATCH 099/215] Update 06-layout-configuration.adoc In doc/user-guide/06-layout-configuration.adoc rename disklayout.conf keyword 'raid' into 'raidarray' see https://github.com/rear/rear/issues/2759 --- doc/user-guide/06-layout-configuration.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user-guide/06-layout-configuration.adoc b/doc/user-guide/06-layout-configuration.adoc index c1660b299c..9baf567b7d 100644 --- a/doc/user-guide/06-layout-configuration.adoc +++ b/doc/user-guide/06-layout-configuration.adoc @@ -620,7 +620,7 @@ For some component keywords the parameters have a form like keyword value1 value2 optionA=valueA optionB=valueB ... ---------------------------------- where the first ones are positional parameters but not the option=value parameters -so empty option=value parameters are allowed, for example see the 'raid' keyword. +so empty option=value parameters are allowed, for example see the 'raidarray' keyword. For details the matching scripts need to be inspected how things actually work for a particular component keyword (i.e. what is implemented in the code). @@ -646,9 +646,9 @@ disk part ---------------------------------- -=== Software RAID === +=== Software RAID arrays === ---------------------------------- -raid /dev/ level= raid-devices= devices= [name=] [metadata=] [uuid=] [layout=] [chunk=] [spare-devices=] [size=] +raidarray /dev/ level= raid-devices= devices= [name=] [metadata=] [uuid=] [layout=] [chunk=] [spare-devices=] [size=] ---------------------------------- === Multipath === From 0d1f893dc06e7aaefca3cde647446595d66c1165 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 6 May 2022 16:36:35 +0200 Subject: [PATCH 100/215] Update 06-layout-configuration.adoc In doc/user-guide/06-layout-configuration.adoc add the new disklayout.conf keyword 'raiddisk' see https://github.com/rear/rear/issues/2759 --- doc/user-guide/06-layout-configuration.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user-guide/06-layout-configuration.adoc b/doc/user-guide/06-layout-configuration.adoc index 9baf567b7d..0be8960f3c 100644 --- a/doc/user-guide/06-layout-configuration.adoc +++ b/doc/user-guide/06-layout-configuration.adoc @@ -651,6 +651,11 @@ part level= raid-devices= devices= [name=] [metadata=] [uuid=] [layout=] [chunk=] [spare-devices=] [size=] ---------------------------------- +=== Software RAID disks === +---------------------------------- +raiddisk +---------------------------------- + === Multipath === ---------------------------------- multipath /dev/ From 825478ee27f916553938afaf5164fec22cb32732 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 10 May 2022 10:43:39 +0200 Subject: [PATCH 101/215] Update 200_partition_layout.sh In layout/save/GNU/Linux/200_partition_layout.sh ensure $disk_label is one of the supported partition tables, cf. https://github.com/rear/rear/issues/2801#issuecomment-1122015129 --- usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh index b747d121c6..6e641d280e 100644 --- a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh @@ -115,6 +115,11 @@ extract_partitions() { parted -s $device print > $TMP_DIR/parted disk_label=$(grep -E "Partition Table|Disk label" $TMP_DIR/parted | cut -d ":" -f "2" | tr -d " ") fi + # Ensure $disk_label is valid to determine the partition name/type in the next step at 'declare type' + # cf. https://github.com/rear/rear/issues/2801#issuecomment-1122015129 + if ! [[ "$disk_label" = "msdos" || "$disk_label" = "gpt" || "$disk_label" = "gpt_sync_mbr" || "$disk_label" = "dasd" ]] ; then + Error "Unsupported partition table '$disk_label' (must be one of 'msdos' 'gpt' 'gpt_sync_mbr' 'dasd')" + fi cp $TMP_DIR/partitions $TMP_DIR/partitions-data From 5ba259b6a1e78f34209c160e8c1be8ac7e18de58 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 10 May 2022 12:29:53 +0200 Subject: [PATCH 102/215] Ensure syntactically correct 'disk' and 'part' entries in disklayout.conf In layout/save/GNU/Linux/200_partition_layout.sh ensure syntactically correct 'disk' and 'part' entries, cf. https://github.com/rear/rear/issues/2801#issuecomment-1122015129 --- .../save/GNU/Linux/200_partition_layout.sh | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh index b747d121c6..ffcbad0904 100644 --- a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh @@ -341,13 +341,23 @@ extract_partitions() { ### Write to layout file while read partition_nr size start type flags junk ; do - ### determine the name of the partition using the number - ### device=/dev/cciss/c0d0 ; partition_prefix=cciss/c0d0p - ### device=/dev/md127 ; partition_prefix=md127p - ### device=/dev/sda ; partition_prefix=sda - ### device=/dev/mapper/mpathbp1 ; partition_prefix=mpathbp + # determine the name of the partition using the number + # device=/dev/cciss/c0d0 ; partition_prefix=cciss/c0d0p + # device=/dev/md127 ; partition_prefix=md127p + # device=/dev/sda ; partition_prefix=sda + # device=/dev/mapper/mpathbp1 ; partition_prefix=mpathbp partition_name="${device%/*}/${partition_prefix#*/}$partition_nr" - echo "part $device $size $start $type $flags $(get_device_name $partition_name)" + partition_device="$( get_device_name $partition_name )" + test -b "$partition_device" || Error "Invalid 'part $device' entry (partition device '$partition_device' is no block device)" + # Ensure syntactically correct 'part' entries of the form + # part disk_device partition_size start_byte partition_label flags partition_device + # Each value must exist and each value must be a single non-blank word. + # When $junk contains something one of the values before was more than a single word: + test "$junk" && Error "Invalid 'part $device' entry (some value is more than a single word)" + # When $flags is empty at least one value is missing: + test "$flags" || Error "Invalid 'part $device' entry (at least one value is missing)" + # Some basic checks on the values happen in layout/save/default/950_verify_disklayout_file.sh + echo "part $device $size $start $type $flags $partition_device" done < $TMP_DIR/partitions } @@ -383,6 +393,11 @@ Log "Saving disks and their partitions" devname=$(get_device_name $disk) devsize=$(get_disk_size ${disk#/sys/block/}) disktype=$(parted -s $devname print | grep -E "Partition Table|Disk label" | cut -d ":" -f "2" | tr -d " ") + # Ensure syntactically correct 'disk' entries: + # Each value must exist and each value must be a single non-blank word so we 'test' without quoting the value: + test $devname || Error "Invalid 'disk' entry (no disk device name for '$disk')" + test $devsize || Error "Invalid 'disk $devname' entry (no device size for '$devname')" + test $disktype || Error "Invalid 'disk $devname' entry (no partition table type for '$devname')" echo "# Disk $devname" echo "# Format: disk " From 26124c940b5c7bf3c2be25d2cd96cb74baf0e73d Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 10 May 2022 12:44:20 +0200 Subject: [PATCH 103/215] Update 06-layout-configuration.adoc In doc/user-guide/06-layout-configuration.adoc tell that the proper command (in particular for scripts) is grep "^keyword " /var/lib/rear/layout/disklayout.conf with a single space after the keyword as delimiter, cf. https://github.com/rear/rear/pull/2800#issuecomment-1121320742 --- doc/user-guide/06-layout-configuration.adoc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/user-guide/06-layout-configuration.adoc b/doc/user-guide/06-layout-configuration.adoc index 0be8960f3c..65c64e3957 100644 --- a/doc/user-guide/06-layout-configuration.adoc +++ b/doc/user-guide/06-layout-configuration.adoc @@ -604,11 +604,16 @@ Lines that start with a # (number sign, hash, or pound sign) are comments. All other lines start with a component keyword. None of the component keywords is a leading substring of another component keyword -(e.g. disk is not a leading substring of opaldisk) so that one can get the lines -that belong to a particular component via simple commands like +(e.g. disk is not a leading substring of opaldisk) so that one could even get +only those lines that belong to a particular component via sloppy commands like ---------------------------------- grep ^keyword /var/lib/rear/layout/disklayout.conf ---------------------------------- +regardless that the proper command (in particular for scripts) is +---------------------------------- +grep "^keyword " /var/lib/rear/layout/disklayout.conf +---------------------------------- +with a single space after the keyword as delimiter. For most component keywords the values are positional parameters (if there is no value a dummy value like 'none' must be used) From 1ffd9285b3470b2498c99d95cb1533d8006a60e5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 12 May 2022 12:04:58 +0200 Subject: [PATCH 104/215] Update 200_partition_layout.sh Added comment why layout/save/GNU/Linux/200_partition_layout.sh must error out when a partition type is reported by parted that is not in the supported list cf. https://github.com/rear/rear/pull/2803#issuecomment-1124800884 --- usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh index 6e641d280e..12fbfd36af 100644 --- a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh @@ -117,6 +117,10 @@ extract_partitions() { fi # Ensure $disk_label is valid to determine the partition name/type in the next step at 'declare type' # cf. https://github.com/rear/rear/issues/2801#issuecomment-1122015129 + # When a partition type is reported by parted that is not in the supported list + # it must error out because our current code here in 200_partition_layout.sh + # does not work with partition types that are not in the supported list + # cf. https://github.com/rear/rear/pull/2803#issuecomment-1124800884 if ! [[ "$disk_label" = "msdos" || "$disk_label" = "gpt" || "$disk_label" = "gpt_sync_mbr" || "$disk_label" = "dasd" ]] ; then Error "Unsupported partition table '$disk_label' (must be one of 'msdos' 'gpt' 'gpt_sync_mbr' 'dasd')" fi From 048db085c1cb0a81186d9675b4126d92223483ab Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 16 May 2022 11:59:25 +0200 Subject: [PATCH 105/215] Update 400_copy_modules.sh In build/GNU/Linux/400_copy_modules.sh have the grep search value to find kernel builtin modules with a leading '/' and a trailing '.' to avoid false substring matches of wrong kernel builtin modules that would falsely skip non-builtin modules from being included cf. https://github.com/rear/rear/pull/2728#issuecomment-995799489 and https://github.com/rear/rear/pull/2728#issuecomment-996103272 --- usr/share/rear/build/GNU/Linux/400_copy_modules.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/build/GNU/Linux/400_copy_modules.sh b/usr/share/rear/build/GNU/Linux/400_copy_modules.sh index fbcbf27081..5d093e8af1 100644 --- a/usr/share/rear/build/GNU/Linux/400_copy_modules.sh +++ b/usr/share/rear/build/GNU/Linux/400_copy_modules.sh @@ -173,9 +173,14 @@ for dummy in "once" ; do modinfo $module 1>/dev/null || continue # Continue with the next module if the current one is a kernel builtin module # cf. https://github.com/rear/rear/issues/2414#issuecomment-668632798 + # and have the grep search value with a leading '/' and a trailing '.' + # to avoid false substring matches of wrong kernel builtin modules + # that would falsely skip non-builtin modules from being included + # cf. https://github.com/rear/rear/pull/2728#issuecomment-995799489 + # and https://github.com/rear/rear/pull/2728#issuecomment-996103272 # Quoting the grep search value is mandatory here ($module might be empty or blank), # cf. "Beware of the emptiness" in https://github.com/rear/rear/wiki/Coding-Style - grep -q "$( echo $module | tr '_-' '..' )" /lib/modules/$KERNEL_VERSION/modules.builtin && continue + grep -q "/$( echo $module | tr '_-' '..' )\." /lib/modules/$KERNEL_VERSION/modules.builtin && continue # Resolve module dependencies: # Get the module file plus the module files of other needed modules. # This is currently only a "best effort" attempt because From 8b996de907477228904cfc097a854f60f5565e0b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 16 May 2022 12:43:29 +0200 Subject: [PATCH 106/215] Update default.conf In default.conf tell that MODULES=( 'loaded_modules' 'additional_module' ) is not supported, cf. https://github.com/rear/rear/issues/2727 --- usr/share/rear/conf/default.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index fdbf489430..2bb5a34690 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1529,6 +1529,7 @@ LIBS=() # (see https://github.com/rear/rear/issues/1355). # The setting MODULES=( 'loaded_modules' ) results that # only those kernel modules that are currently loaded +# (so MODULES=( 'loaded_modules' 'additional_module' ) is not supported) # get included in the recovery system. This results a noticeable # smaller recovery system but on the other hand it means that # recovery will only work on same replacement hardware. From 3b08173bb9f915512dfae269b5f47a2144b5d5dc Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 16 May 2022 14:20:25 +0200 Subject: [PATCH 107/215] Update 220_lvm_layout.sh In layout/save/GNU/Linux/220_lvm_layout.sh removed useless empty line in a comment block. --- usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh b/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh index 278ffe2422..e01dbf4651 100644 --- a/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/220_lvm_layout.sh @@ -203,7 +203,6 @@ local lvs_exit_code # Example output of "lvs --separator=':' --noheadings --units b --nosuffix -o $lvs_fields" # with lvs_fields="origin,lv_name,vg_name,lv_size,lv_layout,pool_lv,chunk_size,stripes,stripe_size,seg_size" # i.e. when the 'lv_layout' field is supported: - # :home:system:6148849664:linear::0:1:0:6148849664 # :root:system:14050918400:linear::0:1:0:14050918400 # :swap:system:1262485504:linear::0:1:0:1262485504 From 339cff93232460fc5098b06c666c925d88459360 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 16 May 2022 15:18:18 +0200 Subject: [PATCH 108/215] Update bootloader-functions.sh In the function find_syslinux_modules_dir in lib/bootloader-functions.sh the fallback 'find /usr' to find the SYSLINUX modules directory may take a very long time on some systems (up to several hours) so tell the user in debug mode what is going on, see https://github.com/rear/rear/issues/2792 and https://github.com/rear/rear/issues/624 --- usr/share/rear/lib/bootloader-functions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/share/rear/lib/bootloader-functions.sh b/usr/share/rear/lib/bootloader-functions.sh index 435c15ea6e..a8ee92a900 100644 --- a/usr/share/rear/lib/bootloader-functions.sh +++ b/usr/share/rear/lib/bootloader-functions.sh @@ -56,6 +56,10 @@ function find_syslinux_modules_dir { # not default location? try to find it # file=/usr/lib/syslinux/modules/efi32/menu.c32 # f23: file=/usr/share/syslinux/menu.c32 + # Because 'find /usr' may take very long time on some systems + # cf. https://github.com/rear/rear/issues/2792 + # tell the user in debug mode what is going on + DebugPrint "Searching whole /usr for SYSLINUX modules directory (you may specify SYSLINUX_MODULES_DIR)" file=$( find /usr -name "$1" 2>/dev/null | tail -1 ) syslinux_modules_dir=$( dirname "$file" ) # /usr/lib/syslinux/modules/efi32 syslinux_modules_dir=${syslinux_modules_dir%/*} # /usr/lib/syslinux/modules From f8f6c1b94a20a0dfa3b81aa0d23a9abcffa5b356 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 17 May 2022 14:20:02 +0200 Subject: [PATCH 109/215] Update default.conf In default.conf explain how to use LIBS properly, see https://github.com/rear/rear/issues/2743 --- usr/share/rear/conf/default.conf | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 2bb5a34690..b769f45765 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1484,7 +1484,27 @@ uniq wc ) -# Library files to include in the rescue/recovery system: +# Special library files to include in the rescue/recovery system: +# Needed programs in the recovery system should be specified via +# PROGS+=( non_mandatory_program ) +# REQUIRED_PROGS+=( mandatory_program ) +# Normally there is no need to specify LIBS for programs +# because needed libraries for "normally" linked programs +# get automatically included via the RequiredSharedObjects function +# which calls ldd to determine the required shared objects +# of the elements in the PROGS and REQUIRED_PROGS arrays +# (cf. COPY_AS_IS and TRUSTED_FILE_OWNERS below). +# Therefore only libraries that are not reported by ldd +# (e.g. when libraries are loaded via dlopen by a program) +# must be specified in LIBS. +# RequiredSharedObjects() is also called for the elements in LIBS +# but it does not descend recursively into (sub)-directories +# so the elements in LIBS must evaluate to actual library files +# (by bash pathname expansion with 'nullglob' that is set in ReaR). +# For example instead of using only LIBS+=( /path/to/libdir ) +# specify LIBS+=( /path/to/libdir/*.so* /path/to/libdir/*/*.so* ) +# for library files in /path/to/libdir and in its direct sub-directories +# cf. https://github.com/rear/rear/issues/2743 LIBS=() # Kernel modules to include in the rescue/recovery system: From b03510ad24785461d19401e782f4836afa0898aa Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 17 May 2022 14:50:04 +0200 Subject: [PATCH 110/215] Exclude dev/watchdog* from recovery system In default.conf add dev/watchdog\* to COPY_AS_IS_EXCLUDE because /dev/watchdog /dev/watchdog\* functionality is not wanted in the ReaR rescue/recovery system because we do not want any automated reboot while disaster recovery happens via "rear recover". Furthermore having dev/watchdog* during "rear mkrescue" may even trigger a system "crash" that is caused by TrendMicro ds_am module touching dev/watchdog in ReaR's build area (/var/tmp/rear.XXX/rootfs). See https://github.com/rear/rear/issues/2798 --- usr/share/rear/conf/default.conf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index b769f45765..5eba7cd6af 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1653,7 +1653,12 @@ COPY_AS_IS=( $SHARE_DIR $VAR_DIR ) # We let them being recreated by device mapper in the recovery system during the recovery process. # Copying them into the recovery system would let "rear recover" avoid the migration process. # See https://github.com/rear/rear/pull/1393 for details. -COPY_AS_IS_EXCLUDE=( $VAR_DIR/output/\* dev/.udev dev/shm dev/shm/\* dev/oracleasm dev/mapper ) +# /dev/watchdog /dev/watchdog\* functionality is not wanted in the ReaR rescue/recovery system +# because we do not want any automated reboot while disaster recovery happens via "rear recover". +# Furthermore having dev/watchdog* during "rear mkrescue" may even trigger a system "crash" that is +# caused by TrendMicro ds_am module touching dev/watchdog in ReaR's build area (/var/tmp/rear.XXX/rootfs). +# See https://github.com/rear/rear/issues/2798 +COPY_AS_IS_EXCLUDE=( $VAR_DIR/output/\* dev/.udev dev/shm dev/shm/\* dev/oracleasm dev/mapper dev/watchdog\* ) # Array of user names that are trusted owners of files where RequiredSharedObjects calls ldd (cf. COPY_AS_IS) # and where a ldd test is run inside the recovery system that tests all binaries for 'not found' libraries. # The default is 'root' plus those standard system users that have a 'bin' or 'sbin' or 'root' home directory From b651c1ee14dbae1cb7f275dd4af9a6006b302f7b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 23 May 2022 15:12:26 +0200 Subject: [PATCH 111/215] Add PRE_RECOVERY_COMMANDS and POST_RECOVERY_COMMANDS This is basically same as https://github.com/rear/rear/pull/2735 but without PRE_BACKUP_COMMANDS and POST_BACKUP_COMMANDS for the reasoning see https://github.com/rear/rear/pull/2735#issuecomment-1115833664 --- usr/share/rear/conf/default.conf | 35 ++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index b769f45765..f208f3eba9 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -3458,15 +3458,38 @@ ELILO_BIN= ################ ---- custom scripts # # NOTE: The scripts can be defined as an array to better handly spaces in parameters. -# The scripts are called like this: eval "${PRE_RECOVERY_SCRIPT[@]}" - -# Call this after Relax-and-Recover did everything in the recover workflow. -# Use $TARGET_FS_ROOT (by default '/mnt/local') to refer to the recovered system. -POST_RECOVERY_SCRIPT= +# The scripts are called like this: +# eval "${PRE_RECOVERY_SCRIPT[@]}" +# +# Alternatively, commands can be executed by using the corresponding +# PRE_RECOVERY_COMMANDS and POST_RECOVERY_COMMANDS array variables +# which evaluate like this: +# for command in "${PRE_RECOVERY_COMMANDS[@]}" ; do +# eval "$command" +# done +# +# Using PRE_RECOVERY_COMMANDS and POST_RECOVERY_COMMANDS +# is simpler when multiple commands should be executed. +# For example, +# PRE_RECOVERY_SCRIPT=( 'echo Hello' ';' 'sleep 3' ) +# can be rewritten as +# PRE_RECOVERY_COMMANDS=( 'echo Hello' 'sleep 3' ) +# or +# PRE_RECOVERY_COMMANDS=( 'echo Hello' ) +# PRE_RECOVERY_COMMANDS+=( 'sleep 3' ) -# Call this before Relax-and-Recover starts to do anything in the recover workflow. You have the rescue system but nothing else +# Those get called at the very beginning of "rear recover". +# The PRE_RECOVERY_COMMANDS are called directly before the PRE_RECOVERY_SCRIPT. +# Nothing was recreated and you have only the plain ReaR rescue/recovery system: +PRE_RECOVERY_COMMANDS=() PRE_RECOVERY_SCRIPT= +# Those get called at the very end of "rear recover". +# The POST_RECOVERY_COMMANDS are called directly after the POST_RECOVERY_SCRIPT. +# Use $TARGET_FS_ROOT (by default '/mnt/local') to access the recreated target system. +POST_RECOVERY_SCRIPT= +POST_RECOVERY_COMMANDS=() + # PRE/POST Backup scripts will provide the ability to run certain tasks before and after a ReaR backup. # for example: # If a small database running on local filesystem and dependent on a local service, you will maintain its data consistency. From f3700b94d8d2186dd181a5d68b4de6c785cf4722 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 23 May 2022 15:21:41 +0200 Subject: [PATCH 112/215] Update 010_pre_recovery_script.sh In usr/share/rear/setup/default/010_pre_recovery_script.sh call PRE_RECOVERY_COMMANDS before PRE_RECOVERY_SCRIPT so PRE_RECOVERY_COMMANDS can also be used to prepare things for PRE_RECOVERY_SCRIPT --- .../rear/setup/default/010_pre_recovery_script.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/setup/default/010_pre_recovery_script.sh b/usr/share/rear/setup/default/010_pre_recovery_script.sh index 005107cc47..dbd186b187 100644 --- a/usr/share/rear/setup/default/010_pre_recovery_script.sh +++ b/usr/share/rear/setup/default/010_pre_recovery_script.sh @@ -1,4 +1,14 @@ + +# The PRE_RECOVERY_COMMANDS are called directly before the PRE_RECOVERY_SCRIPT +# so PRE_RECOVERY_COMMANDS can also be used to prepare things for the PRE_RECOVERY_SCRIPT: + +local command +for command in "${PRE_RECOVERY_COMMANDS[@]}" ; do + Log "Running PRE_RECOVERY_COMMANDS '$command'" + eval "$command" +done + if test "$PRE_RECOVERY_SCRIPT" ; then - Log "Running PRE_RECOVERY_SCRIPT '${PRE_RECOVERY_SCRIPT[@]}'" - eval "${PRE_RECOVERY_SCRIPT[@]}" + Log "Running PRE_RECOVERY_SCRIPT '${PRE_RECOVERY_SCRIPT[@]}'" + eval "${PRE_RECOVERY_SCRIPT[@]}" fi From ae0ab2c343cf9572cc15ffcde4147379016c3972 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 23 May 2022 15:25:14 +0200 Subject: [PATCH 113/215] Update 500_post_recovery_script.sh In usr/share/rear/wrapup/default/500_post_recovery_script.sh call POST_RECOVERY_COMMANDS directly after POST_RECOVERY_SCRIPT so POST_RECOVERY_COMMANDS can also be used to clean up things after POST_RECOVERY_SCRIPT --- .../wrapup/default/500_post_recovery_script.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/wrapup/default/500_post_recovery_script.sh b/usr/share/rear/wrapup/default/500_post_recovery_script.sh index 77751800c5..d55df562a0 100644 --- a/usr/share/rear/wrapup/default/500_post_recovery_script.sh +++ b/usr/share/rear/wrapup/default/500_post_recovery_script.sh @@ -1,4 +1,14 @@ + +# The POST_RECOVERY_COMMANDS are called directly after the POST_RECOVERY_SCRIPT +# so POST_RECOVERY_COMMANDS can also be used to clean up things after the POST_RECOVERY_SCRIPT: + if test "$POST_RECOVERY_SCRIPT" ; then - Log "Running POST_RECOVERY_SCRIPT '${POST_RECOVERY_SCRIPT[@]}'" - eval "${POST_RECOVERY_SCRIPT[@]}" + Log "Running POST_RECOVERY_SCRIPT '${POST_RECOVERY_SCRIPT[@]}'" + eval "${POST_RECOVERY_SCRIPT[@]}" fi + +local command +for command in "${POST_RECOVERY_COMMANDS[@]}" ; do + Log "Running POST_RECOVERY_COMMANDS '$command'" + eval "$command" +done From 0a1d634ed15500bb21f37ac1bbb11c8a4bb11545 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 30 May 2022 15:26:45 +0200 Subject: [PATCH 114/215] Update 200_partition_layout.sh In layout/save/GNU/Linux/200_partition_layout.sh do not error out when there is no partition label type value for a 'disk' entry in disklayout.conf because "rear recover" works in a special case without partition label type value when there is only a 'disk' entry but nothing else for this disk exists in disklayout.conf which can happen when /dev/sdX is an empty SD card slot without medium, see https://github.com/rear/rear/issues/2810 --- .../rear/layout/save/GNU/Linux/200_partition_layout.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh index 36bcc9df4e..a581adc064 100644 --- a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh @@ -406,7 +406,12 @@ Log "Saving disks and their partitions" # Each value must exist and each value must be a single non-blank word so we 'test' without quoting the value: test $devname || Error "Invalid 'disk' entry (no disk device name for '$disk')" test $devsize || Error "Invalid 'disk $devname' entry (no device size for '$devname')" - test $disktype || Error "Invalid 'disk $devname' entry (no partition table type for '$devname')" + # We do not error out when there is no partition label type value because + # "rear recover" works in a special case without partition label type value when there is + # only a 'disk' entry but nothing else for this disk exists in disklayout.conf + # which can happen when /dev/sdX is an empty SD card slot without medium, + # see https://github.com/rear/rear/issues/2810 + test $disktype || LogPrintError "No partition label type for 'disk $devname' (may cause 'rear recover' failure)" echo "# Disk $devname" echo "# Format: disk " From decd4655f084688640283184472dd2399bae6297 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 31 May 2022 13:17:51 +0200 Subject: [PATCH 115/215] Tell to use COPY_AS_IS to include additional modules in case of MODULES=( 'loaded_modules' ) In default.conf describe that one can use the generic COPY_AS_IS method to include certain modules in addition to the currently loaded modules when MODULES=( 'loaded_modules' ) is specified, see https://github.com/rear/rear/issues/2727#issuecomment-1141973114 --- usr/share/rear/conf/default.conf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 5eba7cd6af..b58e43fc90 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1548,11 +1548,12 @@ LIBS=() # automatically included but this may not work in any case # (see https://github.com/rear/rear/issues/1355). # The setting MODULES=( 'loaded_modules' ) results that -# only those kernel modules that are currently loaded -# (so MODULES=( 'loaded_modules' 'additional_module' ) is not supported) -# get included in the recovery system. This results a noticeable -# smaller recovery system but on the other hand it means that -# recovery will only work on same replacement hardware. +# only those kernel modules that are currently loaded get included +# in the recovery system so MODULES=( 'loaded_modules' 'additional_module' ) +# is not supported but you may use COPY_AS_IS+=( ...) to include +# certain modules in addition to the currently loaded modules. This +# results a noticeable smaller recovery system but on the other hand +# it means that recovery will only work on same replacement hardware. # The very special setting MODULES=( 'no_modules' ) enforces that # no kernel modules at all get included in the recovery system # regardless of what modules are currently loaded. Usually this From 66c7feda0e3112ff30865ac2e74ecd83283aa9b0 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Tue, 31 May 2022 13:28:16 +0200 Subject: [PATCH 116/215] Update the ISO volume label in comments In PR #2457, the default ISO_VOLID value "RELAXRECOVER" got changed to "REAR-ISO", but some code comments have the old value. Update them. --- usr/share/rear/lib/global-functions.sh | 12 ++++++------ .../rear/restore/NETFS/default/400_restore_backup.sh | 6 +++--- .../rear/restore/YUM/default/410_restore_backup.sh | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index 32aeb8ca7d..d97bcf6247 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -478,9 +478,9 @@ function mount_url() { return 0 ;; (iso) - # Check that there is a symbolic link /dev/disk/by-label/RELAXRECOVER - # that points to a block device that uses the filesystem label RELAXRECOVER. - # RELAXRECOVER is the default value of the ISO_VOLID config variable. + # Check that there is a symbolic link /dev/disk/by-label/REAR-ISO + # that points to a block device that uses the filesystem label REAR-ISO. + # REAR-ISO is the default value of the ISO_VOLID config variable. # If no such symbolic link exists create one because it is needed # during "rear recover" when the ISO image contains the backup, # see https://github.com/rear/rear/issues/1893 @@ -489,10 +489,10 @@ function mount_url() { # so that there is nothing to do here unless during "rear recover": test "recover" = "$WORKFLOW" || return 0 # Try to find a block device that uses the filesystem label ISO_VOLID. - # Usually "blkid -L RELAXRECOVER" results '/dev/sr0' or '/dev/sr1' + # Usually "blkid -L REAR-ISO" results '/dev/sr0' or '/dev/sr1' # cf. https://github.com/rear/rear/issues/1893#issuecomment-411034001 # but "blkid -L" is not supported on SLES10 (blkid is too old there) - # so that the traditional form "blkid -l -o device -t LABEL=RELAXRECOVER" + # so that the traditional form "blkid -l -o device -t LABEL=REAR-ISO" # is used which also works and is described in "man blkid" on SLES15: local relaxrecover_block_device="$( blkid -l -o device -t LABEL="$ISO_VOLID" )" # Try to get where the symbolic link /dev/disk/by-label/ISO_VOLID points to. @@ -534,7 +534,7 @@ function mount_url() { wilful_input="" symlink_target="" # When USER_INPUT_RELAXRECOVER_SYMLINK_TARGET has any 'true' value be liberal in what you accept and - # assume choices[0] 'Let /dev/disk/by-label/RELAXRECOVER point to /dev/cdrom' was actually meant: + # assume choices[0] 'Let /dev/disk/by-label/REAR-ISO point to /dev/cdrom' was actually meant: is_true "$USER_INPUT_RELAXRECOVER_SYMLINK_TARGET" && USER_INPUT_RELAXRECOVER_SYMLINK_TARGET="${choices[0]}" while true ; do choice="$( UserInput -I RELAXRECOVER_SYMLINK_TARGET -p "$prompt" -D "${choices[0]}" "${choices[@]}" )" && wilful_input="yes" || wilful_input="no" diff --git a/usr/share/rear/restore/NETFS/default/400_restore_backup.sh b/usr/share/rear/restore/NETFS/default/400_restore_backup.sh index 5a9d2db878..7e36f24723 100644 --- a/usr/share/rear/restore/NETFS/default/400_restore_backup.sh +++ b/usr/share/rear/restore/NETFS/default/400_restore_backup.sh @@ -40,9 +40,9 @@ if test -f $TMP_DIR/backup.splitted ; then Print "" while read backup_splitted_line ; do # The lines in backup.splitted are like - # backup.tar.gz.00 878706688 RELAXRECOVER - # backup.tar.gz.01 878706688 RELAXRECOVER_01 - # backup.tar.gz.02 758343480 RELAXRECOVER_02 + # backup.tar.gz.00 878706688 REAR-ISO + # backup.tar.gz.01 878706688 REAR-ISO_01 + # backup.tar.gz.02 758343480 REAR-ISO_02 # The first word is backup file name, the second a size, the last one is the label/vol_name: backup_file_name=${backup_splitted_line%% *} vol_name=${backup_splitted_line##* } diff --git a/usr/share/rear/restore/YUM/default/410_restore_backup.sh b/usr/share/rear/restore/YUM/default/410_restore_backup.sh index 94bfdad97e..7fe94ff768 100644 --- a/usr/share/rear/restore/YUM/default/410_restore_backup.sh +++ b/usr/share/rear/restore/YUM/default/410_restore_backup.sh @@ -33,9 +33,9 @@ if test -f $TMP_DIR/backup.splitted ; then Print "" while read backup_splitted_line ; do # The lines in backup.splitted are like - # backup.tar.gz.00 878706688 RELAXRECOVER - # backup.tar.gz.01 878706688 RELAXRECOVER_01 - # backup.tar.gz.02 758343480 RELAXRECOVER_02 + # backup.tar.gz.00 878706688 REAR-ISO + # backup.tar.gz.01 878706688 REAR-ISO_01 + # backup.tar.gz.02 758343480 REAR-ISO_02 # The first word is backup file name, the second a size, the last one is the label/vol_name: backup_file_name="${backup_splitted_line%% *}" vol_name="${backup_splitted_line##* }" From 79b54532f8038940105cd63d4e3ebf7bb6dd1d43 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 1 Jun 2022 09:26:10 +0200 Subject: [PATCH 117/215] Replace 'RELAXRECOVER' in variable names In lib/global-functions.sh in the function mount_url() renamed the user config variable USER_INPUT_RELAXRECOVER_SYMLINK_TARGET as USER_INPUT_ISO_SYMLINK_TARGET because the old name contained a user config value (RELAXRECOVER was a default value of ISO_VOLID), see https://github.com/rear/rear/pull/2813#discussion_r885576867 and accordingly also renamed the local variables relaxrecover_symlink_name and relaxrecover_symlink_target as iso_symlink_name and iso_symlink_target --- usr/share/rear/lib/global-functions.sh | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index d97bcf6247..2e883fed7d 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -494,34 +494,34 @@ function mount_url() { # but "blkid -L" is not supported on SLES10 (blkid is too old there) # so that the traditional form "blkid -l -o device -t LABEL=REAR-ISO" # is used which also works and is described in "man blkid" on SLES15: - local relaxrecover_block_device="$( blkid -l -o device -t LABEL="$ISO_VOLID" )" + local iso_block_device="$( blkid -l -o device -t LABEL="$ISO_VOLID" )" # Try to get where the symbolic link /dev/disk/by-label/ISO_VOLID points to. # "readlink -e symlink" outputs nothing when the symlink or its target does not exist: - local relaxrecover_symlink_name="/dev/disk/by-label/$ISO_VOLID" - local relaxrecover_symlink_target="$( readlink $verbose -e "$relaxrecover_symlink_name" )" - # Everything is o.k. when relaxrecover_block_device and relaxrecover_symlink_target are non-empty - # and when the relaxrecover_symlink_target value is the relaxrecover_block_device value. + local iso_symlink_name="/dev/disk/by-label/$ISO_VOLID" + local iso_symlink_target="$( readlink $verbose -e "$iso_symlink_name" )" + # Everything is o.k. when iso_block_device and iso_symlink_target are non-empty + # and when the iso_symlink_target value is the iso_block_device value. # Usually the right symbolic link /dev/disk/by-label/ISO_VOLID is set up automatically by udev. - if ! test "$relaxrecover_block_device" -a "$relaxrecover_symlink_target" -a "$relaxrecover_symlink_target" = "$relaxrecover_block_device" ; then + if ! test "$iso_block_device" -a "$iso_symlink_target" -a "$iso_symlink_target" = "$iso_block_device" ; then # If not everything is o.k. first try fix things automatically: - Log "Symlink '$relaxrecover_symlink_name' does not exist or does not point to a block device with '$ISO_VOLID' filesystem label" + Log "Symlink '$iso_symlink_name' does not exist or does not point to a block device with '$ISO_VOLID' filesystem label" # One of the things that could be not o.k. is that there is no /dev/disk/by-label/ directory. # Usually udev would automatically create it but sometimes that does not work, # cf. https://github.com/rear/rear/issues/1891#issuecomment-411027324 # so that we create a /dev/disk/by-label/ directory if it is not there: mkdir $verbose -p /dev/disk/by-label # Try to let the symbolic link point to the block device that uses the filesystem label ISO_VOLID: - if test -b "$relaxrecover_block_device" ; then - Log "Making symlink '$relaxrecover_symlink_name' point to '$relaxrecover_block_device' because it has filesystem label '$ISO_VOLID'" + if test -b "$iso_block_device" ; then + Log "Making symlink '$iso_symlink_name' point to '$iso_block_device' because it has filesystem label '$ISO_VOLID'" # Below there is a test that /dev/disk/by-label/ISO_VOLID exists which should detect when this 'ln' command failed: - ln $verbose -sf "$relaxrecover_block_device" "$relaxrecover_symlink_name" + ln $verbose -sf "$iso_block_device" "$iso_symlink_name" else # We found no block device that uses the filesystem label ISO_VOLID: Log "No block device with ISO filesystem label '$ISO_VOLID' found (by the blkid command)" # At this point things look not good so that now we need to tell the user about what is wrong: - LogPrintError "A symlink '$relaxrecover_symlink_name' is required that points to the device with the ReaR ISO image" + LogPrintError "A symlink '$iso_symlink_name' is required that points to the device with the ReaR ISO image" rear_workflow="rear $WORKFLOW" - rear_shell_history="$( echo -e "ln -vsf /dev/cdrom $relaxrecover_symlink_name\nls -l $relaxrecover_symlink_name" )" + rear_shell_history="$( echo -e "ln -vsf /dev/cdrom $iso_symlink_name\nls -l $iso_symlink_name" )" unset choices choices[0]="/dev/cdrom is where the ISO is attached to" choices[1]="/dev/sr0 is where the ISO is attached to" @@ -529,35 +529,35 @@ function mount_url() { choices[3]="Use Relax-and-Recover shell and return back to here" choices[4]="Continue '$rear_workflow'" choices[5]="Abort '$rear_workflow'" - prompt="Create symlink '$relaxrecover_symlink_name' that points to the ReaR ISO image device" + prompt="Create symlink '$iso_symlink_name' that points to the ReaR ISO image device" choice="" wilful_input="" symlink_target="" - # When USER_INPUT_RELAXRECOVER_SYMLINK_TARGET has any 'true' value be liberal in what you accept and + # When USER_INPUT_ISO_SYMLINK_TARGET has any 'true' value be liberal in what you accept and # assume choices[0] 'Let /dev/disk/by-label/REAR-ISO point to /dev/cdrom' was actually meant: - is_true "$USER_INPUT_RELAXRECOVER_SYMLINK_TARGET" && USER_INPUT_RELAXRECOVER_SYMLINK_TARGET="${choices[0]}" + is_true "$USER_INPUT_ISO_SYMLINK_TARGET" && USER_INPUT_ISO_SYMLINK_TARGET="${choices[0]}" while true ; do - choice="$( UserInput -I RELAXRECOVER_SYMLINK_TARGET -p "$prompt" -D "${choices[0]}" "${choices[@]}" )" && wilful_input="yes" || wilful_input="no" + choice="$( UserInput -I ISO_SYMLINK_TARGET -p "$prompt" -D "${choices[0]}" "${choices[@]}" )" && wilful_input="yes" || wilful_input="no" case "$choice" in (${choices[0]}) symlink_target="/dev/cdrom" is_true "$wilful_input" && LogPrint "User confirmed symlink target $symlink_target" || LogPrint "Using symlink target $symlink_target by default" # Below there is a test that /dev/disk/by-label/ISO_VOLID exists which should detect when this 'ln' command failed: - ln $verbose -sf $symlink_target "$relaxrecover_symlink_name" + ln $verbose -sf $symlink_target "$iso_symlink_name" break ;; (${choices[1]}) symlink_target="/dev/sr0" LogPrint "Using symlink target $symlink_target" # Below there is a test that /dev/disk/by-label/ISO_VOLID exists which should detect when this 'ln' command failed: - ln $verbose -sf $symlink_target "$relaxrecover_symlink_name" + ln $verbose -sf $symlink_target "$iso_symlink_name" break ;; (${choices[2]}) symlink_target="/dev/sr1" LogPrint "Using symlink target $symlink_target" # Below there is a test that /dev/disk/by-label/ISO_VOLID exists which should detect when this 'ln' command failed: - ln $verbose -sf $symlink_target "$relaxrecover_symlink_name" + ln $verbose -sf $symlink_target "$iso_symlink_name" break ;; (${choices[3]}) @@ -577,8 +577,8 @@ function mount_url() { fi fi # Check if /dev/disk/by-label/$ISO_VOLID exists (as symbolic link or in any other form), if yes assume things are right: - test -e "$relaxrecover_symlink_name" || Error "Cannot mount ISO because there is no '$relaxrecover_symlink_name'" - mount_cmd="mount $relaxrecover_symlink_name $mountpoint" + test -e "$iso_symlink_name" || Error "Cannot mount ISO because there is no '$iso_symlink_name'" + mount_cmd="mount $iso_symlink_name $mountpoint" ;; (var) ### The mount command is given by variable in the url host From 6e8aacef2f82410bda486656093aabb8ba8f3e35 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 1 Jun 2022 10:22:18 +0200 Subject: [PATCH 118/215] Update 010_pre_recovery_script.sh Include the changes from https://github.com/rear/rear/pull/2789 i.e. use LogPrint to show the user the executed commands --- usr/share/rear/setup/default/010_pre_recovery_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/setup/default/010_pre_recovery_script.sh b/usr/share/rear/setup/default/010_pre_recovery_script.sh index dbd186b187..8b4e4a3673 100644 --- a/usr/share/rear/setup/default/010_pre_recovery_script.sh +++ b/usr/share/rear/setup/default/010_pre_recovery_script.sh @@ -4,11 +4,11 @@ local command for command in "${PRE_RECOVERY_COMMANDS[@]}" ; do - Log "Running PRE_RECOVERY_COMMANDS '$command'" + LogPrint "Running PRE_RECOVERY_COMMANDS '$command'" eval "$command" done if test "$PRE_RECOVERY_SCRIPT" ; then - Log "Running PRE_RECOVERY_SCRIPT '${PRE_RECOVERY_SCRIPT[@]}'" + LogPrint "Running PRE_RECOVERY_SCRIPT '${PRE_RECOVERY_SCRIPT[@]}'" eval "${PRE_RECOVERY_SCRIPT[@]}" fi From 795178f52bb5d5f0cf3d7f587e6cc732ac60ce04 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 1 Jun 2022 10:25:12 +0200 Subject: [PATCH 119/215] Update 500_post_recovery_script.sh Include the changes from https://github.com/rear/rear/pull/2789 i.e. use LogPrint to show the user the executed commands --- usr/share/rear/wrapup/default/500_post_recovery_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/wrapup/default/500_post_recovery_script.sh b/usr/share/rear/wrapup/default/500_post_recovery_script.sh index d55df562a0..866c936839 100644 --- a/usr/share/rear/wrapup/default/500_post_recovery_script.sh +++ b/usr/share/rear/wrapup/default/500_post_recovery_script.sh @@ -3,12 +3,12 @@ # so POST_RECOVERY_COMMANDS can also be used to clean up things after the POST_RECOVERY_SCRIPT: if test "$POST_RECOVERY_SCRIPT" ; then - Log "Running POST_RECOVERY_SCRIPT '${POST_RECOVERY_SCRIPT[@]}'" + LogPrint "Running POST_RECOVERY_SCRIPT '${POST_RECOVERY_SCRIPT[@]}'" eval "${POST_RECOVERY_SCRIPT[@]}" fi local command for command in "${POST_RECOVERY_COMMANDS[@]}" ; do - Log "Running POST_RECOVERY_COMMANDS '$command'" + LogPrint "Running POST_RECOVERY_COMMANDS '$command'" eval "$command" done From 560770fd6ba61f7044aa276e5e1f261efe11c285 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 1 Jun 2022 13:13:02 +0200 Subject: [PATCH 120/215] Update default.conf In default.conf describe USB_BOOT_PART_SIZE and USB_DEVICE_BOOT_LABEL, see https://github.com/rear/rear/pull/2660 therein in particular see https://github.com/rear/rear/pull/2660#issuecomment-885582656 Regarding where by default the recovery system and backup get stored see https://github.com/rear/rear/pull/2660#issuecomment-885622061 and how they can be stored separated with an optional boot partition see https://github.com/rear/rear/pull/2660#issuecomment-885643937 --- usr/share/rear/conf/default.conf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index b58e43fc90..54b0b63779 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1044,6 +1044,26 @@ USB_PARTITION_ALIGN_BLOCK_SIZE="8" # USB_PARTITION_ALIGN_BLOCK_SIZE chunk: USB_UEFI_PART_SIZE="1024" # +# USB_BOOT_PART_SIZE specifies the size of an optional boot partition in MiB +# when formatting a medium by the format workflow. +# A boot partition is created when USB_BOOT_PART_SIZE is a positive integer. +# By default we do not create a separated boot partition +# so by default the ReaR recovery system (kernel and initrd) +# and the backup archive get stored in the ReaR data partition +# in a rear/HOSTNAME/TIMESTAMP directory (cf. USB_SUFFIX below): +USB_BOOT_PART_SIZE="0" +# +# The label that is set for a boot partition via the format workflow +# when a boot partition is created (see USB_BOOT_PART_SIZE above). +# That label must be used for settings like +# OUTPUT_URL=usb:///dev/disk/by-label/$USB_DEVICE_BOOT_LABEL +# BACKUP_URL=usb:///dev/disk/by-label/$USB_DEVICE_FILESYSTEM_LABEL +# to get the ReaR recovery system (kernel and initrd) stored +# in the boot partition in a rear/HOSTNAME/TIMESTAMP directory +# while the backup archive gets stored in the ReaR data partition +# there in a matching rear/HOSTNAME/TIMESTAMP directory: +USB_DEVICE_BOOT_LABEL="REARBOOT" +# # Default boot option (i.e. what gets booted automatically after some timeout) # when EXTLINUX boots the USB stick or USB disk or other disk device on BIOS systems. # USB_BIOS_BOOT_DEFAULT="boothd0" boots from 'boothd0' which is usually the same disk From fcc5005ee73e73d22360b3a6e4cca6af9d4b03ba Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 3 Jun 2022 11:07:35 +0200 Subject: [PATCH 121/215] Describe PRE_BACKUP_SCRIPT and POST_BACKUP_SCRIPT properly In default.conf properly describe PRE_BACKUP_SCRIPT and POST_BACKUP_SCRIPT to match what the current implementation does, see https://github.com/rear/rear/pull/2735#issuecomment-1145006984 --- usr/share/rear/conf/default.conf | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index bb34c084db..b0e6f209ee 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -3483,7 +3483,7 @@ ELILO_BIN= ################ ---- custom scripts # -# NOTE: The scripts can be defined as an array to better handly spaces in parameters. +# The scripts can be defined as an array to better handle spaces in parameters. # The scripts are called like this: # eval "${PRE_RECOVERY_SCRIPT[@]}" # @@ -3516,19 +3516,26 @@ PRE_RECOVERY_SCRIPT= POST_RECOVERY_SCRIPT= POST_RECOVERY_COMMANDS=() -# PRE/POST Backup scripts will provide the ability to run certain tasks before and after a ReaR backup. -# for example: -# If a small database running on local filesystem and dependent on a local service, you will maintain its data consistency. -# Stopping it before backup and restarting again after. -# In case of any error during backup, if POST tasks were defined, ReaR will run those POST tasks within ExitTasks Array. -# This will prevent that the database remain stopped. +# PRE_BACKUP_SCRIPT and POST_BACKUP_SCRIPT are called at the beginning and +# at the end of the backup part in the mkbackup/mkbackuponly workflow like this: +# eval "${PRE_BACKUP_SCRIPT[@]}" + +# For example: +# When a database is running on a filesystem which is included in the backup +# you may need to stop a database service before backup via PRE_BACKUP_SCRIPT +# and restart it after the backup was done via POST_BACKUP_SCRIPT. +# In case of an error during backup (and provided PRE_BACKUP_SCRIPT is set), +# ReaR will run the POST_BACKUP_SCRIPT as exit task (via ReaR's EXIT_TASKS array). +# This prevents that the database remains stopped in case of an error during backup +# which also means the database is active after there was an error during backup +# when the POST_BACKUP_SCRIPT restarts the database unconditioned. + +# Called at the beginning of the backup part in the mkbackup/mkbackuponly workflow: +PRE_BACKUP_SCRIPT= -# Call this after Relax-and-Recover finished to do anything in the mkbackup/mkbackuponly workflow. +# Called at the end of the backup part in the mkbackup/mkbackuponly workflow: POST_BACKUP_SCRIPT= -# Call this before Relax-and-Recover starts to do anything in the mkbackup/mkbackuponly workflow. -PRE_BACKUP_SCRIPT= - # Some external backup software request user input # (e.g. to enter paths to exclude or date and time values for point in time restore). # We use here the same default timeout as USER_INPUT_TIMEOUT was set above From 6fcbba895c280c68e8ccf54290348ea845ecf201 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 7 Jun 2022 11:47:02 +0200 Subject: [PATCH 122/215] Clearly describe when POST_BACKUP_SCRIPT is run or not In default.conf describe in more detail when POST_BACKUP_SCRIPT is normally run or run as exit task versus when it is not run, cf. https://github.com/rear/rear/pull/2735#issuecomment-1148430695 --- usr/share/rear/conf/default.conf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index b0e6f209ee..d7cda35106 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -3524,11 +3524,14 @@ POST_RECOVERY_COMMANDS=() # When a database is running on a filesystem which is included in the backup # you may need to stop a database service before backup via PRE_BACKUP_SCRIPT # and restart it after the backup was done via POST_BACKUP_SCRIPT. -# In case of an error during backup (and provided PRE_BACKUP_SCRIPT is set), +# In case of an error exit during backup and provided PRE_BACKUP_SCRIPT is set, # ReaR will run the POST_BACKUP_SCRIPT as exit task (via ReaR's EXIT_TASKS array). -# This prevents that the database remains stopped in case of an error during backup -# which also means the database is active after there was an error during backup +# This prevents that the database remains stopped in case of an error exit during backup +# which also means the database is active after there was an error exit during backup # when the POST_BACKUP_SCRIPT restarts the database unconditioned. +# When only POST_BACKUP_SCRIPT is set without a PRE_BACKUP_SCRIPT +# then POST_BACKUP_SCRIPT is not run in case of an error exit during backup. +# POST_BACKUP_SCRIPT is run if there is some error but no error exit during backup. # Called at the beginning of the backup part in the mkbackup/mkbackuponly workflow: PRE_BACKUP_SCRIPT= From 0f6c923a2fe9ae3516d966310e2b4056dfc9c37d Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 7 Jun 2022 12:06:49 +0200 Subject: [PATCH 123/215] Typo fix in default.conf comment finalize/Linux-i386/620_install_grub2.sh had been moved to finalize/Linux-i386/660_install_grub2.sh via https://github.com/rear/rear/commit/6b1889cbf7bed297a5a695fd01f52865c4527827 as a part of https://github.com/rear/rear/pull/2047 --- usr/share/rear/conf/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index d7cda35106..dd92a51f7a 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -3290,7 +3290,7 @@ BOOTLOADER="" # is used by PC-BIOS to embed boot code). # When GRUB2_INSTALL_DEVICES is not specified, ReaR tries to automatically determine # where to install GRUB2 but then the bootloader installation could get wrong. -# For details see the finalize/Linux-i386/620_install_grub2.sh script. +# For details see the finalize/Linux-i386/660_install_grub2.sh script. # GRUB2_INSTALL_DEVICES is set to a default value here only # if not already set so that the user can set it also via # export GRUB2_INSTALL_DEVICES="/dev/sdb" From 33c6cc6f6ff1f8608f7227f70da2a2f2b6b532e3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 7 Jun 2022 12:42:26 +0200 Subject: [PATCH 124/215] Update 660_install_grub2.sh In finalize/Linux-i386/660_install_grub2.sh explain that it is also used as fallback to install the nowadays most often used bootloader GRUB2 unless the BOOTLOADER variable tells to install another bootloader (other bootloader install scripts check the BOOTLOADER variable), cf. https://github.com/rear/rear/issues/2817#issuecomment-1148488339 --- .../rear/finalize/Linux-i386/660_install_grub2.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/finalize/Linux-i386/660_install_grub2.sh b/usr/share/rear/finalize/Linux-i386/660_install_grub2.sh index 58163d622b..d1c36bd3a1 100644 --- a/usr/share/rear/finalize/Linux-i386/660_install_grub2.sh +++ b/usr/share/rear/finalize/Linux-i386/660_install_grub2.sh @@ -28,8 +28,15 @@ # This is also the reason why more than one disk can be specified # in GRUB2_INSTALL_DEVICES. # -# * There is no guarantee that GRUB2 was used as bootloader on the original system. -# The solution is to specify the BOOTLOADER config variable. +# * When GRUB2 is used as bootloader on the original system +# there is no guarantee that the automatisms in ReaR correctly detect that +# cf. layout/save/default/445_guess_bootloader.sh +# To avoid that automatisms in ReaR may set a wrong BOOTLOADER value +# the solution is to specify the BOOTLOADER config variable. +# This script does not check BOOTLOADER because it is also used as fallback +# to install the nowadays most often used bootloader GRUB2 +# unless the BOOTLOADER variable tells to install another bootloader +# (other bootloader install scripts check the BOOTLOADER variable). # # This script does not error out because at this late state of "rear recover" # (i.e. after the backup was restored) I consider it too hard From ae0b88a94f51ea982a8234b346c95ede388d6288 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 10:56:31 +0200 Subject: [PATCH 125/215] Preparations for ReaR 2.7 See https://github.com/rear/rear/wiki/Release-process Set VERSION=2.7 in usr/sbin/rear --- usr/sbin/rear | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/rear b/usr/sbin/rear index 04979257f9..e97e1880d7 100755 --- a/usr/sbin/rear +++ b/usr/sbin/rear @@ -55,7 +55,7 @@ readonly INITIAL_BASH_FLAGS_AND_OPTIONS_COMMANDS="$( get_bash_flags_and_options_ # Versioning readonly PRODUCT="Relax-and-Recover" readonly PROGRAM=${0##*/} -readonly VERSION=2.6 +readonly VERSION=2.7 readonly RELEASE_DATE=Git # Where users should report bugs: From d969c737a90bd55611afd4cfae3344e621a05b4a Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 11:02:03 +0200 Subject: [PATCH 126/215] Update rear.spec Set 'Version: 2.7' in packaging/rpm/rear.spec see https://github.com/rear/rear/wiki/Release-process --- packaging/rpm/rear.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/rpm/rear.spec b/packaging/rpm/rear.spec index b2b20a1ecf..d55b4707c2 100644 --- a/packaging/rpm/rear.spec +++ b/packaging/rpm/rear.spec @@ -8,7 +8,7 @@ Summary: Relax-and-Recover is a Linux disaster recovery and system migration tool Name: rear -Version: 2.6 +Version: 2.7 Release: 1%{?rpmrelease}%{?dist} # Since some time the license value 'GPLv3' causes build failures in the openSUSE Build Service # cf. https://github.com/rear/rear/issues/2289#issuecomment-559713101 From f17d67796bbb7dd0302ef54043745093174103d0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 11:11:46 +0200 Subject: [PATCH 127/215] Update changelog In packaging/debian/changelog added generic changelog entry for 'rear (2.7-1)' see https://github.com/rear/rear/wiki/Release-process --- packaging/debian/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packaging/debian/changelog b/packaging/debian/changelog index 89aac9b202..3af1e40f33 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -1,3 +1,8 @@ +rear (2.7-1) stable; urgency=low + + * For details see the rear-release-notes.txt file + + -- Johannes Meixner Thu, 9 Jun 2022 09:04:55 UTC rear (2.6-1) stable; urgency=low * Mainly bug fixes From fc2bedbc5fc8da33ea97e848b2e95eaf054708c0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 11:14:45 +0200 Subject: [PATCH 128/215] Update rear.8.adoc In doc/rear.8.adoc set :version: v2.7 :date: 09 June 2022 see https://github.com/rear/rear/wiki/Release-process --- doc/rear.8.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/rear.8.adoc b/doc/rear.8.adoc index d68c915c62..932bf0a262 100644 --- a/doc/rear.8.adoc +++ b/doc/rear.8.adoc @@ -1,6 +1,6 @@ = rear(8) -:version: v2.6 -:date: 17 June 2020 +:version: v2.7 +:date: 09 June 2022 :data-uri: :doctype: manpage :lang: en From 6adb6dff7c3aed88511911f18de7a1d8b30bc093 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 12:36:09 +0200 Subject: [PATCH 129/215] Update rear.8.adoc In doc/rear.8.adoc extend the COPYRIGHT time from '2006-2020' to '2006-2022' --- doc/rear.8.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rear.8.adoc b/doc/rear.8.adoc index 932bf0a262..89941009ea 100644 --- a/doc/rear.8.adoc +++ b/doc/rear.8.adoc @@ -590,7 +590,7 @@ The Relax-and-Recover website is located at: http://relax-and-recover.org/ == COPYRIGHT -(c) 2006-2020 +(c) 2006-2022 The copyright is held by the original authors of the respective code pieces as can be seen in the git history at https://github.com/rear/rear/graphs/contributors From 27fa483fc62a1869cb1fea8d20aa167e82237810 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 12:46:52 +0200 Subject: [PATCH 130/215] Generated rear.8 in doc/ from rear.8.adoc via 'asciidoctor -b manpage rear.8.adoc' --- doc/rear.8 | 1003 ++++++++++++++++++++++++----------------------- doc/rear.8.adoc | 1 - 2 files changed, 512 insertions(+), 492 deletions(-) diff --git a/doc/rear.8 b/doc/rear.8 index b899b6b3f5..5c7bb42bb2 100644 --- a/doc/rear.8 +++ b/doc/rear.8 @@ -1,192 +1,202 @@ '\" t .\" Title: rear -.\" Author: [see the "AUTHORS" section] -.\" Generator: DocBook XSL Stylesheets v1.78.1 -.\" Date: 17 June 2020 +.\" Author: [see the "AUTHOR(S)" section] +.\" Generator: Asciidoctor 2.0.10 +.\" Date: 2022-06-09 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "REAR" "8" "17 June 2020" "\ \&" "\ \&" -.\" ----------------------------------------------------------------- -.\" * Define some portability stuff -.\" ----------------------------------------------------------------- -.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.\" http://bugs.debian.org/507673 -.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html -.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.TH "REAR" "8" "2022-06-09" "\ \&" "\ \&" .ie \n(.g .ds Aq \(aq .el .ds Aq ' -.\" ----------------------------------------------------------------- -.\" * set default formatting -.\" ----------------------------------------------------------------- -.\" disable hyphenation +.ss \n[.ss] 0 .nh -.\" disable justification (adjust text to left margin only) .ad l -.\" ----------------------------------------------------------------- -.\" * MAIN CONTENT STARTS HERE * -.\" ----------------------------------------------------------------- +.de URL +\fI\\$2\fP <\\$1>\\$3 +.. +.als MTO URL +.if \n[.g] \{\ +. mso www.tmac +. am URL +. ad l +. . +. am MTO +. ad l +. . +. LINKSTYLE blue R < > +.\} .SH "NAME" rear \- bare metal disaster recovery and system migration tool .SH "SYNOPSIS" .sp -\fBrear\fR [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fB\-dsSv\fR] [\fB\-D\fR|\fB\-\-debugscripts\fR \fISET\fR] [\fB\-c\fR \fIDIR\fR] [\fB\-C\fR \fICONFIG\fR] [\fB\-r\fR \fIKERNEL\fR] [\-\-] \fICOMMAND\fR [\fIARGS\fR\&...] +\fBrear\fP [\fB\-h\fP|\fB\-\-help\fP] [\fB\-V\fP|\fB\-\-version\fP] [\fB\-dsSv\fP] [\fB\-D\fP|\fB\-\-debugscripts\fP \fISET\fP] [\fB\-c\fP \fIDIR\fP] [\fB\-C\fP \fICONFIG\fP] [\fB\-r\fP \fIKERNEL\fP] [\-\-] \fICOMMAND\fP [\fIARGS\fP...] .SH "DESCRIPTION" .sp -Relax\-and\-Recover (abbreviated ReaR) is the leading Open Source disaster recovery solution\&. It is a modular framework with many ready\-to\-go workflows for common situations\&. -.sp -Relax\-and\-Recover produces a bootable image\&. This image can repartition the system\&. Once that is done it initiates a restore from backup\&. Restores to different hardware are possible\&. Relax\-and\-Recover can therefore be used as a migration tool as well\&. -.sp -Currently Relax\-and\-Recover supports various boot media (incl\&. ISO, PXE, OBDR tape, USB or eSATA storage), a variety of network protocols (incl\&. sftp, ftp, http, nfs, cifs) for storage and backup as well as a multitude of backup strategies (incl\&. IBM Tivoli Storage Manager, MircoFocus Data Protector, Symantec NetBackup, EMC NetWorker, EMC Avamar, FDR/Upstream, NovaStor DC, Rubrik Cloud Data Management (CDM), Bareos, Bacula, rsync, rbme, Borg)\&. This results in a bootable image that is capable of booting via PXE, DVD/CD, bootable tape or virtual provisioning\&. -.sp -Relax\-and\-Recover was designed to be easy to set up, requires no maintenance and is there to assist when disaster strikes\&. Its setup\-and\-forget nature removes any excuses for not having a disaster recovery solution implemented\&. -.sp -Recovering from disaster is made very straight\-forward by a 2\-step recovery process so that it can be executed by operational teams when required\&. When used interactively (e\&.g\&. when used for migrating systems), menus help make decisions to restore to a new (hardware) environment\&. -.sp -Extending Relax\-and\-Recover is made possible by its modular framework\&. Consistent logging and optionally extended output help understand the concepts behind Relax\-and\-Recover and help debug during development\&. -.sp -Relax\-and\-Recover comes with ABSOLUTELY NO WARRANTY; for details see the GNU General Public License at: \m[blue]\fBhttp://www\&.gnu\&.org/licenses/gpl\&.html\fR\m[] +Relax\-and\-Recover (abbreviated ReaR) is the leading Open Source disaster recovery solution. +It is a modular framework with many ready\-to\-go workflows for common situations. +.sp +Relax\-and\-Recover produces a bootable image. This image can repartition the +system. Once that is done it initiates a restore from backup. Restores to +different hardware are possible. Relax\-and\-Recover can therefore be used as a +migration tool as well. +.sp +Currently Relax\-and\-Recover supports various boot media (incl. ISO, PXE, +OBDR tape, USB or eSATA storage), a variety of network protocols (incl. +sftp, ftp, http, nfs, cifs) for storage and backup as well as a multitude +of backup strategies (incl. IBM Tivoli Storage Manager, MircoFocus Data Protector, +Symantec NetBackup, EMC NetWorker, EMC Avamar, FDR/Upstream, NovaStor DC, Rubrik Cloud Data Management (CDM), +Bareos, Bacula, rsync, rbme, Borg). This results in a bootable image that is capable of +booting via PXE, DVD/CD, bootable tape or virtual provisioning. +.sp +Relax\-and\-Recover was designed to be easy to set up, requires no maintenance +and is there to assist when disaster strikes. Its setup\-and\-forget nature +removes any excuses for not having a disaster recovery solution implemented. +.sp +Recovering from disaster is made very straight\-forward by a 2\-step recovery +process so that it can be executed by operational teams when required. +When used interactively (e.g. when used for migrating systems), menus help +make decisions to restore to a new (hardware) environment. +.sp +Extending Relax\-and\-Recover is made possible by its modular framework. +Consistent logging and optionally extended output help understand the concepts +behind Relax\-and\-Recover and help debug during development. +.sp +Relax\-and\-Recover comes with ABSOLUTELY NO WARRANTY; for details see +the GNU General Public License at: \c +.URL "http://www.gnu.org/licenses/gpl.html" "" "" .SH "OPTIONS" .SS "GLOBAL OPTIONS" -.PP +.sp \-h \-\-help .RS 4 usage information .RE -.PP +.sp \-c DIR .RS 4 alternative config directory; instead of /etc/rear .RE -.PP +.sp \-C CONFIG .RS 4 additional config file; absolute path or relative to config directory .RE -.PP +.sp \-d .RS 4 -\fBdebug mode\fR -(run many commands verbosely with debug messages in log file \- also sets \-v) +\fBdebug mode\fP (run many commands verbosely with debug messages in log file \- also sets \-v) .RE -.PP +.sp \-D .RS 4 -\fBdebugscript mode\fR -(log executed commands via -\fIset \-x\fR -\- also sets \-v and \-d) +\fBdebugscript mode\fP (log executed commands via \(aqset \-x\(aq \- also sets \-v and \-d) .RE -.PP +.sp \-\-debugscripts SET .RS 4 -same as \-d \-v \-D but -\fBdebugscript mode\fR -with -\fIset \-SET\fR +same as \-d \-v \-D but \fBdebugscript mode\fP with \(aqset \-SET\(aq .RE -.PP +.sp \-r KERNEL .RS 4 kernel version to use (by default use running kernel) .RE -.PP +.sp \-s .RS 4 -\fBsimulation mode\fR -(show what scripts are run without executing them) +\fBsimulation mode\fP (show what scripts are run without executing them) .RE -.PP +.sp \-S .RS 4 -\fBstep\-by\-step mode\fR -(acknowledge each script individually) +\fBstep\-by\-step mode\fP (acknowledge each script individually) .RE -.PP +.sp \-v .RS 4 -\fBverbose mode\fR -(show messages what ReaR is doing on the terminal) +\fBverbose mode\fP (show messages what ReaR is doing on the terminal) .RE -.PP +.sp \-V \-\-version .RS 4 version information .RE .SS "COMMANDS" -.PP -\fBchecklayout\fR +.sp +\fBchecklayout\fP .RS 4 -check if the disk layout has changed since the last run of mkbackup/mkrescue +check if the disk layout has changed since the last run of +mkbackup/mkrescue .RE -.PP -\fBdump\fR +.sp +\fBdump\fP .RS 4 -dump configuration and system information; please run this to verify your setup +dump configuration and system information; please run this to verify +your setup .RE -.PP -\fBformat\fR +.sp +\fBformat\fP .RS 4 format and label USB or tape media to be used with rear; - -first argument is the USB or tape device to use, eg\&. -\fI/dev/sdX\fR -or -\fI/dev/stX\fR +first argument is the USB or tape device to use, eg. \fI/dev/sdX\fP or +\fI/dev/stX\fP .RE -.PP -\fBhelp\fR +.sp +\fBhelp\fP .RS 4 print full list of commands and options .RE -.PP -\fBmkbackup\fR +.sp +\fBmkbackup\fP .RS 4 -create rescue media and backup the system (only for internal backup methods) +create rescue media and backup the system (only for internal backup +methods) .RE -.PP -\fBmkbackuponly\fR +.sp +\fBmkbackuponly\fP .RS 4 -backup the system (only for internal backup methods) without creating rescue media +backup the system (only for internal backup methods) without creating +rescue media .RE -.PP -\fBmkrescue\fR +.sp +\fBmkrescue\fP .RS 4 create rescue media only .RE -.PP -\fBmountonly\fR +.sp +\fBmountonly\fP .RS 4 use ReaR as live media to mount and repair the system .RE -.PP -\fBrecover\fR +.sp +\fBrecover\fP .RS 4 recover the system; can be used only when running from the rescue media .RE -.PP -\fBrestoreonly\fR +.sp +\fBrestoreonly\fP .RS 4 only restore the backup; can be used only when running from the rescue media .RE -.PP -\fBmkopalpba\fR +.sp +\fBmkopalpba\fP .RS 4 create a pre\-boot authentication (PBA) image to boot from TCG Opal 2\-compliant self\-encrypting disks .RE -.PP -\fBopaladmin\fR +.sp +\fBopaladmin\fP .RS 4 administrate TCG Opal 2\-compliant self\-encrypting disks .RE -.PP -\fBvalidate\fR +.sp +\fBvalidate\fP .RS 4 submit validation information .RE .sp -Use \fIrear \-v help\fR for more advanced commands\&. +Use \(aqrear \-v help\(aq for more advanced commands. .SH "BACKGROUND INFORMATION" .sp The process of bare metal disaster recovery consists of two parts: @@ -196,8 +206,8 @@ The process of bare metal disaster recovery consists of two parts: \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} Recreate the system layout .RE @@ -207,60 +217,71 @@ Recreate the system layout \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} Restore the data to the system .RE .sp -Most backup software solutions are very good at restoring data but do not support recreating the system layout\&. Relax\-and\-Recover is very good at recreating the system layout but works best when used together with supported backup software\&. +Most backup software solutions are very good at restoring data but do not +support recreating the system layout. Relax\-and\-Recover is very good at +recreating the system layout but works best when used together with +supported backup software. .sp -In this combination Relax\-and\-Recover recreates the system layout and calls the backup software to restore the actual data\&. Thus there is no unnecessary duplicate data storage and the Relax\-and\-Recover rescue media can be very small\&. +In this combination Relax\-and\-Recover recreates the system layout and calls +the backup software to restore the actual data. Thus there is no unnecessary +duplicate data storage and the Relax\-and\-Recover rescue media can be very small. .sp -For demonstration and special use purposes Relax\-and\-Recover also includes an internal backup method, NETFS, which can be used to create a simple tar\&.gz archive of the system\&. For all permanent setups we recommend using something more professional for backup, either a traditional backup software (open source or commercial) or rsync with hardlink based solutions, e\&.g\&. RSYNC BACKUP MADE EASY\&. +For demonstration and special use purposes Relax\-and\-Recover also includes +an internal backup method, NETFS, which can be used to create a simple tar.gz +archive of the system. For all permanent setups we recommend using something +more professional for backup, either a traditional backup software (open +source or commercial) or rsync with hardlink based solutions, e.g. RSYNC +BACKUP MADE EASY. .SH "RESCUE IMAGE CONFIGURATION" .sp -The OUTPUT variable defines from where our bootable rescue image will be booted and the OUTPUT_URL variable defines where the rescue image should be send to\&. Possible OUTPUT settings are: -.PP -OUTPUT=\fBRAMDISK\fR +The OUTPUT variable defines from where our bootable rescue image will be +booted and the OUTPUT_URL variable defines where the rescue image should be +send to. Possible OUTPUT settings are: +.sp +OUTPUT=\fBRAMDISK\fP .RS 4 -Create only the Relax\-and\-Recover initramfs\&. +Create only the Relax\-and\-Recover initramfs. .RE -.PP -OUTPUT=\fBISO\fR +.sp +OUTPUT=\fBISO\fP .RS 4 -\fB(Default)\fR -Create a bootable ISO9660 image on disk as -\fIrear\-$(hostname)\&.iso\fR +\fB(Default)\fP Create a bootable ISO9660 image on disk as \fIrear\-$(hostname).iso\fP .RE -.PP -OUTPUT=\fBPXE\fR +.sp +OUTPUT=\fBPXE\fP .RS 4 -Create on a remote PXE/NFS server the required files (such as configuration file, kernel and initrd image) +Create on a remote PXE/NFS server the required files (such as +configuration file, kernel and initrd image) .RE -.PP -OUTPUT=\fBOBDR\fR +.sp +OUTPUT=\fBOBDR\fP .RS 4 -Create a bootable OBDR tape (optionally including the backup archive)\&. Specify the OBDR tape device by using -TAPE_DEVICE\&. +Create a bootable OBDR tape (optionally including the backup archive). +Specify the OBDR tape device by using TAPE_DEVICE. .RE -.PP -OUTPUT=\fBUSB\fR +.sp +OUTPUT=\fBUSB\fP .RS 4 -Create a bootable USB disk\&. +Create a bootable USB disk. .RE -.PP -OUTPUT=\fBRAWDISK\fR +.sp +OUTPUT=\fBRAWDISK\fP .RS 4 -Create a bootable image file named "rear\-$(hostname)\&.raw\&.gz", which +Create a bootable image file named "rear\-$(hostname).raw.gz", which .sp .RS 4 .ie n \{\ \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} supports UEFI boot if syslinux/EFI or Grub 2/EFI is installed, .RE @@ -270,8 +291,8 @@ supports UEFI boot if syslinux/EFI or Grub 2/EFI is installed, \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} supports Legacy BIOS boot if syslinux is installed, .RE @@ -281,407 +302,374 @@ supports Legacy BIOS boot if syslinux is installed, \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} -supports UEFI/Legacy BIOS dual boot if syslinux -\fBand\fR -one of the supported EFI bootloaders are installed\&. +supports UEFI/Legacy BIOS dual boot if syslinux \fBand\fP one of the supported EFI +bootloaders are installed. .RE .RE .sp -When using OUTPUT=ISO, RAMDISK, OBDR, USB, or RAWDISK you should provide the backup target location through the OUTPUT_URL variable\&. Possible OUTPUT_URL settings are: -.PP -OUTPUT_URL=\fBfile://\fR +When using OUTPUT=ISO, RAMDISK, OBDR, USB, or RAWDISK you should +provide the backup target location through the OUTPUT_URL variable. Possible +OUTPUT_URL settings are: +.sp +OUTPUT_URL=\fBfile://\fP .RS 4 -Write the image to disk\&. The default is in -\fI/var/lib/rear/output/\fR\&. +Write the image to disk. The default is in \fI/var/lib/rear/output/\fP. .RE -.PP -OUTPUT_URL=\fBnfs://\fR +.sp +OUTPUT_URL=\fBnfs://\fP .RS 4 -Write the image by mounting the target filesystem via NFS\&. +Write the image by mounting the target filesystem via NFS. .RE -.PP -OUTPUT_URL=\fBcifs://\fR +.sp +OUTPUT_URL=\fBcifs://\fP .RS 4 -Write the image by mounting the target filesystem via CIFS\&. +Write the image by mounting the target filesystem via CIFS. .RE -.PP -OUTPUT_URL=\fBfish://\fR +.sp +OUTPUT_URL=\fBfish://\fP .RS 4 -Write the image using -lftp -and the FISH protocol\&. +Write the image using lftp and the FISH protocol. .RE -.PP -OUTPUT_URL=\fBftp://\fR +.sp +OUTPUT_URL=\fBftp://\fP .RS 4 -Write the image using -lftp -and the FTP protocol\&. +Write the image using lftp and the FTP protocol. .RE -.PP -OUTPUT_URL=\fBftps://\fR +.sp +OUTPUT_URL=\fBftps://\fP .RS 4 -Write the image using -lftp -and the FTPS protocol\&. +Write the image using lftp and the FTPS protocol. .RE -.PP -OUTPUT_URL=\fBhftp://\fR +.sp +OUTPUT_URL=\fBhftp://\fP .RS 4 -Write the image using -lftp -and the HFTP protocol\&. +Write the image using lftp and the HFTP protocol. .RE -.PP -OUTPUT_URL=\fBhttp://\fR +.sp +OUTPUT_URL=\fBhttp://\fP .RS 4 -Write the image using -lftp -and the HTTP (PUT) procotol\&. +Write the image using lftp and the HTTP (PUT) procotol. .RE -.PP -OUTPUT_URL=\fBhttps://\fR +.sp +OUTPUT_URL=\fBhttps://\fP .RS 4 -Write the image using -lftp -and the HTTPS (PUT) protocol\&. +Write the image using lftp and the HTTPS (PUT) protocol. .RE -.PP -OUTPUT_URL=\fBsftp://\fR +.sp +OUTPUT_URL=\fBsftp://\fP .RS 4 -Write the image using -lftp -and the secure FTP (SFTP) protocol\&. +Write the image using lftp and the secure FTP (SFTP) protocol. .RE -.PP -OUTPUT_URL=\fBrsync://\fR +.sp +OUTPUT_URL=\fBrsync://\fP .RS 4 -Write the image using -rsync -and the RSYNC protocol\&. +Write the image using rsync and the RSYNC protocol. .RE -.PP -OUTPUT_URL=\fBsshfs://\fR +.sp +OUTPUT_URL=\fBsshfs://\fP .RS 4 -Write the image using sshfs and the SSH protocol\&. +Write the image using sshfs and the SSH protocol. .RE -.PP -OUTPUT_URL=\fBnull\fR +.sp +OUTPUT_URL=\fBnull\fP .RS 4 -Do not copy the rescue image from -\fI/var/lib/rear/output/\fR -to a remote output location\&. -OUTPUT_URL=null -is useful when another program (e\&.g\&. an -\fIexternal\fR -backup program) is used to save the rescue image from the local system to a remote place, or with -BACKUP_URL=iso:///backup -when the backup is included in the rescue image to avoid a (big) copy of the rescue image at a remote output location\&. In the latter case the rescue image must be manually saved from the local system to a remote place\&. -OUTPUT_URL=null -is only supported together with -BACKUP=NETFS\&. +Do not copy the rescue image from \fI/var/lib/rear/output/\fP to a remote output location. +OUTPUT_URL=null is useful when another program (e.g. an \fIexternal\fP backup program) +is used to save the rescue image from the local system to a remote place, +or with BACKUP_URL=iso:///backup when the backup is included in the rescue image +to avoid a (big) copy of the rescue image at a remote output location. +In the latter case the rescue image must be manually saved from the local system to a remote place. +OUTPUT_URL=null is only supported together with BACKUP=NETFS. .RE .sp -If you do not specify OUTPUT_URL variable then by default it will be aligned to what was defined by variable BACKUP_URL\&. And, the rescue image will then be copied to the same location as your backup of the system disk(s)\&. +If you do not specify OUTPUT_URL variable then by default it will be aligned to what +was defined by variable BACKUP_URL. And, the rescue image will then be copied to the same +location as your backup of the system disk(s). .sp -The ISO_DEFAULT variable defines what default boot option is used on the rescue image\&. Possible values are manual, boothd or automatic\&. Manual will make you boot into the shell directly by default, boothd will boot to the first disk (default) or automatic will automatically start in recovery mode\&. +The ISO_DEFAULT variable defines what default boot option is used on the rescue image. +Possible values are \f(CRmanual\fP, \f(CRboothd\fP or \f(CRautomatic\fP. Manual will make you boot into +the shell directly by default, boothd will boot to the first disk (default) or automatic +will automatically start in recovery mode. .SH "RESCUE IMAGE KERNEL COMMAND LINE OPTIONS" .sp -When booting the rescue image you can edit the kernel command line\&. There are some Relax\-and\-Recover specific kernel command line options: -.PP -\fBdebug\fR +When booting the rescue image you can edit the kernel command line. +There are some Relax\-and\-Recover specific kernel command line options: +.sp +\fBdebug\fP .RS 4 -The rescue system start up scripts in /etc/scripts/system\-setup\&.d/ are run one by one each one after pressing -\fIEnter\fR -and the scripts are run with -\fIset \-x\fR -which prints commands and their arguments as they are executed\&. +The rescue system start up scripts in /etc/scripts/system\-setup.d/ +are run one by one each one after pressing \(aqEnter\(aq and +the scripts are run with \(aqset \-x\(aq which prints commands +and their arguments as they are executed. .RE -.PP -\fBauto_recover\fR or \fBautomatic\fR +.sp +\fBauto_recover\fP or \fBautomatic\fP .RS 4 -Launch -\fIrear recover\fR -automatically (without automated reboot)\&. Together with -\fIdebug\fR -it runs -\fIrear recover\fR -in debug mode\&. +Launch \(aqrear recover\(aq automatically (without automated reboot). +Together with \(aqdebug\(aq it runs \(aqrear recover\(aq in debug mode. .RE -.PP -\fBunattended\fR +.sp +\fBunattended\fP .RS 4 -Launch -\fIrear recover\fR -automatically as with -\fIauto_recover\fR -plus automated reboot after successful -\fIrear recover\fR\&. +Launch \(aqrear recover\(aq automatically as with \(aqauto_recover\(aq +plus automated reboot after successful \(aqrear recover\(aq. .RE -.PP -\fBip= nm= netdev= gw=\fR +.sp +\fBip= nm= netdev= gw=\fP .RS 4 -If IP address plus optionally netmask (default 255\&.255\&.255\&.0), network device (default eth0), and a default gateway are specified only that single network device is set up\&. Example: +If IP address plus optionally netmask (default 255.255.255.0), +network device (default eth0), and a default gateway are specified +only that single network device is set up. Example: .RE .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf -ip=192\&.168\&.100\&.2 nm=255\&.255\&.255\&.0 netdev=eth0 gw=192\&.168\&.100\&.1 +ip=192.168.100.2 nm=255.255.255.0 netdev=eth0 gw=192.168.100.1 .fi -.if n \{\ -.RE -.\} -.PP -\fBnoip\fR +.if n .RE +.sp +\fBnoip\fP .RS 4 -Skip network devices setup by the rescue system start up scripts (overrides ip= nm= netdev= gw=)\&. +Skip network devices setup by the rescue system start up scripts (overrides ip= nm= netdev= gw=). .RE .SH "BACKUP SOFTWARE INTEGRATION" .sp -Currently Relax\-and\-Recover supports the following backup methods\&. Please distinguish carefully between Relax\-and\-Recover support for 3rd party backup software and Relax\-and\-Recover internal backup methods\&. The latter also creates a backup of your data while the former will only integrate Relax\-and\-Recover with the backup software to restore the data with the help of the backup software without actually creating backups\&. This means that for all non\-internal backup software you \fBmust\fR take care of creating backups yourself (unless otherwise noted)\&. +Currently Relax\-and\-Recover supports the following backup methods. Please +distinguish carefully between Relax\-and\-Recover support for 3rd party +backup software and Relax\-and\-Recover internal backup methods. The latter +also creates a backup of your data while the former will only integrate +Relax\-and\-Recover with the backup software to restore the data with the +help of the backup software without actually creating backups. This means +that for all non\-internal backup software you \fBmust\fP take care of creating +backups yourself (unless otherwise noted). .sp -Especially the rear mkbackup command can be confusing as it is only useful for the internal backup methods and has usually no function at all with the other (external) backup methods\&. +Especially the rear mkbackup command can be confusing as it is only +useful for the internal backup methods and has usually no function at all with +the other (external) backup methods. .sp -The following backup methods need to be set in Relax\-and\-Recover with the BACKUP option\&. As mentioned we have two types of BACKUP methods \- \fIinternal\fR and \fIexternal\fR\&. +The following backup methods need to +be set in Relax\-and\-Recover with the BACKUP option. As mentioned we have +two types of BACKUP methods \- \fIinternal\fP and \fIexternal\fP. .sp -The following BACKUP methods are \fIexternal\fR of Relax\-and\-Recover meaning that usually you are responsible of backups being made: -.PP -BACKUP=\fBREQUESTRESTORE\fR +The following BACKUP methods are \fIexternal\fP of Relax\-and\-Recover meaning +that usually you are responsible of backups being made: +.sp +BACKUP=\fBREQUESTRESTORE\fP .RS 4 -\fB(default)\fR -Not really a backup method at all, Relax\-and\-Recover simply halts the recovery and requests that somebody will restore the data to the appropriate location (e\&.g\&. via SSH)\&. This method works especially well with an rsync based backup that is pushed back to the backup client\&. +\fB(default)\fP Not really a backup method at all, Relax\-and\-Recover simply +halts the recovery and requests that somebody will restore the data +to the appropriate location (e.g. via SSH). This method works especially +well with an rsync based backup that is pushed back to the backup +client. .RE -.PP -BACKUP=\fBEXTERNAL\fR +.sp +BACKUP=\fBEXTERNAL\fP .RS 4 -Internal backup method that uses an arbitrary external command to create a backup and restore the data\&. +Internal backup method that uses an arbitrary external command to +create a backup and restore the data. .RE -.PP -BACKUP=\fBDP\fR +.sp +BACKUP=\fBDP\fP .RS 4 -Use Micro Focus Data Protector to restore the data\&. +Use Micro Focus Data Protector to restore the data. .RE -.PP -BACKUP=\fBFDRUPSTREAM\fR +.sp +BACKUP=\fBFDRUPSTREAM\fP .RS 4 -Use FDR/Upstream to restore the data\&. +Use FDR/Upstream to restore the data. .RE -.PP -BACKUP=\fBGALAXY\fR +.sp +BACKUP=\fBGALAXY\fP .RS 4 -Use CommVault Galaxy 5 to restore the data\&. +Use CommVault Galaxy 5 to restore the data. .RE -.PP -BACKUP=\fBGALAXY7\fR +.sp +BACKUP=\fBGALAXY7\fP .RS 4 -Use CommVault Galaxy 7 to restore the data\&. +Use CommVault Galaxy 7 to restore the data. .RE -.PP -BACKUP=\fBGALAXY10\fR +.sp +BACKUP=\fBGALAXY10\fP .RS 4 -Use CommVault Galaxy 10 (or Simpana 10) to restore the data\&. +Use CommVault Galaxy 10 (or Simpana 10) to restore the data. .RE -.PP -BACKUP=\fBNBU\fR +.sp +BACKUP=\fBNBU\fP .RS 4 -Use Symantec NetBackup to restore the data\&. +Use Symantec NetBackup to restore the data. .RE -.PP -BACKUP=\fBTSM\fR +.sp +BACKUP=\fBTSM\fP .RS 4 -Use IBM Tivoli Storage Manager to restore the data\&. The Relax\-and\-Recover result files (e\&.g\&. ISO image) are also saved into TSM\&. There is a (currently experimental) first draft implementation that a TSM backup is created by calling -\fIdsmc incremental\fR -during -\fIrear mkbackup\fR\&. +Use IBM Tivoli Storage Manager to restore the data. The Relax\-and\-Recover +result files (e.g. ISO image) are also saved into TSM. +There is a (currently experimental) first draft implementation +that a TSM backup is created by calling \(aqdsmc incremental\(aq during \(aqrear mkbackup\(aq. .RE -.PP -BACKUP=\fBNSR\fR +.sp +BACKUP=\fBNSR\fP .RS 4 -Using EMC NetWorker (Legato) to restore the data\&. +Using EMC NetWorker (Legato) to restore the data. .RE -.PP -BACKUP=\fBAVA\fR +.sp +BACKUP=\fBAVA\fP .RS 4 -Using EMC Avamar to restore the data\&. +Using EMC Avamar to restore the data. .RE -.PP -BACKUP=\fBSESAM\fR +.sp +BACKUP=\fBSESAM\fP .RS 4 -Using SEP Sesam to restore the data\&. +Using SEP Sesam to restore the data. .RE -.PP -BACKUP=\fBNBKDC\fR +.sp +BACKUP=\fBNBKDC\fP .RS 4 -Using NovaStor DC to restore the data\&. +Using NovaStor DC to restore the data. .RE -.PP -BACKUP=\fBCDM\fR +.sp +BACKUP=\fBCDM\fP .RS 4 -Using Rubrik Cloud Data Management (CDM) to restore the data\&. +Using Rubrik Cloud Data Management (CDM) to restore the data. .RE -.PP -BACKUP=\fBRBME\fR +.sp +BACKUP=\fBRBME\fP .RS 4 -Use Rsync Backup Made Easy (rbme) to restore the data\&. +Use Rsync Backup Made Easy (rbme) to restore the data. .RE -.PP -BACKUP=\fBBAREOS\fR +.sp +BACKUP=\fBBAREOS\fP .RS 4 -Use Open Source backup solution BAREOS (a fork a BACULA) to restore the data\&. +Use Open Source backup solution BAREOS (a fork a BACULA) to restore the data. .RE -.PP -BACKUP=\fBBACULA\fR +.sp +BACKUP=\fBBACULA\fP .RS 4 -Use Open Source backup solution BACULA to restore the data\&. +Use Open Source backup solution BACULA to restore the data. .RE -.PP -BACKUP=\fBDUPLICITY\fR +.sp +BACKUP=\fBDUPLICITY\fP .RS 4 -Use encrypted bandwidth\-efficient backup solution using the rsync algorithm to restore the data\&. +Use encrypted bandwidth\-efficient backup solution using the rsync algorithm to restore the data. .RE -.PP -BACKUP=\fBBORG\fR +.sp +BACKUP=\fBBORG\fP .RS 4 -Use BorgBackup (short Borg) a deduplicating backup program to restore the data\&. Executing -\fIrear mkbackup\fR -will create a Borg backup, see the section -\fIReaR with Borg back end\fR -in the ReaR user\-guide -\fIScenarios\fR -documentation\&. +Use BorgBackup (short Borg) a deduplicating backup program to restore the data. +Executing \(aqrear mkbackup\(aq will create a Borg backup, +see the section \(aqReaR with Borg back end\(aq in the ReaR user\-guide \(aqScenarios\(aq documentation. .RE .sp -The following BACKUP methods are \fIinternal\fR of Relax\-and\-Recover: -.PP -BACKUP=\fBNETFS\fR +The following BACKUP methods are \fIinternal\fP of Relax\-and\-Recover: +.sp +BACKUP=\fBNETFS\fP .RS 4 -Internal backup method which can be used to create a simple backup (tar archive)\&. +Internal backup method which can be used to create a simple backup +(tar archive). .RE -.PP -BACKUP=\fBRSYNC\fR +.sp +BACKUP=\fBRSYNC\fP .RS 4 -Use rsync to restore data\&. +Use rsync to restore data. .RE .sp -If your favourite backup software is missing from this list, please submit a patch or ask us to implement it for you\&. +If your favourite backup software is missing from this list, please submit +a patch or ask us to implement it for you. .sp -When using BACKUP=NETFS you must provide the backup target location through the BACKUP_URL variable\&. Possible BACKUP_URL settings are: -.PP -BACKUP_URL=\fBfile://\fR +When using BACKUP=NETFS you must provide the backup target location +through the BACKUP_URL variable. Possible BACKUP_URL settings are: +.sp +BACKUP_URL=\fBfile://\fP .RS 4 -To backup to local disk, use -BACKUP_URL=file:///directory/path/ +To backup to local disk, use BACKUP_URL=file:///directory/path/ .RE -.PP -BACKUP_URL=\fBnfs://\fR +.sp +BACKUP_URL=\fBnfs://\fP .RS 4 -To backup to NFS disk, use -BACKUP_URL=nfs://nfs\-server\-name/share/path +To backup to NFS disk, use BACKUP_URL=nfs://nfs\-server\-name/share/path .RE -.PP -BACKUP_URL=\fBtape://\fR +.sp +BACKUP_URL=\fBtape://\fP .RS 4 -To backup to tape device, use -BACKUP_URL=tape:///dev/nst0 -or alternatively, simply define -TAPE_DEVICE=/dev/nst0 +To backup to tape device, use BACKUP_URL=tape:///dev/nst0 or alternatively, +simply define TAPE_DEVICE=/dev/nst0 .RE -.PP -BACKUP_URL=\fBrsync://\fR -.RS 4 -When backup method -BACKUP=RSYNC -is chosen then we need to define a corresponding -BACKUP_URL -rule: .sp -.if n \{\ +BACKUP_URL=\fBrsync://\fP .RS 4 -.\} +When backup method BACKUP=RSYNC is chosen then we need to define a corresponding BACKUP_URL rule: +.sp +.if n .RS 4 .nf BACKUP_URL=rsync://[user@]host[:port]/path BACKUP_URL=rsync://[user@]host[:port]::/path .fi -.if n \{\ -.RE -.\} +.if n .RE .RE -.PP -BACKUP_URL=\fBcifs://\fR +.sp +BACKUP_URL=\fBcifs://\fP .RS 4 To backup to a Samba share (CIFS), use -BACKUP_URL=cifs://cifs\-server\-name/share/path\&. To provide credentials for CIFS mounting use a -\fI/etc/rear/cifs\fR -credentials file and define -BACKUP_OPTIONS="cred=/etc/rear/cifs" -and pass along: +BACKUP_URL=cifs://cifs\-server\-name/share/path. To provide credentials for +CIFS mounting use a \fI/etc/rear/cifs\fP credentials file and define +BACKUP_OPTIONS="cred=/etc/rear/cifs" and pass along: .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf username=_username_ password=_secret password_ domain=_domain_ .fi -.if n \{\ -.RE -.\} +.if n .RE .RE -.PP -BACKUP_URL=\fBusb://\fR +.sp +BACKUP_URL=\fBusb://\fP .RS 4 -To backup to USB storage device, use -BACKUP_URL=usb:///dev/disk/by\-label/REAR\-000 -or use a partition device node or a specific filesystem label\&. Alternatively, you may specify -the partition device using USB_DEVICE=/dev/disk/by\-label/REAR\-000 if needed\&. +To backup to USB storage device, use BACKUP_URL=usb:///dev/disk/by\-label/REAR\-000 +or use a partition device node or a specific filesystem label. Alternatively, you +may specify the partition device using USB_DEVICE=/dev/disk/by\-label/REAR\-000 if needed. .sp -If you combine this with -OUTPUT=USB -you will end up with a bootable USB device\&. +If you combine this with OUTPUT=USB you will end up with a bootable USB +device. .RE -.PP -BACKUP_URL=\fBsshfs://\fR +.sp +BACKUP_URL=\fBsshfs://\fP .RS 4 To backup to a remote server via sshfs (SSH protocol), use -BACKUP_URL=sshfs://user@remote\-system\&.name\&.org/home/user/backup\-dir/ +BACKUP_URL=sshfs://user@remote\-system.name.org/home/user/backup\-dir/ .sp -It is advisable to add -\fBServerAliveInterval 15\fR -in the -/root/\&.ssh/config -file for the remote system (remote\-system\&.name\&.org)\&. +It is advisable to add \fBServerAliveInterval 15\fP in the /root/.ssh/config +file for the remote system (remote\-system.name.org). .RE -.PP -BACKUP_URL=\fBiso://\fR -.RS 4 -To include the backup within the ISO image\&. It is important that the -BACKUP_URL -and -OUTPUT_URL -variables are different\&. E\&.g\&. .sp -.if n \{\ +BACKUP_URL=\fBiso://\fP .RS 4 -.\} +To include the backup within the ISO image. It is important that the BACKUP_URL and +OUTPUT_URL variables are different. E.g. +.sp +.if n .RS 4 .nf BACKUP_URL=iso:///backup/ OUTPUT_URL=nfs://server/path/ .fi -.if n \{\ -.RE -.\} +.if n .RE .RE .sp -When using BACKUP=NETFS and BACKUP_PROG=tar there is an option to select BACKUP_TYPE=incremental or BACKUP_TYPE=differential to let rear make incremental or differential backups until the next full backup day e\&.g\&. via FULLBACKUPDAY="Mon" is reached or when the last full backup is too old after FULLBACKUP_OUTDATED_DAYS has passed\&. Incremental or differential backup is currently only known to work with BACKUP_URL=nfs\&. Other BACKUP_URL schemes may work but at least BACKUP_URL=usb requires USB_SUFFIX to be set to work with incremental or differential backup\&. -.SH "SUPPORT FOR SELF-ENCRYPTING DISKS" +When using BACKUP=NETFS and BACKUP_PROG=tar there is an option to select +BACKUP_TYPE=incremental or BACKUP_TYPE=differential to let rear make +incremental or differential backups until the next full backup day +e.g. via FULLBACKUPDAY="Mon" is reached or when the last full backup +is too old after FULLBACKUP_OUTDATED_DAYS has passed. +Incremental or differential backup is currently only known to work +with BACKUP_URL=nfs. Other BACKUP_URL schemes may work but +at least BACKUP_URL=usb requires USB_SUFFIX to be set +to work with incremental or differential backup. +.SH "SUPPORT FOR SELF\-ENCRYPTING DISKS" .sp -Relax\-and\-Recover supports self\-encrypting disks (SEDs) compliant with the TCG Opal 2 specification if the sedutil\-cli executable is installed\&. +Relax\-and\-Recover supports self\-encrypting disks (SEDs) compliant with the TCG +Opal 2 specification if the \f(CRsedutil\-cli\fP executable is installed. .sp Self\-encrypting disk support includes .sp @@ -690,8 +678,8 @@ Self\-encrypting disk support includes \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} recovery (saving and restoring the system\(cqs SED configuration), .RE @@ -701,8 +689,8 @@ recovery (saving and restoring the system\(cqs SED configuration), \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} setting up SEDs, including assigning a disk password, .RE @@ -712,172 +700,205 @@ setting up SEDs, including assigning a disk password, \h'-04'\(bu\h'+03'\c .\} .el \{\ -.sp -1 -.IP \(bu 2.3 +. sp -1 +. IP \(bu 2.3 .\} -providing a pre\-boot authentication (PBA) system to unlock SEDs at boot time\&. +providing a pre\-boot authentication (PBA) system to unlock SEDs at boot time. .RE .sp -To prepare booting from an SED, run rear mkopalpba, then create the rescue system\&. +To prepare booting from an SED, run rear mkopalpba, then create the rescue +system. .sp -To set up an SED, boot the Relax\-and\-Recover rescue system and run rear opaladmin setupERASE DEVICE (\fIDEVICE\fR being the disk device path like /dev/sda)\&. +To set up an SED, boot the Relax\-and\-Recover rescue system and run \f(CRrear +opaladmin setupERASE DEVICE\fP (\fIDEVICE\fP being the disk device path like +\f(CR/dev/sda\fP). .sp -For complete information, consult the section "Support for TCG Opal 2\-compliant Self\-Encrypting Disks" in the Relax\-and\-Recover user guide\&. +For complete information, consult the section "Support for TCG Opal 2\-compliant +Self\-Encrypting Disks" in the Relax\-and\-Recover user guide. .SH "CONFIGURATION" .sp -To configure Relax\-and\-Recover you have to edit the configuration files in \fI/etc/rear/\fR\&. All \fI*\&.conf\fR files there are part of the configuration, but only \fIsite\&.conf\fR and \fIlocal\&.conf\fR are intended for the user configuration\&. All other configuration files hold defaults for various distributions and should not be changed\&. +To configure Relax\-and\-Recover you have to edit the configuration files in +\fI/etc/rear/\fP. All \fI*.conf\fP files there are part of the configuration, but +only \fIsite.conf\fP and \fIlocal.conf\fP are intended for the user configuration. +All other configuration files hold defaults for various distributions and +should not be changed. .sp -In \fI/etc/rear/templates/\fR there are also some template files which are used by Relax\-and\-Recover to create configuration files (mostly for the boot environment)\&. Modify the templates to adjust the information contained in the emails produced by Relax\-and\-Recover\&. You can use these templates to prepend your own configurations to the configuration files created by Relax\-and\-Recover, for example you can edit \fIPXE_pxelinux\&.cfg\fR to add some general pxelinux configuration you use\&. +In \fI/etc/rear/templates/\fP there are also some template files which are used +by Relax\-and\-Recover to create configuration files (mostly for the boot +environment). Modify the templates to adjust the information contained in +the emails produced by Relax\-and\-Recover. You can use these templates to +prepend your own configurations to the configuration files created by +Relax\-and\-Recover, for example you can edit \fIPXE_pxelinux.cfg\fP to add some +general pxelinux configuration you use. .sp -In almost all circumstances you have to configure two main settings and their parameters: The backup method and the output method\&. +In almost all circumstances you have to configure two main settings and their +parameters: The backup method and the output method. .sp -The backup method defines, how your data is to be saved and whether Relax\-and\-Recover should backup your data as part of the mkrescue process or whether you use an external application, e\&.g\&. backup software to archive your data\&. +The backup method defines, how your data is to be saved and whether Relax\-and\-Recover +should backup your data as part of the mkrescue process or whether you use an +external application, e.g. backup software to archive your data. .sp -The output method defines how the rescue system is written to disk and how you plan to boot the failed computer from the rescue system\&. +The output method defines how the rescue system is written to disk and how you +plan to boot the failed computer from the rescue system. .sp -See the default configuration file \fI/usr/share/rear/conf/default\&.conf\fR for an overview of the possible methods and their options\&. +See the default configuration file \fI/usr/share/rear/conf/default.conf\fP for +an overview of the possible methods and their options. .sp -An example to use TSM for backup and ISO for output would be to add these lines to \fI/etc/rear/local\&.conf\fR (no need to define a BACKUP_URL when using an external backup solution): +An example to use TSM for backup and ISO for output would be to add +these lines to \fI/etc/rear/local.conf\fP (no need to define a BACKUP_URL +when using an external backup solution): .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf BACKUP=TSM OUTPUT=ISO .fi -.if n \{\ -.RE -.\} +.if n .RE .sp -And if all your systems use NTP for time synchronisation, you can also add these lines to \fI/etc/rear/site\&.conf\fR +And if all your systems use NTP for time synchronisation, you can also +add these lines to \fI/etc/rear/site.conf\fP .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf TIMESYNC=NTP .fi -.if n \{\ -.RE -.\} +.if n .RE .sp -Do not forget to distribute the \fIsite\&.conf\fR to all your systems\&. +Do not forget to distribute the \fIsite.conf\fP to all your systems. .sp -The resulting ISO image will be created in \fI/var/lib/rear/output/\fR\&. You can now modify the behaviour by copying the appropriate configuration variables from \fIdefault\&.conf\fR to \fIlocal\&.conf\fR and changing them to suit your environment\&. +The resulting ISO image will be created in \fI/var/lib/rear/output/\fP. +You can now modify the behaviour by copying the appropriate +configuration variables from \fIdefault.conf\fP to \fIlocal.conf\fP and +changing them to suit your environment. .SH "EXIT STATUS" -.PP +.sp 0 .RS 4 -Successful program execution\&. +Successful program execution. .RE -.PP +.sp >0 .RS 4 -Usage, syntax or execution errors\&. Check the log file in -\fI/var/log/rear/\fR -for more information\&. +Usage, syntax or execution errors. Check the log file in +\fI/var/log/rear/\fP for more information. .RE .SH "EXAMPLES" .sp -To print out the current settings for BACKUP and OUTPUT methods and some system information\&. This command can be used to see the supported features for the given release and platform\&. +To print out the current settings for BACKUP and OUTPUT methods and some +system information. This command can be used to see the supported features +for the given release and platform. .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf # rear dump .fi -.if n \{\ -.RE -.\} +.if n .RE .sp -To create a new rescue environment\&. Do not forget to copy the resulting rescue system away so that you can use it in the case of a system failure\&. +To create a new rescue environment. Do not forget to copy the resulting +rescue system away so that you can use it in the case of a system failure. .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf # rear \-v mkrescue .fi -.if n \{\ -.RE -.\} +.if n .RE .sp -To create a new rescue image together with a complete archive of your local system run the command: +To create a new rescue image together with a complete archive of your local +system run the command: .sp -.if n \{\ -.RS 4 -.\} +.if n .RS 4 .nf # rear \-v mkbackup .fi -.if n \{\ -.RE -.\} +.if n .RE .SH "FILES" -.PP +.sp /usr/sbin/rear .RS 4 -The program itself\&. +The program itself. .RE -.PP -/etc/rear/local\&.conf +.sp +/etc/rear/local.conf .RS 4 -System specific configuration can be set here\&. +System specific configuration can be set here. .RE -.PP -/etc/rear/site\&.conf +.sp +/etc/rear/site.conf .RS 4 -Site specific configuration can be set here (not created by default)\&. +Site specific configuration can be set here (not created by default). .RE -.PP +.sp /var/log/rear/ .RS 4 -Directory holding the log files\&. +Directory holding the log files. .RE -.PP -/tmp/rear\&.#### +.sp +/tmp/rear.#### .RS 4 -Relax\-and\-Recover working directory\&. If Relax\-and\-Recover exits with an error, you must remove this directory manually\&. +Relax\-and\-Recover working directory. If Relax\-and\-Recover exits with an error, +you must remove this directory manually. .RE -.PP +.sp /usr/share/rear .RS 4 -Relax\-and\-Recover script components\&. +Relax\-and\-Recover script components. .RE -.PP -/usr/share/rear/conf/default\&.conf +.sp +/usr/share/rear/conf/default.conf .RS 4 -Relax\-and\-Recover default values\&. Contains a complete set of parameters and its explanation\&. Please do not edit or modify\&. Copy values to -\fIlocal\&.conf\fR -or -\fIsite\&.conf\fR -instead\&. +Relax\-and\-Recover default values. Contains a complete set of parameters +and its explanation. Please do not edit or modify. Copy values to +\fIlocal.conf\fP or \fIsite.conf\fP instead. .RE .SH "BUGS" .sp -Feedback is welcome, please report any issues or improvements to our issue\-tracker at: \m[blue]\fBhttp://github\&.com/rear/issues/\fR\m[] +Feedback is welcome, please report any issues or improvements to our +issue\-tracker at: \c +.URL "http://github.com/rear/issues/" "" "" .sp -Furthermore, we welcome pull requests via GitHub\&. +Furthermore, we welcome pull requests via GitHub. .SH "SEE ALSO" .sp -Relax\-and\-Recover comes with extensive documentation located in \fI/usr/share/doc\fR\&. +Relax\-and\-Recover comes with extensive documentation located in +\fI/usr/share/doc\fP. .SH "AUTHORS" .sp -The ReaR project was initiated in 2006 by Schlomo Schapiro (\m[blue]\fBhttps://github\&.com/schlomo\fR\m[]) and Gratien D\(cqhaese (\m[blue]\fBhttps://github\&.com/gdha\fR\m[]) and has since then seen a lot of contributions by many authors\&. As ReaR deals with bare metal disaster recovery, there is a large amount of code that was contributed by owners and users of specialized hardware and software\&. Without their combined efforts and contributions ReaR would not be the universal Linux bare metal disaster recovery solution that it is today\&. -.sp -As time passed the project was lucky to get the support of additional developers to also help as maintainers: Dag Wieers (\m[blue]\fBhttps://github\&.com/dagwieers\fR\m[]), Jeroen Hoekx (\m[blue]\fBhttps://github\&.com/jhoekx\fR\m[]), Johannes Meixner (\m[blue]\fBhttps://github\&.com/jsmeix\fR\m[]), Vladimir Gozora (\m[blue]\fBhttps://github\&.com/gozora\fR\m[]) and S\('ebastien Chabrolles (\m[blue]\fBhttps://github\&.com/schabrolles\fR\m[])\&. We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers\&. Please refer to the MAINTAINERS (\m[blue]\fBhttps://github\&.com/rear/rear/blob/master/MAINTAINERS\fR\m[]) file for the list of active and past maintainers\&. -.sp -To see the full list of authors and their contributions please look at the git history (\m[blue]\fBhttps://github\&.com/rear/rear/graphs/contributors\fR\m[])\&. We are very thankful to all authors and encourage anybody interested to take a look at our source code and to contribute what you find important\&. -.sp -Relax\-and\-Recover is a collaborative process using Github at \m[blue]\fBhttp://github\&.com/rear/\fR\m[] -.sp -The Relax\-and\-Recover website is located at: \m[blue]\fBhttp://relax\-and\-recover\&.org/\fR\m[] +The ReaR project was initiated in 2006 by Schlomo Schapiro (\c +.URL "https://github.com/schlomo" "" ")" +and Gratien D\(cqhaese ( +.URL "https://github.com/gdha" "" ")" +and has since then seen a lot of contributions by many authors. As ReaR deals with bare metal disaster recovery, there is a large amount of code that was contributed by owners and users of specialized hardware and software. Without their combined efforts and contributions ReaR would not be the universal Linux bare metal disaster recovery solution that it is today. +.sp +As time passed the project was lucky to get the support of additional developers to also help as maintainers: Dag Wieers (\c +.URL "https://github.com/dagwieers" "" ")," +Jeroen Hoekx ( +.URL "https://github.com/jhoekx" "" ")," +Johannes Meixner ( +.URL "https://github.com/jsmeix" "" ")," +Vladimir Gozora ( +.URL "https://github.com/gozora" "" ")" +and Sébastien Chabrolles ( +.URL "https://github.com/schabrolles" "" ")." +We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. Please refer to the MAINTAINERS ( +.URL "https://github.com/rear/rear/blob/master/MAINTAINERS" "" ")" +file for the list of active and past maintainers. +.sp +To see the full list of authors and their contributions please look at the git history (\c +.URL "https://github.com/rear/rear/graphs/contributors" "" ")." +We are very thankful to all authors and encourage anybody interested to take a look at our source code and to contribute what you find important. +.sp +Relax\-and\-Recover is a collaborative process using Github at \c +.URL "http://github.com/rear/" "" "" +.sp +The Relax\-and\-Recover website is located at: \c +.URL "http://relax\-and\-recover.org/" "" "" .SH "COPYRIGHT" .sp -(c) 2006\-2020 +(c) 2006\-2022 .sp -The copyright is held by the original authors of the respective code pieces as can be seen in the git history at \m[blue]\fBhttps://github\&.com/rear/rear/graphs/contributors\fR\m[] +The copyright is held by the original authors of the respective code pieces as can be seen in the git history at \c +.URL "https://github.com/rear/rear/graphs/contributors" "" "" .sp -Relax\-and\-Recover comes with ABSOLUTELY NO WARRANTY; for details see the GNU General Public License at \m[blue]\fBhttp://www\&.gnu\&.org/licenses/gpl\&.html\fR\m[] +Relax\-and\-Recover comes with ABSOLUTELY NO WARRANTY; for details +see the GNU General Public License at \c +.URL "http://www.gnu.org/licenses/gpl.html" "" "" \ No newline at end of file diff --git a/doc/rear.8.adoc b/doc/rear.8.adoc index 89941009ea..80afac41b7 100644 --- a/doc/rear.8.adoc +++ b/doc/rear.8.adoc @@ -98,7 +98,6 @@ the GNU General Public License at: http://www.gnu.org/licenses/gpl.html *format*:: format and label USB or tape media to be used with rear; - + first argument is the USB or tape device to use, eg. _/dev/sdX_ or _/dev/stX_ From 87ddeeaed6f225005ae19998fece6f1d5366eda6 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 9 Jun 2022 13:09:59 +0200 Subject: [PATCH 131/215] Added the changes in https://github.com/rear/rear/pull/2816 to make the automated Packit build test work --- packaging/rpm/rear.spec | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packaging/rpm/rear.spec b/packaging/rpm/rear.spec index d55b4707c2..9ed7704de0 100644 --- a/packaging/rpm/rear.spec +++ b/packaging/rpm/rear.spec @@ -42,6 +42,10 @@ Requires: syslinux # (in addition to the default installed bootloader grub2) while on ppc ppc64 the # default installed bootloader yaboot is also used to make the bootable ISO image. +# Needed for Packit builds, +# see https://github.com/rear/rear/pull/2816 +BuildRequires: make + ### Mandatory dependencies on all distributions: Requires: binutils Requires: ethtool @@ -160,6 +164,9 @@ fi %{__rm} -rf %{buildroot} %{__make} install DESTDIR="%{buildroot}" +%check +%{__make} validate + %files # defattr: is required for SLES 11 and RHEL/CentOS 5 builds on openSUSE Build Service (#2135) %defattr(-, root, root, 0755) From 35c1e316baff5418e4ab8fec04222565bb7480c2 Mon Sep 17 00:00:00 2001 From: antonvoznia Date: Sat, 19 Mar 2022 20:36:27 +0100 Subject: [PATCH 132/215] Add make in build requirements for RPM --- packaging/rpm/rear.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/rpm/rear.spec b/packaging/rpm/rear.spec index b2b20a1ecf..021ab04882 100644 --- a/packaging/rpm/rear.spec +++ b/packaging/rpm/rear.spec @@ -42,6 +42,8 @@ Requires: syslinux # (in addition to the default installed bootloader grub2) while on ppc ppc64 the # default installed bootloader yaboot is also used to make the bootable ISO image. +BuildRequires: make + ### Mandatory dependencies on all distributions: Requires: binutils Requires: ethtool From f0f840c82c083bcee72dd1d68588cabfba59fbd4 Mon Sep 17 00:00:00 2001 From: antonvoznia Date: Sat, 19 Mar 2022 20:51:10 +0100 Subject: [PATCH 133/215] Run make validate in RPM build --- packaging/rpm/rear.spec | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/rpm/rear.spec b/packaging/rpm/rear.spec index 021ab04882..daa48da23c 100644 --- a/packaging/rpm/rear.spec +++ b/packaging/rpm/rear.spec @@ -162,6 +162,9 @@ fi %{__rm} -rf %{buildroot} %{__make} install DESTDIR="%{buildroot}" +%check +%{__make} validate + %files # defattr: is required for SLES 11 and RHEL/CentOS 5 builds on openSUSE Build Service (#2135) %defattr(-, root, root, 0755) From c09ee23e5dc715257374853c31631b7c587d0b8c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 14 Jun 2022 16:16:22 +0200 Subject: [PATCH 134/215] Created doc/rear-release-notes.txt via: cat release-notes-2-7.md | pandoc -t plain --columns=78 | iconv -f UTF-8 -t ASCII//TRANSLIT >rear-release-notes.txt --- doc/rear-release-notes.txt | 3901 ++++++++++++++++++++++++------------ 1 file changed, 2570 insertions(+), 1331 deletions(-) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index ce403483d4..4ce8013c80 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -1,13 +1,4 @@ -Relax-and-Recover - - o Features - o Documentation - o Downloads - o Support - o Development - o Events - -Release Notes for Relax-and-Recover version 2.6 +Release Notes for Relax-and-Recover version 2.7 This document contains the release notes for the open source project Relax-and-Recover. @@ -22,14 +13,14 @@ go to http://creativecommons.org/licenses/by-nd/3.0/ Overview -Relax-and-Recover is a GNU/Linux system administrator tool and framework used -to create bootable disaster recovery images which makes bare metal disaster -recovery (including backup restore) easier. System administrators use the -Relax-and-Recover framework to set up a disaster recovery procedure as part -of their disaster recovery policy (which does not replace in any way a backup -policy). Relax-and-Recover does not implement backup but complements it -because backup (and restore) happens via external backup software that is -only called by Relax-and-Recover. +Relax-and-Recover (abbreviated ReaR) is a GNU/Linux system administrator tool +and framework used to create bootable disaster recovery images which makes +bare metal disaster recovery (including backup restore) easier. System +administrators use the Relax-and-Recover framework to set up a disaster +recovery procedure as part of their disaster recovery policy (which +complements their existing backup policy). Relax-and-Recover does not +implement backup but complements it because backup (and restore) happens via +external backup software that is only called by Relax-and-Recover. Product Features @@ -42,137 +33,138 @@ The most recent release of Relax-and-Recover is supported on most GNU/Linux based systems with kernel 2.6 or higher. It provides the following functionality: - o Hot maintenance capability. A recovery/rescue image can be made online +- Hot maintenance capability. A recovery/rescue image can be made online while the system is running - o Command line interface. Relax-and-Recover does not require a graphical +- Command line interface. Relax-and-Recover does not require a graphical interface to run, neither in creation mode, nor in recovery mode (console is enough) - o Support included for most common file systems, such as ext2, ext3, and +- Support included for most common file systems, such as ext2, ext3, and ext4. Other filesystems like reiserfs, jfs, xfs, and btrfs are also implemented, but are less tested. (Feedback is appreciated) - o Selected Hardware RAID and (eg. HP SmartArray) and mirroring solutions +- Selected Hardware RAID and (eg. HP SmartArray) and mirroring solutions (eg. DRBD) are supported - o NVME and mmcblk disks are supported +- NVME and mmcblk disks are supported - o LVM root volumes are supported +- LVM root volumes are supported - o Multipath support for SAN storage +- Multipath support for SAN storage - o UEFI support (including UEFI USB booting) +- UEFI support (including UEFI USB booting) - o Integrates with internal backup programs such as: +- Integrates with internal backup programs such as: - - GNU tar (BACKUP=NETFS, BACKUP_PROG=tar) - - GNU tar (BACKUP=NETFS, BACKUP_PROG=tar, BACKUP_TYPE=incremental, + - GNU tar (BACKUP=NETFS, BACKUP_PROG=tar) + - GNU tar (BACKUP=NETFS, BACKUP_PROG=tar, BACKUP_TYPE=incremental, FULLBACKUPDAY="Mon") for using incremental backups with a weekly full - backup. Be aware, old tar archives will not be removed automatically! - - GNU tar (BACKUP=NETFS, BACKUP_PROG=tar, BACKUP_TYPE=differential, - FULLBACKUPDAY="Mon") for using differential backups with a weekly - full backup. Be aware, old tar archives will not be removed - automatically! - - GNU tar with openssl encryption (BACKUP=NETFS, BACKUP_PROG=tar, + backup (old tar archives will not be removed automatically) + - GNU tar (BACKUP=NETFS, BACKUP_PROG=tar, BACKUP_TYPE=differential, + FULLBACKUPDAY="Mon") for using differential backups with a weekly full + backup (old tar archives will not be removed automatically) + - GNU tar with openssl encryption (BACKUP=NETFS, BACKUP_PROG=tar, BACKUP_PROG_CRYPT_ENABLED=1) - - rsync on local devices (BACKUP=NETFS, BACKUP_PROG=rsync), such USB - and local disks - - rsync over the network (BACKUP=RSYNC, BACKUP_PROG=rsync) - - Multiple backup methods (read the documentation) - - Any partition (e.g. a Windows partition) via BACKUP=BLOCKCLONE. See + - rsync on local devices (BACKUP=NETFS, BACKUP_PROG=rsync), such USB and + local disks + - rsync over the network (BACKUP=RSYNC, BACKUP_PROG=rsync) + - Multiple backup methods. See the documentation about Using Multiple + Backups for Relax-and-Recover + - Any partition (e.g. a Windows partition) via BACKUP=BLOCKCLONE. See the documention about BLOCKCLONE - - BACKUP=ZYPPER is SLES12 only (Experimental) - - BACKUP=YUM is for RedHat architectures ony (Experimental) - o Integrates with external backup solutions such as: - - - Tivoli Storage Manager (BACKUP=TSM) - - Data Protector (BACKUP=DP) - - Symantec NetBackup (BACKUP=NBU) - - Galaxy 5, 6, and 7 (BACKUP=GALAXY) - - Galaxy 10 [Commvault Simpana] (BACKUP=GALAXY10) - - Bacula (BACKUP=BACULA) - - Bareos (BACKUP=BAREOS) (A fork of Bacula) - - Rsync Backup Made Easy (BACKUP=RBME) - - Duplicity/Duply (BACKUP=DUPLICITY) - - EMC Networker, also known as Legato (BACKUP=NSR) - - EMC Avamar (BACKUP=AVA) - - SEP Sesam (BACKUP=SESAM) - - FDR/Upstream (BACKUP=FDRUPSTREAM) - - NovaStor DataCenter (BACKUP=NBKDC) - - Borg Backup (BACKUP=BORG) - - Rubrik Cloud Data Management (BACKUP=CDM) (New) - o Integrates with Disaster Recovery Linux Manager (DRLM) - - o Udev support (except for some really ancient udev versions) which is the + - BACKUP=ZYPPER is for SUSE systems only (Experimental) + - BACKUP=YUM is for RedHat architectures ony (Experimental) + +- Integrates with external backup solutions such as: + + - Tivoli Storage Manager (BACKUP=TSM) + - Data Protector (BACKUP=DP) + - Symantec NetBackup (BACKUP=NBU) + - Galaxy 5, 6, and 7 (BACKUP=GALAXY) + - Galaxy 10 [Commvault Simpana] (BACKUP=GALAXY10) + - Bacula (BACKUP=BACULA) + - Bareos (BACKUP=BAREOS) (A fork of Bacula) + - Rsync Backup Made Easy (BACKUP=RBME) + - Duplicity/Duply (BACKUP=DUPLICITY) + - EMC Networker, also known as Legato (BACKUP=NSR) + - EMC Avamar (BACKUP=AVA) + - SEP Sesam (BACKUP=SESAM) + - FDR/Upstream (BACKUP=FDRUPSTREAM) + - Novastor NovaBACKUP DC (BACKUP=NBKDC) + - Borg Backup (BACKUP=BORG) + - Rubrik Cloud Data Management (BACKUP=CDM) + +- Integrates with Disaster Recovery Linux Manager (DRLM) + +- Udev support (except for some really ancient udev versions) which is the base for many important features: - - kernel drivers for network cards and storage adapters are loaded via + - kernel drivers for network cards and storage adapters are loaded via udev - - deal with network persistent names in udev rules - - firmware loading - - persistent storage device names (though Relax-and-Recover does - nothing with this) - o Systemd support for the more recent Linux distributions + - deal with network persistent names in udev rules + - firmware loading + - persistent storage device names (though Relax-and-Recover does nothing + with this) + +- Systemd support for the more recent Linux distributions - o System migration and reconfiguration ('MIGRATION_MODE') +- System migration and reconfiguration ('MIGRATION_MODE') - - facilitate recovery on hardware, that is not the same as the original + - facilitate recovery on hardware, that is not the same as the original system - - network and storage drivers are adjusted - - map hard disks if they do not match (e.g. hda -> sda) - - remap network MAC addresses - - use another IP address, or using dhcp via templates or from kernel + - network and storage drivers are adjusted + - map hard disks if they do not match (e.g. sda -> sdb) + - remap network MAC addresses + - use another IP address, or using dhcp via templates or from kernel command line - - rebuild the initial ramdisk if needed (for new storage drivers) - - migration to SAN storaged - o Support backup software: Bacula, both locally attached tapes (with - bextract) and network-based backups. Also, in combination with OBDR - tapes. + - rebuild the initial ramdisk if needed (for new storage drivers) + - migration to SAN storage - o Create OBDR tapes with method mkbackup and put the backup onto the tape - to have a single-tape bootable recovery solution +- Support backup software: Bacula, both locally attached tapes (with + bextract) and network-based backups. Also, in combination with OBDR tapes. - o Label the OBDR tape with the method format to avoid accidental overwrites - with OBDR +- Create OBDR tapes with method mkbackup and put the backup onto the tape to + have a single-tape bootable recovery solution (label the OBDR tape with + the method format to avoid accidental overwrites with OBDR) - o Create bootable disk (eSATA, USB ...) medium with the backup included: +- Create bootable disk (eSATA, USB ...) medium with the backup included: BACKUP_URL=usb:///dev/device Together with OUTPUT=USB we have a complete solution on hard disks (booting of it and restoring data). - o DHCP client support (IPv4 and IPv6). Dhcp client activation can be forced +- DHCP client support (IPv4 and IPv6). Dhcp client activation can be forced via the variable USE_DHCLIENT=yes (define in /etc/rear/local.conf). It is also possible to force DHCP at boot time with kernel option dhcp - o USE_STATIC_NETWORKING=y, will cause statically configured network - settings to be applied even when USE_DHCLIENT is in effect +- USE_STATIC_NETWORKING=y, will cause statically configured network settings + to be applied even when USE_DHCLIENT is in effect - o Save layout and compare layouts for automation of making - Relax-and-Recover snapshots (checklayout option) +- Save layout and compare layouts for automation of making Relax-and-Recover + snapshots (checklayout option) - o External USB booting uses extlinux (instead of syslinux), and therefore, +- External USB booting uses extlinux (instead of syslinux), and therefore, the USB disk must first be formatted with an ext2, ext3, ext4 or btrfs based file system - o VLAN tagging, teaming and bridge support +- VLAN tagging, teaming and bridge support - o Add timestamp of ReaR run with rc code to the syslog or messages file; +- Add timestamp of ReaR run with rc code to the syslog or messages file; sending mail report is also possible - o The possibility to backup any partition (in particular a Windows +- The possibility to backup any partition (in particular a Windows partition) via the BACKUP type BLOCKCLONE - o Unattended ReaR recovery has been improved +- Unattended ReaR recovery has been improved - o Improved security model related to SSH keys +- Improved security model related to SSH keys - - SSH_FILES='avoid_sensitive_files' (see details in /usr/share/rear/ - conf/default.conf) - - SSH_UNPROTECTED_PRIVATE_KEYS='no' (see details in /usr/share/rear/ - conf/default.conf) + - SSH_FILES='avoid_sensitive_files' (see details in + /usr/share/rear/conf/default.conf) + - SSH_UNPROTECTED_PRIVATE_KEYS='no' (see details in + /usr/share/rear/conf/default.conf) NOTE: Features marked Experimental are prone to change with future releases. @@ -182,41 +174,1297 @@ The first release of Relax-and-Recover, version 1.0, was posted to the web in July 2006. For each release, this chapter lists the new features and defect fixes. All releases are cumulative. Unless otherwise noted all releases of Relax-and-Recover are intended to work backward compatible with previous -versions. In addition to the GPL disclaimer of warranty and liability there -is no guarantee that things work backward compatible. In general the older -the system is the less likely it is that a newer Relax-and-Recover version -works. For each Relax-and-Recover version upgrade and for each change of a -software that is used by Relax-and-Recover and for each change of your basic -system you must re-validate that your disaster recovery procedure still works -for you. +versions. In addition to the GPL disclaimer of warranty and liability there is +no guarantee that things work backward compatible. In general the older the +system is the less likely it is that a newer Relax-and-Recover version works. +For each Relax-and-Recover version upgrade and for each change of a software +that is used by Relax-and-Recover and for each change of your basic system you +must re-validate that your disaster recovery procedure still works for you. -The references pointing to fix #nr or issue #nr refer to our issues tracker. +The references pointing to fix #nr or issue #nr refer to our GitHub issues +tracker. + +Version 2.7 (June 2022) + +Abstract + +New features, bigger enhancements, and possibly backward incompatible changes: + +- ReaR's default TMPDIR is now /var/tmp (it was /tmp before) i.e. ReaR's + default working area is now /var/tmp/rear.XXXXX because ReaR needs lots of + space (from 300MB up to more than 1GB) to build the ReaR rescue/recovery + system e.g. as ISO image and even much more space to also store the backup + archive e.g. when the backup should be included in the ISO image. The + system's default temporary directory /tmp is no longer suited as default + temporary directory for ReaR because nowadays /tmp is often a "tmpfs" that + is in RAM so /tmp has RAM/swap limitations and file-hierarchy(7) + recommends to use /var/tmp for cases as ReaR. If the user sets TMPDIR + explicitly, it still takes precedence. + +- Stop ReaR from possibly overwriting its own disk and backup drives for + OUTPUT=USB and OUTPUT=RAWDISK via new WRITE_PROTECTED_... config variables + (see /usr/share/rear/conf/default.conf) where UUIDs or filesystem labels + can be specified so that disks that contain such UUIDs or filesystem + labels will be 'write protected' during "rear recover". + +- Added initial LUKS2 support. + +- Overhauled RAID code with changed RAID related entries in + /var/lib/rear/layout/disklayout.conf so users who use RAID and a selfmade + /etc/rear/disklayout.conf must adapt their RAID related entries + +- Initial preliminary basic support to automatically resize an active last + partition on RAID0 and RAID1 disks to be able to automatically resize + RAID0 and RAID1 arrays + +- Automatically shrink LVs if needed during "rear recover". This enables to + run "rear recover" with automated LVM LVs shrinking as needed on a bit + smaller replacement disk (e.g. when a nominally same sized replacement + disk is actually a bit smaller than the original disk). This automated LVs + shrinking is not intended when disk(s) are substantially smaller. To + migrate onto a substantially smaller replacement disk the user must in + advance manually adapt his disklayout.conf file before he runs "rear + recover". + +- Support to wipe disks before recreating partitions/volumes/filesystems/... + (see the new DISKS_TO_BE_WIPED in /usr/share/rear/conf/default.conf). This + is currently new and experimental functionality so that currently by + default via DISKS_TO_BE_WIPED='false' no disk is wiped to avoid possible + regressions until this new feature was more tested by interested users via + an explicit DISKS_TO_BE_WIPED setting. + +- Error out when files greater or equal ISO_FILE_SIZE_LIMIT should be + included in the ISO: See the reasoning in + /usr/share/rear/conf/default.conf why the default ISO_FILE_SIZE_LIMIT is + 2GiB and why we error out when files >= ISO_FILE_SIZE_LIMIT should be + included in the ISO. + +- RAWDISK: Add local rescue partition installation capability: Introduce a + configuration variable 'RAWDISK_INSTALL_GPT_PARTITION_NAME', which is + unset by default. When set, the rescue system will be installed to local + disk partitions having the corresponding name. + +- Allow setting a PBA-specific firmware configuration via the new + OPAL_PBA_FIRMWARE_FILES configuration variable. + +- Support OPAL 2 self-encrypting NVMe disk drives. + +- OPALPBA: Provide a permanent unlocking mode as a workaround when the + firmware did not initialize properly during a 'simple' reboot so the only + reliable way to boot was a power cycle after Opal disks were unlocked. To + reboot with unlocked disks there is now a new unlocking mode "permanent" + via OPAL_PBA_UNLOCK_MODE (see default.conf). Reactivating locking is then + the responsibility of the user. Additionally enhanced security by + disabling shell access via keyboard interrupt and switching to a password + hash for OPAL_PBA_DEBUG_PASSWORD. + +- POWER architecture (ppc64) support for BACKUP=DP: Other platforms + supporting the Data Protector Disk Agent, but not the Cell Console (GUI) + that is only available on i386, x86_64 and ia64 are now supported with + ReaR. This includes in particular ppc64. Checks unavailable on clients + without the Data Protector Cell Console (GUI) are skipped. In such cases + backup restore can be done using Data Protector GUI only. + +- NSR enhancement for point-in-time recovery with EMC Networker client: A + new variable NSR_CLIENT_REQUESTRESTORE for the BACKUP=NSR workflow has + been introduced to deal with situations in which the NSR client is + allowed/granted to perform its recovery action "on its own" but does not + have full control of the EMC networker service/environment, for details + see its default.conf section. + +- BACKUP=NBKDC: Enhanced ReaR backup and restore to also work with NovaStor + DataCenter 8.0 and higher plus fixed wording "NovaBACKUP DC" -> "NovaStor + DC" + +- Support for systemd and parallel restore with Data Protector BACKUP=DP: + Support starting Data Protector daemon 'omni.socket' via systemd in the + recovery system. Restore performance has been increased by doing parallel + restores now. + +- Now "rear format" has in addition to the '-efi' switch a '-bios' switch. + If none is given (i.e. by default) it will now do hybrid formatting with a + BIOS boot partition (on GPT) and an EFI system partition. This is a + starting point for implementing OUTPUT=USB support for UEFI and BIOS dual + boot from the same medium. See https://github.com/rear/rear/issues/2698 + +- Overhauled serial console support code. A serial console of the ReaR + recovery system can now be specified separately for the kernel and the + recovery system bootloader via the generic config variables + USE_SERIAL_CONSOLE and SERIAL_CONSOLE_DEVICES and specific config + variables SERIAL_CONSOLE_DEVICES_KERNEL SERIAL_CONSOLE_DEVICE_SYSLINUX + SERIAL_CONSOLE_DEVICE_GRUB (see /usr/share/rear/conf/default.conf for + details) + +- Support for HTTP sources when using PXE: There is a new config variable + PXE_HTTP_URL to specify a HTTP download source for PXE. See the + PXE_HTTP_URL description in /usr/share/rear/conf/default.conf + +- In /usr/share/rear/conf/default.conf changed ISO_VOLID from "RELAXRECOVER" + to "REAR-ISO" so the first ISO has the label "REAR-ISO" (8 characters) and + subsequent ISOs get the labels "REAR-ISO_01" "REAR-ISO_02" ... respectively + that have 11 characters (the maximum length for FAT volume names) so + things work now by default when the ISO image is used to (manually) create + a FAT bootable USB stick Accordingly 'RELAXRECOVER' in variable names was + replaced. In particular the user config variable for automated input + USER_INPUT_RELAXRECOVER_SYMLINK_TARGET was renamed as + USER_INPUT_ISO_SYMLINK_TARGET so only that new name will work. + +Details (mostly in chronological order - newest topmost): + +- In finalize/Linux-i386/660_install_grub2.sh explain that it is also used + as fallback to install the nowadays most often used bootloader GRUB2 + unless the BOOTLOADER variable tells to install another bootloader (other + bootloader install scripts check the BOOTLOADER variable), + cf. https://github.com/rear/rear/issues/2817#issuecomment-1148488339 + +- Replace 'RELAXRECOVER' in variable names: In lib/global-functions.sh in + the function mount_url() renamed the user config variable for automated + input USER_INPUT_RELAXRECOVER_SYMLINK_TARGET as + USER_INPUT_ISO_SYMLINK_TARGET because the old name contained a user config + value: "RELAXRECOVER" was an old default value of ISO_VOLID, see + https://github.com/rear/rear/pull/2457 and + https://github.com/rear/rear/pull/2813#discussion_r885576867 + +- In default.conf properly describe PRE_BACKUP_SCRIPT and POST_BACKUP_SCRIPT + to match what the implementation does, see + https://github.com/rear/rear/pull/2735#issuecomment-1145006984 and for + special cases see + https://github.com/rear/rear/pull/2735#issuecomment-1148620157 + +- Add PRE_RECOVERY_COMMANDS and POST_RECOVERY_COMMANDS as alternative to + PRE_RECOVERY_SCRIPT and POST_RECOVERY_SCRIPT see the description in + default.conf how to use them and how they work. See + https://github.com/rear/rear/pull/2811 and see also + https://github.com/rear/rear/pull/2735 therein in particular + https://github.com/rear/rear/pull/2735#issuecomment-1134686196 + Additionally use LogPrint to show the user the executed commands, see + https://github.com/rear/rear/pull/2789 + +- In default.conf describe the new user config variables USB_BOOT_PART_SIZE + and USB_DEVICE_BOOT_LABEL see https://github.com/rear/rear/pull/2660 + +- Exclude dev/watchdog* from the ReaR recovery system: In default.conf add + dev/watchdog* to COPY_AS_IS_EXCLUDE because watchdog functionality is not + wanted in the recovery system because we do not want any automated reboot + functionality while disaster recovery happens via "rear recover", see + https://github.com/rear/rear/pull/2808 Furthermore having a copy of + dev/watchdog* during "rear mkrescue" in ReaR's build area may even trigger + a system crash that is caused by a buggy TrendMicro ds_am module touching + dev/watchdog in ReaR's build area (/var/tmp/rear.XXX/rootfs), see + https://github.com/rear/rear/issues/2798 + +- In output/default/940_grub2_rescue.sh for GRUB_RESCUE set 'root=/dev/ram0 + vga=normal rw' (the same is already done for other boot media) to avoid a + "Kernel panic ... Unable to mount root fs on unknown-block(0,0)" that could + otherwise happen in certain cases, see + https://github.com/rear/rear/pull/2791 + +- In build/GNU/Linux/400_copy_modules.sh always include loaded kernel + modules i.e. also for MODULES=() or MODULES=( 'moduleX' 'moduleY' ) + include the currently loaded kernel modules, + cf. https://github.com/rear/rear/issues/2727#issuecomment-994731345 + +- In layout/save/GNU/Linux/200_partition_layout.sh ensure $disk_label is one + of the supported partition tables (i.e. one of 'msdos' 'gpt' + 'gpt_sync_mbr' 'dasd') and ensure syntactically correct 'disk' and 'part' + entries in disklayout.conf (each value must exist and each value must be a + single non-blank word), see the last part about "error out directly ... when + things failed" in + https://github.com/rear/rear/issues/2801#issuecomment-1122015129 But do + not error out when there is no partition label type value for a 'disk' + entry in disklayout.conf because "rear recover" works in a special case + without partition label type value when there is only a 'disk' entry but + nothing else for this disk exists in disklayout.conf which can happen when + /dev/sdX is an empty SD card slot without medium, see + https://github.com/rear/rear/issues/2810 + +- In default.conf explain how to use LIBS properly, see + https://github.com/rear/rear/issues/2743 + +- In the function find_syslinux_modules_dir in lib/bootloader-functions.sh + the fallback 'find /usr' to find the SYSLINUX modules directory may take a + very long time on some systems (up to several hours) so tell the user in + debug mode what is going on, see https://github.com/rear/rear/issues/2792 + and https://github.com/rear/rear/issues/624 + +- In default.conf tell that MODULES=( 'loaded_modules' 'additional_module' ) + is not supported (the generic COPY_AS_IS method can be used to include + additional modules) cf. https://github.com/rear/rear/issues/2727 + +- Use disklayout.conf keyword 'raidarray' instead of 'raid' because we have + a new 'raiddisk' keyword and a keyword must not be a leading substring of + another keyword so we have now 'raidarray' and 'raiddisk', see + https://github.com/rear/rear/issues/2759 and + https://github.com/rear/rear/commit/53757eab1447c712fb7c8e44be9c8b3b3ffd9faa + +- In layout/save/default/450_check_bootloader_files.sh use /[e]tc/grub.cfg + and /[b]oot/.../grub.cfg with '*' globbing patterns (as in the EFI|GRUB2-EFI + case) to find any of grub.cgf or grub2.cfg in /etc/ or in /boot/ (e.g. in + openSUSE Leap 15.3 there is /boot/grub2/grub.cfg), + cf. https://github.com/rear/rear/pull/2796#issuecomment-1118387393 + +- In default.conf added '[e]tc/crypttab' to FILES_TO_PATCH_PATTERNS, + cf. https://github.com/rear/rear/pull/2795#discussion_r859670066 + +- New FILES_TO_PATCH_PATTERNS in default.conf to verify file hashes at the + end of recover after file restore from backup and show an error when + config files were restored from an outdated backup so the restored files + do not match the recreated system, see + https://github.com/rear/rear/pull/2795 and + https://github.com/rear/rear/issues/2785 and + https://github.com/rear/rear/issues/2787 + +- In backup/RSYNC/default/450_calculate_req_space.sh comment out remote + space check with rsync see https://github.com/rear/rear/issues/2760 + +- In layout/save/default/450_check_bootloader_files.sh and + layout/save/default/450_check_network_files.sh added bash globbing + characters [] around the first letter to ensure that with 'shopt -s + nullglob' files that do not exist will not appear so nonexistent files are + not appended to CHECK_CONFIG_FILES + cf. https://github.com/rear/rear/pull/2796#issuecomment-1117171070 + +- In backup/DUPLICITY/default/500_make_duplicity_backup.sh disable SC2068 + cf. https://github.com/rear/rear/issues/1040 + +- In backup/YUM/default/500_make_backup.sh and + backup/NETFS/default/500_make_backup.sh and + prep/NETFS/default/070_set_backup_archive.sh fixed SC2068 + cf. https://github.com/rear/rear/issues/1040 + +- In rescue/default/850_save_sysfs_uefi_vars.sh disable SC2045 + cf. https://github.com/rear/rear/issues/1040 + +- In layout/save/GNU/Linux/240_swaps_layout.sh and lib/output-functions.sh + fixed SC2045 cf. https://github.com/rear/rear/issues/1040 + +- In lib/_input-output-functions.sh disable SC2218 + cf. https://github.com/rear/rear/issues/1040 + +- In layout/save/GNU/Linux/230_filesystem_layout.sh aviod SC1087 + cf. https://github.com/rear/rear/issues/1040 + +- In default.conf tell that USB_RETAIN_BACKUP_NR is only supported when + EXTLINUX is used as bootloader for USB and in + output/USB/Linux-i386/300_create_extlinux.sh add an explanatory comment + how that code works, + cf. https://github.com/rear/rear/pull/2794#issuecomment-1106286485 + +- In layout/save/GNU/Linux/210_raid_layout.sh fixed and overhauled the + RAID10 'layout' support code (i.e. what belongs to the mdadm -layout + option). This was triggered by ShellCheck SC2034 and SC2066 for the old + code, see https://github.com/rear/rear/pull/2768 + +- In finalize/Fedora/i386/550_rebuild_initramfs.sh fixed SC2068 and SC2145 + via some generic code cleanup + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 and + see https://github.com/rear/rear/pull/2771 + +- In rescue/GNU/Linux/310_network_devices.sh fix SC2091 see + https://github.com/rear/rear/pull/2776 + +- In default.conf increase USB_UEFI_PART_SIZE to 1024 MiB, + cf. https://github.com/rear/rear/pull/1205 in particular to also make + things work by default when additional third-party kernel modules and + firmware (e.g. from Nvidia) are used, + cf. https://github.com/rear/rear/issues/2770#issuecomment-1068935688 + +- In doc/rear.8 and doc/rear.8.adoc fixed typo '/dev/disk/by-path/REAR-000' + -> '/dev/disk/by-label/REAR-000' + +- In default.conf better describe USB_DEVICE + cf. https://github.com/rear/rear/issues/2770#issuecomment-1068831482 + +- In output/default/940_grub_rescue.sh removed a misleading comment that + shows a wrong get_version function call and in + output/USB/Linux-i386/850_make_USB_bootable.sh fixed a wrong get_version + function call that is needed since the ShellCheck SC2068 fixed get_version + function in lib/layout-functions.sh + +- In layout/recreate/default/120_confirm_wipedisk_disks.sh and + layout/prepare/Linux-s390/205_s390_enable_disk.sh and + rescue/GNU/Linux/310_network_devices.sh and + rescue/GNU/Linux/230_storage_and_network_modules.sh and + restore/default/990_move_away_restored_files.sh and + restore/NBU/default/300_create_nbu_restore_fs_list.sh and + skel/default/etc/scripts/system-setup.d/00-functions.sh and + skel/default/etc/scripts/dhcp-setup-functions.sh and + wrapup/default/990_copy_logfile.sh and + usr/share/rear/lib/output-functions.sh and lib/layout-functions.sh and + lib/filesystems-functions.sh and lib/linux-functions.sh avoid ShellCheck + SC2068 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 + +- In output/ISO/Linux-i386/800_create_isofs.sh avoid ShellCheck SC2068 and + SC2145 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 + +- In output/TSM/default/960_dsmc_verify_isofile.sh avoid ShellCheck SC2068 + and SC2145 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 plus + better error checking and messaging and other code cleanup + +- In output/default/940_grub2_rescue.sh and + output/default/940_grub_rescue.sh avoid ShellCheck SC2068 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 + +- In output/default/950_email_result_files.sh avoid ShellCheck SC2068 and + SC2145 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 and + replaced ...IfError function calls by calling Error + +- In restore/FDRUPSTREAM/default/260_copy_log_and_report.sh fixed ShellCheck + reported error SC2199 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 and + fixed possible "bash: conditional binary operator expected" error + +- In output/USB/Linux-i386/300_create_extlinux.sh and + rescue/GNU/Linux/310_network_devices.sh avoid ShellCheck reported error + SC2199 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 + +- In prep/NBKDC/default/400_prep_nbkdc.s and lib/sesam-functions.sh avoid + ShellCheck false error indication SC1097 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 + +- In lib/_input-output-functions.sh fixed ShellCheck reported errors SC2145 + and SC2068 and avoided ShellCheck false error indication SC1087 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062945160 + +- In usr/sbin/rear avoid ShellCheck SC1075 and fixed ShellCheck SC2145 + cf. https://github.com/rear/rear/issues/1040 + +- In output/RAWDISK/Linux-i386/260_create_syslinux_efi_bootloader.sh fixed + ShellCheck SC2235 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In rear/lib/sesam-functions.sh fixed ShellCheck SC2221 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In lib/filesystems-functions.sh in function xfs_parse fixed ShellCheck + SC2179 and SC2128 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + furthermore 'xfs_opts' is local in both functions xfs_parse and create_fs + in layout/prepare/GNU/Linux/131_include_filesystem_code.sh + +- In lib/layout-functions.sh fixed ShellCheck SC2178 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In skel/default/etc/scripts/system-setup.d/00-functions.sh ignore + ShellCheck SC2119 and SC2120 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1062703092 + +- In restore/DUPLICITY/default/150_restore_duply.sh avoid ShellCheck SC2103 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In layout/prepare/GNU/Linux/100_include_partition_code.sh fixed ShellCheck + SC2199 and SC2076 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In output/ISO/Linux-ia64/400_create_local_efi_dir.sh fixed ShellCheck + SC2050 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In layout/save/GNU/Linux/230_filesystem_layout.sh fixed ShellCheck SC2030 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In verify/NBU/default/380_request_client_destination.sh fixed ShellCheck + SC2018 and SC2019 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- In backup/YUM/default/500_make_backup.sh fixed ShellCheck SC2000 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034870262 + +- Simpler code in prep/Linux-s390/305_include_s390_tools.sh to set a fixed + bootdir="/boot" that also fixes ShellCheck SC1066 + cf. https://github.com/rear/rear/issues/1040#issuecomment-1034890880 + +- New GRUB2_SEARCH_ROOT_COMMAND in default.conf to overrule the setting in + output/USB/Linux-i386/100_create_efiboot.sh see + https://github.com/rear/rear/pull/2763 and + https://github.com/rear/rear/issues/2500 + +- New EXCLUDE_IP_ADDRESSES and EXCLUDE_NETWORK_INTERFACES directives: These + new array variables enable to exclude specific IP addresses or network + interfaces when building the network configuration used in the rescue + environment. This is typically useful when floating IP addresses are used. + Not excluding these may lead to outage if the floating IP address is used + by another system at time the system is getting recovered, see + https://github.com/rear/rear/pull/2736 + +- multipath: fix exclusion of still wanted devices: The current code + excluding multipath devices is broken when a device being excluded matches + other devices. This leads to excluding wanted devices. This happens when + having custom alias for multipath devices or there are more than 26 + multipath devices and 'mpatha' is getting excluded, which leads to + excluding all 'mpathaX' devices are well, see + https://github.com/rear/rear/pull/2750 + +- Update 06-layout-configuration.adoc: Enhance the "disk layout file syntax" + description: Describe that one cannot rely on backward compatibility. + Describe positional parameters vs. option=value parameters. + +- Update default.conf: In default.conf describe that when OUTPUT_URL is set + OUTPUT_OPTIONS does not inherit the BACKUP_OPTIONS value + cf. https://github.com/rear/rear/issues/2753 + +- Include dmsetup and dmeventd as PROGS in conf/GNU/Linux.conf because older + releases of os-prober (1.74 and below) use dmsetup as a fallback solution + for mounting when grub-mount is missing but without dmsetup and dmeventd + also in the recovery system that would block indefinitely at "Installing + GRUB2 boot loader...", for details see + https://github.com/rear/rear/pull/2748 + +- In the function cmdline_add_console in lib/serial-functions.sh add + 'console=tty0' as fallback to cmdline only if no real serial device was + found otherwise on a machine with e.g. ttyS0 and ttyS1 where only ttyS0 is + real only kernel messages (but nothing else) would appear on ttyS0, see + https://github.com/rear/rear/pull/2749 + +- Skip unneeded /usr/lib/syslog-ng/loggen/ from recovery system: In + conf/GNU/Linux.conf do no longer copy all in /usr/lib/syslog-ng/ but only + copy .so files in /usr/lib/syslog-ng/ (same as for rsyslog). This skips in + particular /usr/lib*/syslog-ng/loggen/ because 'loggen' is not included in + the recovery system ('loggen' does not appear in any ReaR code). See + https://github.com/rear/rear/issues/2743 + +- For BACKUP=CDM removed need for the whole /usr/lib64 in the recovery + system. See https://github.com/rear/rear/pull/2747 and the related issues + https://github.com/rear/rear/issues/2266 + https://github.com/rear/rear/issues/2314 + https://github.com/rear/rear/issues/2685 + https://github.com/rear/rear/issues/2700 + +- Update 400_copy_modules.sh: Do no longer error out if 'cp -a -L' failed to + to copy all contents of /lib/modules/... but only tell the user about the + issue so he could inspect his system and decide, see + https://github.com/rear/rear/issues/2739#issuecomment-1014304127 + +- Support for HTTP sources when using PXE: There is a new config variable + PXE_HTTP_URL to specify the HTTP download source for PXE. If the variable + is set the PXE config file will generate an additional 3rd boot option + (besides the unchanged standard options "rear" via TFTP and "local") + namely "rear-http" which includes the HTTP URL information to download the + kernel and initrd data. See https://github.com/rear/rear/pull/2738 + +- Also add ssh-add prog when ssh-agent is added. Since ssh-agent gets added + it makes sense to add ssh-add as well. See + https://github.com/rear/rear/pull/2729 + +- In layout/prepare/default/420_autoresize_last_partitions.sh automatically + resize active last partitions on RAID0 disks, see + https://github.com/rear/rear/issues/2696 Therefore in + layout/save/GNU/Linux/210_raid_layout.sh a new 'raiddisk' entry that has + same syntax as a 'disk' entry is written to disklayout.conf to have size + (and partition label) of a RAID device like "raiddisk /dev/md127 + 23599054848 gpt" because knowing the RAID device size is a precondition to + be able to automatically resize a RAID0 array. See + https://github.com/rear/rear/pull/2730 + +- In layout/prepare/default/420_autoresize_last_partitions.sh automatically + resize active last partitions on RAID1 disks and fixed fallback setting so + that now the user can specify AUTORESIZE_EXCLUDE_PARTITIONS=() in his + local.conf. Additionally in layout/prepare/default/250_compare_disks.sh + show info about old disk and sizes and new disks and sizes in case of + MIGRATION_MODE to make it easier to answer the questions to which new + disks old disk should be mapped. See + https://github.com/rear/rear/pull/2726 + +- Fix modinfo_filename() in build/GNU/Linux/400_copy_modules.sh Output the + original $module_filename also when it is a symlink because in the code + below 'cp -L' copies the symlink taget content as a new regular file with + file name as the name of the symlink so the copied content can be still + found under its original name, cf. + https://github.com/rear/rear/issues/2677#issuecomment-997859219 + Additionally in case of MODULES=( 'all_modules' ) also use 'cp -L' to copy + the actual content to avoid dangling symlinks in the recovery system. See + https://github.com/rear/rear/pull/2731 + +- In usr/share/rear/output/PXE/default/800_copy_to_tftp.sh fixed pxe file cp + and permissions for sshfs target to prevent cp error "failed to preserve + ownership" for sshfs and to prevent issue with write permissions trying to + override image on later runs. See https://github.com/rear/rear/pull/2723 + +- Improved wiping disks (see https://github.com/rear/rear/pull/2721): In + layout/recreate/default/120_confirm_wipedisk_disks.sh skip disks that do + not exist on the bare hardware in the recovery system + cf. https://github.com/rear/rear/issues/2715 and exclude disks that are + write-protected + cf. https://github.com/rear/rear/pull/2703#issuecomment-979928423 and show + in any case a user confirmation dialog for the disks that will be wiped. + In layout/recreate/default/150_wipe_disks.sh do no longer open (and close) + LUKS volumes because encrypted volumes contain meaningless data unless + opened and unencrypted so there is no need to wipe anything inside an + encrypted LUKS container, cf. "Regarding LUKS" in + https://github.com/rear/rear/pull/2514#issuecomment-743483945 + +- Overhauled RAID code (see https://github.com/rear/rear/pull/2714) based on + https://github.com/rear/rear/pull/2702 that is about initial software RAID + IMSM support i.e. with the changes in that pull request: Completely + overhauled layout/save/GNU/Linux/210_raid_layout.sh: No longer a subshell + that appends all stdout to disklayout.conf but explicit append to + disklayout.conf where needed to be safe against accidental things written + to disklayout.conf. Handle each mdadm option in one place i.e. parse and + prepare output. Handle options ordered by importance, mandatory first, + then optional ones. Basic tests that mandatory options are syntactically + valid plus Error if not. Overhauled + layout/prepare/GNU/Linux/120_include_raid_code.sh: The FEATURE_MDADM_UUID + code is meanwhile obsolete because all mdadm versions in supported ReaR + Linux distributions support '-uuid'. New + layout/recreate/default/220_verify_layout.sh to verify if RAID devices are + recreated with the UUIDs in disklayout.conf because mdadm silently ignores + this option when creating IMSM arrays (both containers and the volumes + inside them) and picks a random UUID + cf. https://github.com/rear/rear/pull/2702#issuecomment-970395567 Support + user specified DISKS_TO_BE_WIPED to mitigate + https://github.com/rear/rear/issues/2715 see the DISKS_TO_BE_WIPED + description in default.conf + +- Update 100_create_layout_file.sh: Additionally show the filesystem LABEL + in the 'lsblk' output of the original system as comment in disklayout.conf + to make it easier to understand subsequent data in particular for RAID + where the array name is shown as LABEL in 'lsblk' for example like + "/dev/sda ... linux_raid_member any:raid1sdab" - see also "one cannot see ... + /dev/md/ symlinks as NAME ... /dev/md/raid1sdab -> ../md127" in + https://github.com/rear/rear/pull/2702#issuecomment-951769031 and see also + https://github.com/rear/rear/commit/1a8a88c20d01f01fac65e296e9481ab9172c3ac7 + +- Update 100_create_layout_file.sh: Additionally show UUID in 'lsblk' output + of the original system as comment in disklayout.conf to make it easier to + compare UUIDs of the original system with what was recreated, cf. "... + shouldn't it [ReaR] restore the [RAID] arrays with the same UUIDs ...?" in + https://github.com/rear/rear/pull/2702#issuecomment-968904230 and + https://github.com/rear/rear/pull/2714#issuecomment-970279152 + +- Update 050_start_required_nfs_daemons.sh: Start rpcbind with the -s option + to avoid that in some cases rpcbind is not started / not available in the + recovery system cf. https://github.com/rear/rear/issues/2672 + +- New function is_multipath_used in layout-functions.sh (see + https://github.com/rear/rear/pull/2708) to get 'multipath -l' out of the + function is_multipath_path by adding a separated function + is_multipath_used that runs 'multipath -l' (which is expensive because it + scans all devices) to check if multipath is used and remembers the result + in MULTIPATH_IS_USED that is used in subsequent calls of is_multipath_used + (without calling 'multipath -l') so we can call is_multipath_used many + times via is_multipath_path which is called in + layout/save/GNU/Linux/200_partition_layout.sh for each disk device. This + should avoid long computing time when there are very may disk devices, + cf. https://github.com/rear/rear/pull/2597#issuecomment-814950019 + +- Now "rear format" has in addition to the -efi switch a -bios switch. If + none is given (i.e. by default) it will now do hybrid formatting with a + BIOS boot partition (on GPT) and an EFI system partition. This is a + starting point for implementing OUTPUT=USB support for UEFI and BIOS dual + boot from the same medium see https://github.com/rear/rear/issues/2698 and + https://github.com/rear/rear/pull/2705 + +- Update write-protect-functions.sh: In the function + is_write_protected_by_id() assume a disk without any of UUID PTUUID + PARTUUID WWN is empty and meant to be used to recreate the system so it + should not be write-protected + cf. https://github.com/rear/rear/pull/2703#discussion_r757393547 + +- Enhanced disk write-protection, see https://github.com/rear/rear/pull/2703 + and https://github.com/rear/rear/pull/2626 by WRITE_PROTECTED_IDS with + generic functionality, + cf. https://github.com/rear/rear/pull/2626#issuecomment-950953826 together + with the new WRITE_PROTECTED_ID_TYPES which defaults to UUID PTUUID + PARTUUID WWN so that the user can specify different lsblk columns as + needed in his particular environment + cf. https://github.com/rear/rear/pull/2703#issuecomment-962418441 + +- Stop ReaR from overwriting its own disk and backup drives for OUTPUT=USB + and OUTPUT=RAWDISK via new WRITE_PROTECTED_... config variables (see + default.conf) where UUIDs or filesystem labels can be specified so that + disks that contain such UUIDs or filesystem labels will be 'write + protected' during "rear recover", see + https://github.com/rear/rear/issues/1271 and + https://github.com/rear/rear/pull/2626 + +- Overhauled serial console support code. See + https://github.com/rear/rear/pull/2699 A serial console of the ReaR + recovery system can now be specified separately for the kernel and the + recovery system bootloader via the generic config variables + USE_SERIAL_CONSOLE and SERIAL_CONSOLE_DEVICES and specific config + variables SERIAL_CONSOLE_DEVICES_KERNEL SERIAL_CONSOLE_DEVICE_SYSLINUX + SERIAL_CONSOLE_DEVICE_GRUB see usr/share/rear/conf/default.conf for + details cf. https://github.com/rear/rear/issues/2663 and + https://github.com/rear/rear/pull/2697 + +- More control over serial devices used for console. See + https://github.com/rear/rear/pull/2697 + +- In layout/prepare/default/300_map_disks.sh overhauled the automapping of + original 'disk' devices and 'multipath' devices to current block devices + in the currently running recovery system so that now it automatically + finds an existing unique disk size mapping also when there is a unique + mapping between more than two disks, see + https://github.com/rear/rear/issues/2690 + +- Fix multiple VG recreation, see https://github.com/rear/rear/pull/2691 Do + not use global variables in diskrestore.sh: Fix a problem introduced in + commits b184194f37dd22a7e55655ff388579297239e73c and + 311bfb3da1d5e47a2ff144123a2457e634f67893 (PR #1806) that shows up when + there are multiple VGs to restore. Using variables + create_thin_volumes_only and create_logical_volumes to propagate + information from VG creation to LV creation does not work well in the case + of multiple VGs, because the variables are global and if there are + multiple VGs, their values will leak from one VG to another. The generated + diskrestore.sh script does not guarantee that the LVs of a given VG are + created immediately after their VG and before creating another VG. + Currently, the script first creates all VGs and then all LVs, so all the + LVs in all VGs will see the value of create_logical_volumes and + create_thin_volumes_only from the last VG, not from their own. This + matters when different VGs behave differently (typically if one has a thin + pool and the other does not). Fix by replacing the scalar values by arrays + of VG names. If a given VG is in the array, it is the equivalent of the + former scalar value being 1 for the given VG, if it is not in the array, + it is an equivalent of a former value of 0. For the create_volume_group + variable the change is not needed, but do it nevertheless for symmetry + with other variables. + +- Error out in sbin/rear when it failed to source or Source() mandatory + files to make it obvious when there are syntax errors in user config files + and/or when there are syntax errors in our lib/*-functions.sh code, see + https://github.com/rear/rear/issues/2686 + +- Overhauled output/USB/Linux-i386/100_create_efiboot.sh In particular no + longer use "cp -p" because '-p' may fail because it copies to a VFAT + filesystem on the EFI partition see + https://github.com/rear/rear/issues/2683 and now always use "cp -L" to + ensure the real content gets copied. See + https://github.com/rear/rear/pull/2684 + +- Overhauled the description in user-guide/05-integration.adoc what "rear + savelayout" and "rear checklayout" are menat to do, cf. + https://github.com/rear/rear/commit/7a3884bed1fb6cb61654c4e266ba4c35094bd9f4#r56879062 + +- Fixed https://github.com/rear/rear/issues/2667 "rear does not entirely + remove build area any more" by using double quotes for + remove_temporary_mountpoint "$BUILD_DIR/..." and fixed + https://github.com/rear/rear/issues/2676 "False ERROR '.../outputfs not + empty, cannot remove' when OUTPUT_URL is unset" by making + scheme_supports_filesystem() fail-safe if $1 is empty or blank and also + making scheme_accepts_files() fail-safe in the same way, see + https://github.com/rear/rear/pull/2675 + +- On POWER require bootlist & ofpathname if needed: The ofpathname binary is + called by grub2-install. Therefore, it is required in the rescue system in + order to make the recovered system bootable, except for the PowerNV (Not + Virtualized - bare metal) case. Under PowerVM, we also need the bootlist + executable to make the system bootable. See + https://github.com/rear/rear/pull/2665 + +- ReaR's default TMPDIR is now /var/tmp (it was /tmp before) i.e. ReaR's + default working area is now /var/tmp/rear.XXXXX ReaR needs lots of space + (from 300MB up to more than 1GB) to build the ReaR rescue/recovery system + e.g. as ISO image and even much more space to also store the backup + archive e.g. when the backup should be included in the ISO image. The + system's default temporary directory /tmp is no longer suited as default + temporary directory for ReaR because nowadays /tmp is often a "tmpfs" that + is in RAM so /tmp has RAM/swap limitations. file-hierarchy(7) recommends + to use /var/tmp for such cases. If the user sets TMPDIR explicitly, it + still takes precedence. See https://github.com/rear/rear/pull/2664 + +- Cleanup of GRUB2 config code in particular for OUTPUT=USB: Cleanup of the + create_grub2_cfg function: Describe its usage in a comment. Have its + helper functions inside create_grub2_cfg to make them not globally + accessible. Initial steps to clean up how the GRUB root device is set via + one single method. DebugPrint output how GRUB2 is configured to help the + user to find errors or bugs. Better GRUB2 menue entry names. Consistent + calling of the create_grub2_cfg function always as create_grub2_cfg + "/path/to/kernel" "/path/to/initrd" >/path/to/grub.cfg also in + output/ISO/Linux-i386/250_populate_efibootimg.sh. Set GRUB2 timeout to new + config variable GRUB2_TIMEOUT that is by default 300 seconds. Set GRUB2 + default boot menu entry to GRUB2_DEFAULT_BOOT that is by default + 'chainloader' which should usually boot the original system and reference + GRUB2 boot menu entries by 'menuentry ... -id=...' to have meaningful names + instead of meaningless numbers. Sleep 3 seconds exactly before GRUB2 shows + its menu (it replaces what there is on the screen) so that the user could + see possible GRUB2 (error) messages on the screen. Additionally cleanup of + the get_root_disk_UUID function. See + https://github.com/rear/rear/pull/2662 + +- Fixed serial console for syslinux: It writes only one serial line matching + the configured device (when found) to the config. It also writes it for + the syslinux config in case it is used without extlinux. See + https://github.com/rear/rear/pull/2650 + +- Fix function create_grub2_cfg() for the none EFI cases and adds serial + console configuration to the GRUB2 config. See + https://github.com/rear/rear/pull/2661 + +- Simplifications for USB_BOOTLOADER="grub" and choose the right GRUB2 + config /boot/grub/grub.cfg or /boot/grub2/grub.cfg see + https://github.com/rear/rear/pull/2659 + +- Cleanup USB format scripts and related code: Now OUTPUT=USB should behave + sufficiently OK on BIOS systems with syslinux/extlinux as bootloader with + a separated boot partition and without it (still the default). To use + OUTPUT=USB on BIOS with syslinux/extlinux with a separated boot partition + size of e.g. 1024 MiB specify in etc/rear/local.conf something like + OUTPUT=USB USB_BOOT_PART_SIZE=1024 + OUTPUT_URL=usb:///dev/disk/by-label/REARBOOT BACKUP=NETFS + BACKUP_URL=usb:///dev/disk/by-label/REAR-000 See + https://github.com/rear/rear/pull/2660 + +- Fixed format/USB/default/300_format_usb_disk.sh for none EFI cases by + adding a BIOS boot partition and setting the 'legacy_boot' flag in case of + GPT partitioning. See https://github.com/rear/rear/pull/2656 + +- Added support for GRUB2 as bootloader for OUTPUT=USB via new config + variable USB_BOOTLOADER=grub see https://github.com/rear/rear/pull/2655 + +- With the new default USER_INPUT_MAX_CHARS=0 input is not truncated and it + also makes correcting the input possible (before [Enter] is pressed) see + https://github.com/rear/rear/issues/2622 + +- Update default.conf: Since https://github.com/rear/rear/pull/2633 export + TMPDIR="..." in ReaR config files like etc/rear/local.conf does no longer + work so now export TMPDIR="..." must be called before calling 'rear' + cf. https://github.com/rear/rear/issues/2654 + +- Treat RAID layout value '-unknown-' same as an empty value: In + layout/save/GNU/Linux/210_raid_layout.sh treat the RAID layout value + '-unknown-' same as an empty value to avoid that + layout/prepare/GNU/Linux/120_include_raid_code.sh will create a 'mdadm' + command in diskrestore.sh like "mdadm ... -layout=-unknown- ..." which would + fail during "rear recover" with something like "mdadm: layout -unknown- + not understood for raid0", see https://github.com/rear/rear/issues/2616 + +- Avoid false Error() exit in copy_binaries() in + build/GNU/Linux/390_copy_binaries_libraries.sh by skipping 'cp' errors in + copy_binaries() (regardless what the reason was why 'cp' failed) but then + add the affected binary to REQUIRED_PROGS to verify later that the binary + actually exists in the recovery system which should be still sufficient to + let ReaR error out if something actually went wrong with needed binaries. + See https://github.com/rear/rear/pull/2643 + +- NSR enhancement for point-in-time recovery with EMC Networker client: A + new variable NSR_CLIENT_REQUESTRESTORE for the BACKUP=NSR workflow has + been introduced to deal with situations in which the NSR client is + allowed/granted to perform its recovery action "on its own" but does not + have full control of the EMC networker service/environment, for details + see its default.conf section. See https://github.com/rear/rear/pull/2641 + +- Workaround against DRBD9 restore error "refusing to be primary while peer + is not outdated" see https://github.com/rear/rear/issues/2634 + +- Cleanup rsync and fix error reporting: See + https://github.com/rear/rear/pull/2632 Cleanups of rsync code to use + better variable names and local variables, stop using StopIfError, use + better redirections etc. Fixed a problem with rsync error detection that + caused rsync errors during backup restore to be ignored, see + https://github.com/rear/rear/issues/2612 Now a warning is displayed and if + BACKUP_INTEGRITY_CHECK is true, it is elevated to an error and ReaR + aborts. Furthermore check fake super support only when RSYNC_PROTO=ssh and + removed a broken test for -fake-super arg to rsync, cf. + https://github.com/rear/rear/pull/2577 + +- Show some stdout and stderr messages also in non-debug modes: In non-debug + modes (in particular also in verbose mode) stdout and stderr are + redirected to a temporary file + STDOUT_STDERR_FILE=TMP_DIR/rear.WORKFLOW.stdout_stderr so in non-debug + modes stdout and stderr of all programs is still available for the Error() + function to extract some latest messages that get shown on the usrer's + terminal and those extracted lines are also copied into the log + cf. https://github.com/rear/rear/issues/2623 and + https://github.com/rear/rear/issues/2416 and + https://github.com/rear/rear/pull/2498 Furthermore the log files and the + stdout/stderr file can now only be read by root. Additionally when there + is something still mounted within the build area when rear finishes, the + user is informed what is mounted and that he must manually umount it + before he can (also manually) remove the build area. Finally the Error() + and cleanup_build_area_and_end_program() functions are made fail-safe + against not yet existing log files and not yet sourced other functions in + case of early Error() exits in usr/sbin/rear see + https://github.com/rear/rear/pull/2633 + +- Fixed accidental destructive backup removal in exit task and cleanup + handling of URL mountpoints: See https://github.com/rear/rear/pull/2625 + Cleanup of temporary mount point handling, particularly for output. + Unification of mount point umount and cleanup move to the mount_url() and + umount_url() functions. Replaced the various "rm -rf" of the mountpoint by + "rmdir" which fixes https://github.com/rear/rear/issues/2611 Added lazy + umount in case normal umount does not succeed. If build dir is kept + (cf. KEEP_BUILD_DIR), propose a safe way to remove it to the user via "rm + -Rf -one-file-system" instead of just "rm -Rf" where the user risks to + remove everything below that mountpoint if still mounted. Fixes also some + other bugs noted in the process: Filesystem-specific umount command not + called + https://github.com/rear/rear/commit/20359a987662cc0c3fcfa52d62d1feac7cd55850#r51319634 + Unknown schemes considered invalid, see the discussion under + https://github.com/rear/rear/pull/932 Identical scripts under DUPLICITY + and YUM replaced by symlinks. Reverted + https://github.com/rear/rear/pull/782 that had reintroduced + https://github.com/rear/rear/issues/465 which got re-reported as + https://github.com/rear/rear/issues/2611 Reverted + https://github.com/rear/rear/pull/578 because it is not clear how + .lockfile can exist in the unmounted filesystem, and if it does, it is a + bug. Reverted + https://github.com/rear/rear/commit/d850c4094238a03c9b926b88d7e1582ecd28af52 + because it became meanwhile obsoleted by + https://github.com/rear/rear/commit/a8fdc445d0d6f7f9184dc6633817928f96aae9bc + +- Add BACKUP_DUPLICITY_OPTIONS to + restore/DUPLICITY/default/400_restore_duplicity.sh Additionnal + BACKUP_DUPLICITY_OPTIONS options were missing which can cause the recover + to fail (options may contain endpoint information for example), see + https://github.com/rear/rear/issues/2619 + +- In prep/default/400_save_directories.sh also exclude mountpoints that are + below mountpoints of "type autofs": Those are below an ancestor mountpoint + that is owned/created by the automounter. It is possible to create a + sub-mountpoint below an automounted mountpoint but the fact that the + sub-mountpoint is not local means it should be excluded (i.e. there is no + need to recreate the non-local sub-mountpoint directory), see + https://github.com/rear/rear/issues/2610 + +- Better description and error checking for GRUB_RESCUE with UEFI plus some + alignment with the create_grub2_cfg function, + cf. https://github.com/rear/rear/issues/2545 Better describe GRUB_RESCUE + with UEFI in default.conf and in output/default/940_grub2_rescue.sh better + error checking plus some alignment with how create_grub2_cfg() creates a + GRUB2 config file i.e. no longer "insmod" the GRUB2 modules efi_gop + efi_uga video_bochs video_cirrus + cf. https://github.com/rear/rear/pull/2609#issuecomment-831883795 and the + discussion in https://github.com/rear/rear/issues/2388 see + https://github.com/rear/rear/pull/2609 + +- Fix setting boot path in case of UEFI partition (ESP) on MD RAID: The code + finalize/Linux-i386/670_run_efibootmgr.sh did not support Software RAID. + It has to execute the efibootmgr command on the members of the Software + RAID, see https://github.com/rear/rear/issues/2595 The ESP may be located + on a RAID device. In this case, we need to determine the physical RAID + components and call efibootmgr on them. In addition, clean up + finalize/Linux-i386/670_run_efibootmgr.sh and add more logging to ease + analysis when something goes wrong, see + https://github.com/rear/rear/pull/2608 + +- In prep/RSYNC/default/050_prep_rsync.sh adding sleep period to give rsync + daemon time to wake up, see https://github.com/rear/rear/pull/2599 + +- Have unused LVM PV devices only as comment in disklayout.conf: PVs that + are not part of a VG are documented as comment in disklayout.conf but they + are not recreated because they were not used on the original system so + there is no need to recreate them by "rear recover" see + https://github.com/rear/rear/issues/2596 and + https://github.com/rear/rear/pull/2603 + +- Automatically shrink LVs if needed during "rear recover". This enables to + run "rear recover" with automated LVM LVs shrinking as needed on a bit + smaller replacement disk (e.g. when a nominally same sized replacement + disk is actually a bit smaller than the original disk). This automated LVs + shrinking is not intended when disk(s) are substantially smaller. To + migrate onto a substantially smaller replacement disk the user must in + advance manually adapt his disklayout.conf file before he runs "rear + recover". In layout/prepare/GNU/Linux/110_include_lvm_code.sh assume the + command "lvcreate -L 123456b -n LV VG" had failed because of "Volume group + ... has insufficient free space" and then try as fallback attempt using all + remaining free space in the VG via "lvcreate -l 100%FREE -n LV VG". In + layout/save/GNU/Linux/220_lvm_layout.sh sort the 'lvs' output lines by + size of the LVs so only some biggest LVs may get automatically shrinked + (if needed) because we assume that the data of the backup can still be + restored into a big LV after it was shrinked a bit. See + https://github.com/rear/rear/pull/2591 + +- Replaced possibly misleading "USB device" by generic "USB or disk device" + wording in user messages (e.g. when a built-in disk /dev/sdc is used that + is no USB device). The word "USB" indicates that those messages belong to + the OUTPUT=USB method, + cf. https://github.com/rear/rear/pull/2589#issuecomment-805649510 and + https://github.com/rear/rear/issues/2588 and + https://github.com/rear/rear/pull/2589 + +- Wipe disks before recreating partitions/volumes/filesystems/... see + https://github.com/rear/rear/issues/799 See the new DISKS_TO_BE_WIPED in + default.conf and for details see + usr/share/rear/layout/recreate/default/README.wipe_disks This is currently + new and experimental functionality so that currently by default via + DISKS_TO_BE_WIPED='false' no disk is wiped to avoid possible regressions + until this new feature was more tested by interested users via an explicit + DISKS_TO_BE_WIPED setting in local.conf see + https://github.com/rear/rear/pull/2514 + +- In etc/scripts/system-setup.d/41-load-special-modules.sh load the nvram + kernel module if possible to make /dev/nvram appear because /dev/nvram + should be there when installing GRUB, see + https://github.com/rear/rear/issues/2554 and include the nvram kernel + module in the recovery system because nvram could be a module in + particular on POWER architecture see + https://github.com/rear/rear/issues/2554#issuecomment-764720180 and + https://github.com/rear/rear/pull/2580 + +- Make 400_copy_modules.sh fail-safe for newer modinfo output (in particular + modinfo in kmod-27 since SLES15-SP3) because otherwise copying builtin + kernel "modules" would let "rear mkrescue" error out with e.g. "ERROR: + unix exists but no module file?". Furthermore ensure that kernel modules + that should be loaded during recovery system startup (i.e. those in + MODULES_LOAD) get always copied into the recovery system. See + https://github.com/rear/rear/pull/2579 + +- In 300_format_usb_disk.sh adapt USB_UEFI_PART_SIZE fallback value to 512 + to be in compliance with the new default.conf setting of 512 MiB since + https://github.com/rear/rear/commit/9a6b9a109aa77afc6c96cf05bbd7988cf0310d61 + that was done in relation to https://github.com/rear/rear/pull/2576 see + also https://github.com/rear/rear/issues/2602 + +- Do not specify '-F 16' for mkfs.vfat and also no '-o fat=16' when mounting + it but rely on the mkfs.vfat automatic FAT type selection (which is FAT16 + up to 511 MiB and FAT32 starting at 512 MiB at least on openSUSE Leap 15.2 + with mkfs.vfat from dosfstools-4.1) and rely on the automatic FAT type + detection when mounting, cf. https://github.com/rear/rear/issues/2575 and + increase the default ESP size to USB_UEFI_PART_SIZE="512" to get by + default (via mkfs.vfat) a FAT32 ESP to be in compliance that "the ESP + should officially use a FAT32 filesystem" to avoid possible FAT16 ESP + issues with certain UEFI firmware. See + https://github.com/rear/rear/pull/2576 + +- Update 230_filesystem_layout.sh: Increase the 'docker info' timeout value + from 5 seconds to 10 seconds because sometimes 'docker info' needs more + than 5 seconds to finish, + cf. https://github.com/rear/rear/pull/2572#issuecomment-784110872 + +- Support loop devices in format_usb_disk see + https://github.com/rear/rear/pull/2555 + +- Update 110_include_lvm_code.sh: Make sure we delete the volume group + before re-creating it. The issue happens in Migration mode when ReaR is + not trying to use vgcfgrestore. See https://github.com/rear/rear/pull/2564 + +- In usr/share/rear/output/ISO/Linux-i386/700_create_efibootimg.sh add 2 + more 32MiB blocks to be on the safe side against inexplicable failures + like "cp: error writing + '/tmp/rear.XXX/tmp/efi_virt/./EFI/BOOT/elilo.conf': No space left on + device" where the above calculated $efi_img_sz is a bit too small in + practice, see https://github.com/rear/rear/issues/2552 + +- Add terminal password check via 'TTY_ROOT_PASSWORD' and describe it in + default.conf, see https://github.com/rear/rear/pull/2539 + +- Changes for NetBackup (NBU) support: Copy NetBackup PBX related files to + the rescue system and start vxpbx_exchanged on boot. Add /usr/openv/tmp + directory to the NBU skeleton. See https://github.com/rear/rear/pull/2544 + +- RAWDISK: Add local rescue partition installation capability: Introduce a + configuration variable 'RAWDISK_INSTALL_GPT_PARTITION_NAME', which is + unset by default. When set, the rescue system will be installed to local + disk partitions having the corresponding name. RAWDISK: fix local rescue + partition installs (unmount before copying): While the recently created + EFI boot partition is still mounted, its file system might not be fully + synced. Unmounting ensures that the underlying loop block device is up to + date before possibly being copied to local disk partitions. See + https://github.com/rear/rear/pull/2538 + +- BACKUP=BACULA: Make bacula aware of different location of config files see + https://github.com/rear/rear/pull/2535 + +- In layout/prepare/GNU/Linux/131_include_filesystem_code.sh fixed vfat mkfs + using uuid from layout if possible see + https://github.com/rear/rear/pull/2546 + +- Completely overhauled USB filesystem kernel module handling: In + prep/USB/default/380_copy_usb_fs_module.sh do no longer unconditionally + add the USB filesystem to the MODULES and MODULES_LOAD arrays because + there are systems where the USB filesystem (e.g. 'ext3') is no kernel + module and then (via MODULES_LOAD and /etc/modules) "modprobe ext3" would + fail in 40-start-udev-or-load-modules.sh with a false alarm "modprobe: + FATAL: ... ext3 not found ..." message that would appear on the user's + terminal during recovery system startup. See + https://github.com/rear/rear/pull/2537 + +- Add DP_LD_LIBRARY_PATH to include shared libraries used by BACKUP=DP see + https://github.com/rear/rear/pull/2549 + +- POWER architecture (ppc64) support for BACKUP=DP: Other platforms + supporting the Data Protector Disk Agent, but not the Cell Console (GUI) + that is only available on i386, x86_64 and ia64 are now supported with + ReaR. This includes in particular ppc64. Checks unavailable on clients + without the Data Protector Cell Console (GUI) are skipped. In such cases + backup restore can be done using Data Protector GUI only. + +- BACKUP=NBKDC: Enhanced ReaR backup and restore to also work with NovaStor + DataCenter 8.0 and higher plus fixed wording "NovaBACKUP DC" -> "NovaStor + DC", see https://github.com/rear/rear/issues/2518 + +- Provide final power to the user to skip the ISO_FILE_SIZE_LIMIT test: Skip + the assert_ISO_FILE_SIZE_LIMIT function when there is no usable + ISO_FILE_SIZE_LIMIT set so the user could specify ISO_FILE_SIZE_LIMIT=0 in + his etc/rear/local.conf if he wants to skip the ISO_FILE_SIZE_LIMIT test. + Aditionally enforce maximum 2GiB ISO_FILE_SIZE_LIMIT when the MODULES + array contains 'loaded_modules' because MODULES+=( udf ) has no effect in + this case unless it is loaded (which normally isn't). + +- Error out when files greater or equal ISO_FILE_SIZE_LIMIT should be + included in the ISO: See the reasoning in default.conf why the default + ISO_FILE_SIZE_LIMIT is 2GiB and why we error out when files >= + ISO_FILE_SIZE_LIMIT should be included in the ISO. It is needed to aviod + possibly disastrous failures later during "rear recover" that users who + exceed that limit in exceptional cases get a hard (error) info to + thoroughly test and verify whether or not things actually work for them. + See https://github.com/rear/rear/pull/2525 + +- Completely overhauled ldd test in 990_verify_rootfs.sh: Now if needed a + special LD_LIBRARY_PATH is set only in the bash that runs ldd and that + bash exits after ldd finished so the special LD_LIBRARY_PATH setting is + gone with it. So there is no need to remember and restore some previously + set LD_LIBRARY_PATH because nothing was changed in the bash that runs + rear. Furthermore reporting when a binary requires additional libraries + happens now in the same 'for' loop after the ldd test was run which + further simplifies the code. See https://github.com/rear/rear/pull/2523 + +- Skip the ldd test for ReaR files: In build/default/990_verify_rootfs.sh + skip the ldd test for ReaR files (mainly bash scripts) where it does not + make sense, cf. + https://github.com/rear/rear/issues/2519#issuecomment-731196820 + +- In 500_clone_keyboard_mappings.sh have more neutral wording in messages + about possible keyboard issues and don't be needlessly verbose: It seems + newer Debian-based systems (including Ubuntu) no longer contain any + keymaps directory as part of the base system by default so including + multi-keyboard support into the recovery system is impossible there. If + the distro provides console-multi-keyboard support, ReaR includes it + (without being verbose). If the distro has decided that this is not + necessary, ReaR aligns with it (without being verbose). If the user has + installed multi-keyboard support, ReaR aligns with it (without being + verbose). Only when including the current keyboard mapping failed + (i.e. when 'dumpkeys' failed) it shows subsequent messages on the user's + terminal in any case. See https://github.com/rear/rear/issues/2519 and + https://github.com/rear/rear/pull/2520 + +- Restore LD_LIBRARY_PATH after reporting binaries with 'not found' + libraries: In build/default/990_verify_rootfs.sh restore the + LD_LIBRARY_PATH after reporting binaries with 'not found' shared object + dependencies so that results are the same for the first 'ldd' run that + finds binaries with 'not found' shared objects dependencies and the second + 'ldd' run that reports those 'not found' shared objects dependencies, see + https://github.com/rear/rear/issues/2508#issuecomment-725914211 + +- Improved TCG Opal 2 documentation doc/user-guide/13-tcg-opal-support.adoc + cf. https://github.com/rear/rear/issues/2511 Better explained OS + installation according to the suggestion in + https://github.com/Drive-Trust-Alliance/sedutil/issues/301#issuecomment-723010843 + Additionally some information from an article on Ask Ubuntu are included + https://askubuntu.com/a/1271171/1120528 + +- Make recreating LUKS volumes work with optional cryptsetup options: The + "cryptseup luksFormat" command does not require any of the type, cipher, + key-size, hash, uuid option values because if omitted a cryptseup default + value is used, cf. + https://github.com/rear/rear/pull/2504#issuecomment-720341023 The right + UUID values are mandatory for LUKS volumes that will be mounted during + startup of the recreated system. But this does not mean ReaR should error + out when there is no cryptsetup uuid value because it is possible to run + "rear recover" with enforced MIGRATION_MODE and manually correct the + restored /mnt/local/etc/crypttab file to use the new UUIDs before the + initrd is recreated and the bootloader is (re)-installed + cf. https://github.com/rear/rear/issues/2509 + +- Allow setting a PBA-specific firmware configuration via the new + OPAL_PBA_FIRMWARE_FILES configuration variable. Add '.../amdgpu/*' firmware + files for AMD graphics hardware (if present) on TCG Opal pre-boot + authentication (PBA) images, cf. https://github.com/rear/rear/issues/2474 + +- Support OPAL 2 self-encrypting NVMe disk drives: Normally the OPAL device + itself is used directly but NVMe devices have one or more namespaces per + primary device and these namespaces act as disks, + cf. https://github.com/rear/rear/issues/2475 + +- Added initial LUKS2 support, see https://github.com/rear/rear/issues/2204 + Added new parameter 'type' to 'crypt' keyword used in disklayout.conf. + Using this parameter allows to recreate the same version of LUKS as on the + original system. Added LUKS version detection, parsing depending on + version and usage of 'type' parameter. + +- New 036_valid_output_methods.sh to error out for unsupported OUTPUT + methods: For "rear mkbackup/mkrescue/mkbackuponly/mkopalpba" (i.e. for all + workflows that run the 'prep' stage) check that the OUTPUT method is + actually implemented i.e. check that a usr/share/rear/output/$OUTPUT + directory exists and error out when an OUTPUT method seems to be not + supported to ensure that the user cannot specify a non-working OUTPUT in + etc/rear/local.conf + (cf. usr/share/rear/prep/default/035_valid_backup_methods.sh) see + https://github.com/rear/rear/issues/2501 + +- Redirect stdout and stderr to the log only in debug modes. In normal modes + stdout and stderr are discarded (go now to /dev/null). The reason is to + get rid of messages from command stdout and stderr that don't make sense + for the user on their own unless there is more information in the log that + provides context about the called command (e.g. what exact command was + called in debugscript mode) see https://github.com/rear/rear/issues/2416 + and https://github.com/rear/rear/pull/2498 + +- More verbose messages when components are excluded so that is is easier + for the user to see directly on his terminal what the actual results are + when he specified to exclude components in his etc/rear/local.conf and + what components are automatically excluded by ReaR. The verbosity depends + on if ReaR is run in verbose '-v' mode or even in debug '-d' or '-D' mode. + +- Inform the user about unsupported LUKS2 volumes but do not error out see + https://github.com/rear/rear/issues/2491 plus some general code cleanup + and improvements in layout/save/GNU/Linux/260_crypt_layout.sh + +- Update 05-integration.adoc: Explain that error messages in ReaR's log may + not come from ReaR itself but from programs that are called by ReaR + because stdout and stderr are redirected into ReaR's log file, + cf. https://github.com/rear/rear/issues/2479 + +- Have 'cd WORKING_DIR' in Source() function: Ensure that after each sourced + file we are back in ReaR's usual working directory which is the current + working directory when usr/sbin/rear is launched that is also the working + directory of all the other scripts and config files that get sourced via + the Source() function in lib/framework-functions.sh see + https://github.com/rear/rear/issues/2461 + +- In 400_copy_modules.sh skip copying kernel modules that are builtin + modules. The new behaviour is that when modules are listed in + modules.builtin and are also shown by modinfo then those modules are now + skipped. Before for such modules the modules file(s) would have been + included in the recovery system. See + https://github.com/rear/rear/issues/2414 + +- In backup/NETFS/default/500_make_backup.sh filter purely informational tar + messages from output: Suppress purely informational tar messages from + output like "tar: Removing leading / from member names" or "tar: Removing + leading / from hard link targets" or "tar: /path/to/socket: socket + ignored" but keep actual tar error or warning messages like "tar: + /path/to/file: file changed as we read it" and show only messages that are + prefixed with the name of the failed program (like 'tar:' or 'dd:') which + works when tar or dd fail but falsely suppresses messages from openssl + which needs to be fixed later, + cf. https://github.com/rear/rear/pull/2466#discussion_r466347471 + +- Fixed exit code logic in backup/NETFS/default/500_make_backup.sh: Commit + https://github.com/rear/rear/commit/2674807f6ad48da80b193ae977d345ad2ef5fdd1 + removed the BACKUP_PROG_CRYPT_OPTIONS="cat" dummy code which broke the + exit code logic because pipes_rc and backup_prog_shortnames did no longer + match. Additionally ensure that the number of elements in both arrays are + always the same. + +- Use ReaR specific TMP_DIR (not TMPDIR or hardcoded /tmp) plus some general + cleanup work as noticed "by the way" in scripts, see + https://github.com/rear/rear/issues/2460 and + https://github.com/rear/rear/pull/2462 + +- Adaptions and enhancements for BACKUP=CDM because the RBS agent file name + changed in Rubrik CDM v5.1 from rubrik-agent-sunos5.10.sparc.tar.gz to + rubrik-agent-solaris.sparc.tar.gz see + https://github.com/rear/rear/issues/2441 + +- Let 'dd' read and write up to 1M=1024*1024 bytes at a time to speed up + things cf. https://github.com/rear/rear/issues/2369 and + https://github.com/rear/rear/issues/2458 + +- In default.conf change ISO_VOLID from "RELAXRECOVER" to "REAR-ISO" so the + first ISO has the label "REAR-ISO" (8 characters) and subsequent ISOs get + the labels "REAR-ISO_01" "REAR-ISO_02" ... respectively that have 11 + characters (the maximum length for FAT volume names) so things work now by + default when the ISO image is used to (manually) create a FAT bootable USB + stick, cf. https://github.com/rear/rear/issues/1565 and + https://github.com/rear/rear/issues/2456 + +- OPALPBA: Provide a permanent unlocking mode as a workaround because + https://github.com/rear/rear/pull/2426 did not resolve the issue + https://github.com/rear/rear/issues/2425 where for an HPE ML10Gen9 server + it seems the firmware did not initialize properly during a 'simple' reboot + so the only reliable way to boot was a power cycle after Opal disks were + unlocked. To reboot with unlocked disks there is now a new unlocking mode + "permanent" via OPAL_PBA_UNLOCK_MODE (see default.conf). Reactivating + locking is then the responsibility of the user. Additionally enhanced + security by disabling shell access via keyboard interrupt and switching to + a password hash for OPAL_PBA_DEBUG_PASSWORD. + +- Use single quotes in the password settings examples because single quotes + avoid issues with the special bash characters like $ in the password, + cf. https://github.com/rear/rear/pull/2178 + +- Update format-workflow.sh: For "rear format" show "Use 'rear format - + -help' for more information" also when a "rear format" argument is not + accepted to help the user what the right syntax is (e.g. "rear format + help" and "rear format - help" do not work, only "rear format - -help" + works) and provide a more helpful example "rear -v format - -efi /dev/sdX" + that is actually neded for UEFI. + +- Support for systemd and parallel restore with Data Protector BACKUP=DP: + Support starting Data Protector daemon 'omni.socket' via systemd in the + recovery system. Restore performance has been increased by doing parallel + restores now. Additional new script + usr/share/rear/finalize/DP/default/500_restore_ssc.sh to restore the + client certificate when Data Protector Secure Communication is used. Plus + several general script cleanup things. See + https://github.com/rear/rear/pull/2443 + +- Supply default value for root in grub in UEFI ISO: Sometimes the search + command in GRUB2 used in UEFI ISO does not find the root device. This was + seen at least in Debian Buster running in Qemu (VirtualBox works fine, + RHEL/CentOS in Qemu works fine as well). To make ReaR work in this case, + set $root to a sensible value before trying search. The GRUB2 image + created by grub-mkstandalone has $root set to memdisk, which can't work. + This essentially matches how it used to work before + https://github.com/rear/rear/pull/2293 and fixes + https://github.com/rear/rear/issues/2434 + +- Update 300_format_usb_disk.sh: In + format/USB/default/300_format_usb_disk.sh when prompting user for size of + EFI system partition on USB disk if no valid value is specified also use + 400 MiB as default (same as the default value in default.conf) + cf. https://github.com/rear/rear/pull/1205#issuecomment-433336977 and + https://github.com/rear/rear/commit/89e6a47c82fbfe8d059a4647755f94750fe28acc + +- Replaced $DATE usage with consistent variables START_SECONDS + START_DATE_TIME_NUMBER and START_DATE_TIME_STRING see + https://github.com/rear/rear/pull/2452 + +- Various TCG Opal Improvements: OPALPBA, Ubuntu: Fix incomplete file + exclusions: Fixes omissions reported in + https://github.com/rear/rear/issues/2436 Uses COPY_AS_IS_EXCLUDE where + possible. Removes files from the PBA system's staging directory where not + copying could not be configured. OPALPBA: Improve boot splash on Ubuntu + 20.04: Takes into account a change from VT1 to VT7 to hide log messages + during the boot process. Improves Plymouth boot splash usage to hide log + messages during the reboot after unlocking disks. opaladmin: Add + sub-commands 'deactivate', 'reactivate': Makes turning on and off hardware + encryption easier on provisioned drives. TCG Opal-2: simplify quoting in + messages and replace 'StopIfError' use + cf. https://github.com/rear/rear/commit/daf35e235d0770c663ff8dba866dddec76586a27 + Explicitly issuing an unlock command before reactivating locking ensures + that the device remains in an unlocked state when locking is reactivated. + +- Add an explanatory comment in lib/_input-output-functions.sh that using + the ...IfError functions can result unexpected behaviour in certain cases + cf. https://github.com/rear/rear/pull/2443#pullrequestreview-440609407 and + https://github.com/rear/rear/issues/534 and + https://github.com/rear/rear/issues/1415#issuecomment-315692391 + +- Migrate XFS configuration files. When in MIGRATION_MODE, migrate/rename + XFS configuration files so they follow disk mapping set by user. See + https://github.com/rear/rear/issues/2333 Version 2.6 (June 2020) Abstract -New features, bigger enhancements, and possibly backward incompatible -changes: +New features, bigger enhancements, and possibly backward incompatible changes: - o Initial preliminary first basic support for IBM Z architecture "s390x" - (a.k.a. "z Systems", formerly "System/390", simply called "s390") : +- Initial preliminary first basic support for IBM Z architecture "s390x" + (a.k.a. "z Systems", formerly "System/390", simply called "s390"): Currently primarily for Red Hat and SUSE enterprise systems so that - interested users can try out early how far things work in their - particular IBM Z environments and provide feedback preferably plus needed - fixes, adaptions, and enhancements to ReaR upstream at https://github.com - /rear/rear (issues #2137 #2142). - - o New BACKUP=CDM method to support Rubrik Cloud Data Management (CDM): The - Rubrik CDM backup and restore method for ReaR allows Rubrik CDM to - perform bare metal recovery of Linux systems. It does this by including - the Rubrik CDM RBS agent files in the ReaR recovery system so that during + interested users can try out early how far things work in their particular + IBM Z environments and provide feedback preferably plus needed fixes, + adaptions, and enhancements to ReaR upstream at + https://github.com/rear/rear (issues #2137 #2142). + +- New BACKUP=CDM method to support Rubrik Cloud Data Management (CDM): The + Rubrik CDM backup and restore method for ReaR allows Rubrik CDM to perform + bare metal recovery of Linux systems. It does this by including the Rubrik + CDM RBS agent files in the ReaR recovery system so that during rear recover the Rubrik CDM RBS agent can be used to restore from Rubrik CDM (issue #2248). - o New use-case for BLOCKCLONE backup method with the "multiple backups" - approach for a more complex LUKS setup with different passphrases or - keys: The usual support for LUKS-encrypted filesystems means that during +- New use-case for BLOCKCLONE backup method with the "multiple backups" + approach for a more complex LUKS setup with different passphrases or keys: + The usual support for LUKS-encrypted filesystems means that during reare recover a new encrypted filesystem will be created with new encryption keys. To recover the exact LUKS setup including its keys from the original system it is now possible to use the "multiple backups" @@ -227,19 +1475,19 @@ changes: encrypted filesystem before creating its image and before restoring it (issue #2200). - o Initial draft implementation of the new 'mountonly' workflow to use ReaR +- Initial draft implementation of the new 'mountonly' workflow to use ReaR as rescue system, therein mount the filesystems of the target system so - that one can manually repair it. This is described in doc/user-guide/ - 04-scenarios.adoc (issue #2247). + that one can manually repair it. This is described in + doc/user-guide/04-scenarios.adoc (issue #2247). - o Enable creation of non consecutive partitions. parted is not capable of +- Enable creation of non consecutive partitions. parted is not capable of creating non-consecutive partitions. To still be able to do so, the trick consists in creating dummy partitions to fill the gaps between partition numbers. Allocation of these dummy partitions is done from the end of the target partition, because parted is not capable of resizing a partition from the beginning (issues #2081 #1793 #1771 #1681). - o Improved handling of partition label type for multipath disks plus +- Improved handling of partition label type for multipath disks plus including of multipath disks in backup (unless AUTOEXCLUDE_MULTIPATH is true). Now the partition label type for multipath devices (same as is already done for normal disks) is stored in 'multipath' entries in @@ -247,37 +1495,37 @@ changes: syntax is now documented in doc/user-guide/06-layout-configuration.adoc (issues #2234 #2236). - o The config variable WAIT_SECS default value was aligned with the +- The config variable WAIT_SECS default value was aligned with the USER_INPUT_TIMEOUT default value so that now the default timeout for user input is always 300 seconds. The former WAIT_SECS default value of 30 seconds was too short because the 'read -t' timeout interrupts ongoing - user input. The timeout must be sufficiently long for the user to read - and understand the possibly unexpected user input request message and - then some more time to make a decision what to enter and finally even - more time to enter the right value (issue #2257). + user input. The timeout must be sufficiently long for the user to read and + understand the possibly unexpected user input request message and then + some more time to make a decision what to enter and finally even more time + to enter the right value (issue #2257). - o Enhanced default AUTOEXCLUDE_PATH=( /media /run /mnt /tmp ) to +- Enhanced default AUTOEXCLUDE_PATH=( /media /run /mnt /tmp ) to automatically exclude temporarily mounted things (e.g. USB devices) because mountpoints for temporarily mounted things are usually sub-directories below /media /run (like /run/media) /mnt and /tmp (issue #2239). - o Now we have CLONE_ALL_USERS_GROUPS="true" by default. See the explanation +- Now we have CLONE_ALL_USERS_GROUPS="true" by default. See the explanation in default.conf what that means. (issue #2345) - o Replaced grub-mkimage with more sophisticated grub-mkstandalone when +- Replaced grub-mkimage with more sophisticated grub-mkstandalone when building GRUB2 image for using GRUB2 as recovery system UEFI bootloader (issue #2293). Details (mostly in chronological order - newest topmost): - o Added DHCP auto-configuration for Network Manager (Ubuntu 20.04 Desktop): +- Added DHCP auto-configuration for Network Manager (Ubuntu 20.04 Desktop): While ReaR supports DHCP auto-detection during 'rear mkrescue' of some DHCP setups, the Network Manager flavor on Ubuntu 20.04 Desktop was not supported so DHCP for the recovery system was not auto-enabled there (issue #2427). - o OPALPBA (OUTPUT=RAWDISK): Workaround for EFI firmware glitch which +- OPALPBA (OUTPUT=RAWDISK): Workaround for EFI firmware glitch which prevents booting the real OS after unlocking SEDs. Symptom: Boot hangs when Grub hands over control to the Linux kernel. Seen on an HPE ML10Gen9 server. The workaround is to add and remove a UEFI 'next boot' entry @@ -285,624 +1533,625 @@ Details (mostly in chronological order - newest topmost): boot order to determine the effective boot entry for the next boot (issue #2425). - o Update default.conf: Describe LUKS_CRYPTSETUP_OPTIONS usage with some +- Update default.conf: Describe LUKS_CRYPTSETUP_OPTIONS usage with some examples (issue #2389). - o Overhauled how SSH config files are parsed for 'IdentityFile' values to +- Overhauled how SSH config files are parsed for 'IdentityFile' values to find (and remove) unprotected SSH keys in the recovery system. Now "find - ./etc/ssh" ensures that SSH 'Include' config files e.g. in /etc/ssh/ - ssh_config.d/ are also parsed (issue #2421). + ./etc/ssh" ensures that SSH 'Include' config files e.g. in + /etc/ssh/ssh_config.d/ are also parsed (issue #2421). - o Update default.conf: Describe usage of non-quoted globbing patterns like - VAR+=( /directory/* ) versus quoted globbing patterns like VAR+=( '/ - directory/*' ) and also describe how to prepend to an array in contrast +- Update default.conf: Describe usage of non-quoted globbing patterns like + VAR+=( /directory/* ) versus quoted globbing patterns like VAR+=( + '/directory/*' ) and also describe how to prepend to an array in contrast to append to it (issue #2417). - o When creating md5sums of the files in the recovery system be safe against +- When creating md5sums of the files in the recovery system be safe against blanks or special characters in file names by using appropriate options for all commands in the pipe (issues #2407 #1372). - o Evaluate OUTPUT_LFTP_OPTIONS at the beginning of lftp operations. Before +- Evaluate OUTPUT_LFTP_OPTIONS at the beginning of lftp operations. Before OUTPUT_LFTP_OPTIONS had been evaluated after connection to destination host was established which prohibited using OUTPUT_LFTP_OPTIONS also for connection specific settings. Now OUTPUT_LFTP_OPTIONS is moved to the - beginning of lftp command before any other lftp command is executed - (issue #2410). + beginning of lftp command before any other lftp command is executed (issue + #2410). - o For BACKUP=BORG implemented pagination when selecting the Borg archive to +- For BACKUP=BORG implemented pagination when selecting the Borg archive to restore from: During "rear recover" all Borg archives are shown. In case of a long list only the last items had been visble to the user. Now pagination happens with a configurable number of Borg archives, see BORGBACKUP_RESTORE_ARCHIVES_SHOW_MAX in default.conf (issue #2408). - o For BACKUP=BORG updated generated locale filename from "rear.UTF-8" to +- For BACKUP=BORG updated generated locale filename from "rear.UTF-8" to "en_US.UTF-8" (issue #2402). - o Use plain ${COPY_AS_IS[*]} instead of quoted "${COPY_AS_IS[@]}" in the - tar command call that copies things into the recovery system to ensure - "things work as usually expected" for any methods that are used to add - elements to the COPY_AS_IS array and better explain in default.conf how - COPY_AS_IS works, in particular that symlinks cannot be followed and that - files or directories that contain blanks or other $IFS characters cannot - be specified (issue #2405). +- Use plain ${COPY_AS_IS[*]} instead of quoted "${COPY_AS_IS[@]}" in the tar + command call that copies things into the recovery system to ensure "things + work as usually expected" for any methods that are used to add elements to + the COPY_AS_IS array and better explain in default.conf how COPY_AS_IS + works, in particular that symlinks cannot be followed and that files or + directories that contain blanks or other $IFS characters cannot be + specified (issue #2405). - o For RAWDISK output add distribution-specific GRUB2 module 'linuxefi.mod' +- For RAWDISK output add distribution-specific GRUB2 module 'linuxefi.mod' otherwise GRUB2 could not boot a UEFI rescue medium on Ubuntu 18.04 or 20.04 (issue #2419). - o ReaR was using hard-coded set of Grub2 modules for UEFI boot-loader. New +- ReaR was using hard-coded set of Grub2 modules for UEFI boot-loader. New GRUB2_MODULES_UEFI and GRUB2_MODULES_UEFI_LOAD config variables for installing GRUB2 as recovery system UEFI bootloader so that user can add or remove GRUB2 modules as needed (issues #2283 #2293 #2392). - o No longer load GRUB2 modules efi_gop and efi_uga in function - create_grub2_cfg : In the create_grub2_cfg function in lib/ - bootloader-functions.sh do no longer enfore loading the GRUB2 modules +- No longer load GRUB2 modules efi_gop and efi_uga in function + create_grub2_cfg: In the create_grub2_cfg function in + lib/bootloader-functions.sh do no longer enfore loading the GRUB2 modules efi_gop and efi_uga because loading only the module all_video is sufficient and fail-safe because moddep.lst contains "all_video: efi_gop efi_uga" (issue #2388). - o No longer load video_bochs and video_cirrus by the create_grub2_cfg - function. In the create_grub2_cfg function in usr/share/rear/lib/ - bootloader-functions.sh that is used in case of UEFI to set up GRUB2 as - bootloader for the recovery system do no longer let GRUB2 load the - modules video_bochs and video_cirrus because those are not available as - GRUB2 modules in case of UEFI (x86_64-efi) and the generic "insmod - all_video" that is still there should be sufficient for GRUB2 (issue # - 2388). - - o Allow to boot original system from Grub menu (UEFI): This change adds - following: Possibility to boot original system for UEFI boot with OUTPUT= - NETFS and OUTPUT=USB (similarly to non UEFI ReaR rescue system). Replaces - grub-mkimage for building of Grub2 boot image for OUTPUT=USB with +- No longer load video_bochs and video_cirrus by the create_grub2_cfg + function. In the create_grub2_cfg function in + usr/share/rear/lib/bootloader-functions.sh that is used in case of UEFI to + set up GRUB2 as bootloader for the recovery system do no longer let GRUB2 + load the modules video_bochs and video_cirrus because those are not + available as GRUB2 modules in case of UEFI (x86_64-efi) and the generic + "insmod all_video" that is still there should be sufficient for GRUB2 + (issue #2388). + +- Allow to boot original system from Grub menu (UEFI): This change adds + following: Possibility to boot original system for UEFI boot with + OUTPUT=NETFS and OUTPUT=USB (similarly to non UEFI ReaR rescue system). + Replaces grub-mkimage for building of Grub2 boot image for OUTPUT=USB with build_bootx86_efi() (grub-mkstandalone). Replaces separate Grub configuration for OUTPUT=USB with create_grub2_cfg(). Using build_bootx86_efi() and create_grub2_cfg() in OUTPUT=USB will unify process of Grub boot image creation with OUTPUT=NETFS. (issue #2326) - o Replace grub-mkimage with more sophisticated grub-mkstandalone when +- Replace grub-mkimage with more sophisticated grub-mkstandalone when building GRUB2 image for using GRUB2 as recovery system UEFI bootloader (issue #2293). - o When transferring ReaR recovery system ISO, create also destination +- When transferring ReaR recovery system ISO, create also destination directory structure (including parents). This is useful because when destination directory structure does not exist (or is incomplete), Lftp - puts files into first available directory in OUTPUT_URL scheme (issue # - 2401). + puts files into first available directory in OUTPUT_URL scheme (issue + #2401). - o Have CLONE_ALL_USERS_GROUPS="true" by default and explain it in +- Have CLONE_ALL_USERS_GROUPS="true" by default and explain it in default.conf (issue #2345). - o New OUTPUT_LFTP_OPTIONS config variable for lftp custom parameters (issue +- New OUTPUT_LFTP_OPTIONS config variable for lftp custom parameters (issue #2384). - o Update 950_check_missing_programs.sh: When checking for required programs +- Update 950_check_missing_programs.sh: When checking for required programs also test for "basename program" because when required programs are specified with absolute path those programs appears in the ReaR recovery system in /bin/ so testing their original path would falsely fail during "rear recover" (issue #2206). - o New prep/USB//350_safeguard_error_out.sh safeguard scripts to let "rear +- New prep/USB//350_safeguard_error_out.sh safeguard scripts to let "rear mkrescue/mkbackup" error out in case of false usage of OUTPUT=USB on non PC-compatible (non-i386/x86/x86_64) architectures because with OUTPUT=USB on those architectures the USB medium cannot be booted (for those architectures there are no scripts that install a bootloader) and - documented that in the OUTPUT=USB section in default.conf (issues #2348 # - 2396). + documented that in the OUTPUT=USB section in default.conf (issues #2348 + #2396). - o Error out for unsupported workflows in any case: Before init/default/ - 050_check_rear_recover_mode.sh did only error out when in the recovery - system an unsupported workflow should be run. Now it also errors out when - on the normal/original system an unsupported workflow that is likely - destructive (in particular recover layoutonly restoreonly finalizeonly - and mountonly) should be run (issues #2387 #2395). +- Error out for unsupported workflows in any case: Before + init/default/050_check_rear_recover_mode.sh did only error out when in the + recovery system an unsupported workflow should be run. Now it also errors + out when on the normal/original system an unsupported workflow that is + likely destructive (in particular recover layoutonly restoreonly + finalizeonly and mountonly) should be run (issues #2387 #2395). - o Improve BorgBackup (short: Borg) integration into ReaR: Several +- Improve BorgBackup (short: Borg) integration into ReaR: Several improvements e.g. error handling, output to logfile etc. (issue #2382). - o In backup/NETFS/default/500_make_backup.sh add "bs=1M" to SPLIT_COMMAND= - "dd of=$backuparchive bs=1M" to let 'dd' read and write up to 1MiB at a - time to speed up things for example from only 500KiB/s (with the 'dd' - default of 512 bytes) via a 100MBit network connection to about its full - capacity (issue #2369). +- In backup/NETFS/default/500_make_backup.sh add "bs=1M" to + SPLIT_COMMAND="dd of=$backuparchive bs=1M" to let 'dd' read and write up + to 1MiB at a time to speed up things for example from only 500KiB/s (with + the 'dd' default of 512 bytes) via a 100MBit network connection to about + its full capacity (issue #2369). - o Error out during "rear mkrescue/mkbackup" when LUKS version 2 is used +- Error out during "rear mkrescue/mkbackup" when LUKS version 2 is used because LUKS version 2 is not suppported. When LUKS version 2 is used it fails at least to determine the hash value so we use an empty hash value as a simple test if gathering crypt information was successful and error out if not. (issues #2204 #2381) - o Auto-detect DHCP client with systemd-networkd plus completely overhauled +- Auto-detect DHCP client with systemd-networkd plus completely overhauled and simplified code in prep/GNU/Linux/210_include_dhclient.sh with - resulting simplification of skel/default/etc/scripts/system-setup.d/ - 58-start-dhclient.sh (issue #2375). - - o Filter out duplicate entries in COPY_AS_IS but keep the ordering of the - elements i.e. only the first occurrence of an element is kept. Also - remove duplicates in the copy_as_is_filelist_file with 'sort -u' because - here the ordering does not matter. (issue #2377) - - o Update 250_find_all_libs.sh: Removed unreliably working code that intends - to filter out duplicates in the LIBS and COPS_AS_IS arrays via echo "$ - {ARRAY[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ' that fails when array - elements contain spaces so better code should be used e.g. like printf - '%s\n' "${ARRAY[@]}" | awk '!seen[$0]++' but filtering out duplicates is - not needed and by the way cleaned up the whole script a bit. (issue # - 2377) - - o Skip Longhorn Engine replica devices (issue #2373). - - o Update 400_prep_yum.sh: For BACKUP=YUM error out if + resulting simplification of + skel/default/etc/scripts/system-setup.d/58-start-dhclient.sh (issue + #2375). + +- Filter out duplicate entries in COPY_AS_IS but keep the ordering of the + elements i.e. only the first occurrence of an element is kept. Also remove + duplicates in the copy_as_is_filelist_file with 'sort -u' because here the + ordering does not matter. (issue #2377) + +- Update 250_find_all_libs.sh: Removed unreliably working code that intends + to filter out duplicates in the LIBS and COPS_AS_IS arrays via + echo "${ARRAY[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ' + that fails when array elements contain spaces so better code + should be used e.g. like + printf '%s\n' "${ARRAY[@]}" | awk '!seen[$0]++' + but filtering out duplicates is not needed and by the way + cleaned up the whole script a bit. (issue #2377) + +- Skip Longhorn Engine replica devices (issue #2373). + +- Update 400_prep_yum.sh: For BACKUP=YUM error out if BACKUP_PROG_CRYPT_ENABLED is set because BACKUP=YUM does not support backup encryption (issue #2374). - o Fail-safe patching of etc/ssh/sshd_config in the recovery system to - ensure the needed ReaR settings/overrides actually apply. For example if - etc/ssh/sshd_config contains a Match section at the end of the file e.g. - appending "PermitRootLogin yes" will not be effective as a global setting - but only in the Match section. Therefore now the needed ReaR settings are - inserted at the beginning and all subsequent entries of that kind are - disabled (commented out) to ensure that the ReaR global setting is the - only one that is effective. (issue #2362) - - o Run what was build/default/985_fix_broken_links.sh now earlier as build/ - default/490_fix_broken_links.sh before files get modified in recovery - system because otherwise e.g. if etc/ssh/sshd_config is a symbolic link - build/default/500_ssh_setup.sh will fail to apply patches (issue #2360). - - o Use simpler and more fail-safe syntax ARRAY+=( additional elements ) +- Fail-safe patching of etc/ssh/sshd_config in the recovery system to ensure + the needed ReaR settings/overrides actually apply. For example if + etc/ssh/sshd_config contains a Match section at the end of the file + e.g. appending "PermitRootLogin yes" will not be effective as a global + setting but only in the Match section. Therefore now the needed ReaR + settings are inserted at the beginning and all subsequent entries of that + kind are disabled (commented out) to ensure that the ReaR global setting + is the only one that is effective. (issue #2362) + +- Run what was build/default/985_fix_broken_links.sh now earlier as + build/default/490_fix_broken_links.sh before files get modified in + recovery system because otherwise e.g. if etc/ssh/sshd_config is a + symbolic link build/default/500_ssh_setup.sh will fail to apply patches + (issue #2360). + +- Use simpler and more fail-safe syntax ARRAY+=( additional elements ) everywhere (issue #2364). - o Update 58-start-dhclient.sh: Fixed issue #2354. +- Update 58-start-dhclient.sh: Fixed issue #2354. - o Added usr/share/rear/restore/OPALPBA/ directory to pass the test for - $SHARE_DIR/restore/$BACKUP in prep/default/035_valid_backup_methods.sh +- Added usr/share/rear/restore/OPALPBA/ directory to pass the test for + SHARE_(D)IR/restore/BACKUP in prep/default/035_valid_backup_methods.sh because the mkopalpba workflow uses the nonexistent BACKUP method OPALPBA to create a PBA image (issue #2351). - o Fix OPALPBA: Could not change password on multiple SEDs at once. "rear - opaladmin changePW" (when used without any extra argument or with - multiple arguments) is meant to change the disk password on multiple +- Fix OPALPBA: Could not change password on multiple SEDs at once. "rear + opaladmin changePW" (when used without any extra argument or with multiple + arguments) is meant to change the disk password on multiple self-encrypting drives in a single invocation. However, it stopped after successfully changing the password on the first drive and the password of subsequent drives had not been changed. (issue #2349) - o Update 035_valid_backup_methods.sh: Simplified prep/default/ - 035_valid_backup_methods.sh to make it work more predictable and - fail-safe (issue #2353). +- Update 035_valid_backup_methods.sh: Simplified + prep/default/035_valid_backup_methods.sh to make it work more predictable + and fail-safe (issue #2353). - o Have a hardcoded '-iso-level 3' option in output/ISO/Linux-ppc64le/ - 820_create_iso_image.sh because it is also hardcoded in output/ISO/ - Linux-i386/820_create_iso_image.sh and it seems to also work in general - on PPC64LE architecture (issue #2344). +- Have a hardcoded '-iso-level 3' option in + output/ISO/Linux-ppc64le/820_create_iso_image.sh because it is also + hardcoded in output/ISO/Linux-i386/820_create_iso_image.sh and it seems to + also work in general on PPC64LE architecture (issue #2344). - o Update default.conf: Adapt the explanation in default.conf how +- Update default.conf: Adapt the explanation in default.conf how ISO_MAX_SIZE works (issue #2347). - o Cleaned up and enhanced the ISO_MAX_SIZE implementation in backup/NETFS/ - default/500_make_backup.sh to make it in particular more safe against - erroneous settings of the ISO_MAX_SIZE variable (issue #2344). +- Cleaned up and enhanced the ISO_MAX_SIZE implementation in + backup/NETFS/default/500_make_backup.sh to make it in particular more safe + against erroneous settings of the ISO_MAX_SIZE variable (issue #2344). - o Added required NFS users for proper rpcbind startup: Usually 'rpcuser' is +- Added required NFS users for proper rpcbind startup: Usually 'rpcuser' is used but 'rpc' is used in RHEL7.x and '_rpc' is used in Debian 10 (issues #2341 #2342 #2250) - o Cleaned up and enhanced 320_migrate_network_configuration_files.sh plus - some bugfixes where the old code presumably did not work. (issues #2310 # - 2313 #2312) +- Cleaned up and enhanced 320_migrate_network_configuration_files.sh plus + some bugfixes where the old code presumably did not work. (issues #2310 + #2313 #2312) - o Update 890_finish_checks.sh: In the WARNING message at the end of "rear - recover" when no bootloader could be installed also mention usr/share/ - rear/finalize/Linux-ppc64le/660_install_grub2.sh as an example script how - one could install a bootloader on POWER architecture. Furthermore since - finalize/default/110_bind_mount_proc_sys_dev_run.sh it is no longer - needed to manually mount /proc. (issues #2339 #2045) +- Update 890_finish_checks.sh: In the WARNING message at the end of "rear + recover" when no bootloader could be installed also mention + usr/share/rear/finalize/Linux-ppc64le/660_install_grub2.sh as an example + script how one could install a bootloader on POWER architecture. + Furthermore since finalize/default/110_bind_mount_proc_sys_dev_run.sh it + is no longer needed to manually mount /proc. (issues #2339 #2045) - o Update 095_exclude_non_essential_files.sh: Added also /usr/lib/grub2 and +- Update 095_exclude_non_essential_files.sh: Added also /usr/lib/grub2 and /usr/share/grub2 to COPY_AS_IS_EXCLUDE because since openSUSE Leap 15.1 - things were moved from /usr/lib/grub2/ to /usr/share/grub2/ (issue # - 2338). + things were moved from /usr/lib/grub2/ to /usr/share/grub2/ (issue #2338). - o Update 270_create_grub2_efi_bootloader.sh: Test for Grub 2 EFI components - directories /usr/lib/grub/x86_64-efi and now also for /usr/lib/grub2/ - x86_64-efi or /usr/share/grub2/x86_64-efi because since openSUSE Leap - 15.1 things were moved from /usr/lib/grub2/ to /usr/share/grub2/ (issue # - 2338). +- Update 270_create_grub2_efi_bootloader.sh: Test for Grub 2 EFI components + directories /usr/lib/grub/x86_64-efi and now also for + /usr/lib/grub2/x86_64-efi or /usr/share/grub2/x86_64-efi because since + openSUSE Leap 15.1 things were moved from /usr/lib/grub2/ to + /usr/share/grub2/ (issue #2338). - o Update uefi-functions.sh: Use /usr//grub/x86_64-efi/partmap.lst instead - of /usr/lib/grub*/x86_64-efi/partmap.lst because since openSUSE Leap 15.1 - things were moved from /usr/lib/grub2/ to /usr/share/grub2/ (issue # - 2338). +- Update uefi-functions.sh: Use /usr//grub/x86_64-efi/partmap.lst instead of + /usr/lib/grub*/x86_64-efi/partmap.lst because since openSUSE Leap 15.1 + things were moved from /usr/lib/grub2/ to /usr/share/grub2/ (issue #2338). - o Update 630_install_grub.sh and 650_install_elilo.sh: Show the actual +- Update 630_install_grub.sh and 650_install_elilo.sh: Show the actual missing directory in the Error message (issue #2337). - o Moved backup/default/005_valid_backup_methods.sh to prep/default/ - 035_valid_backup_methods.sh to also check for valid backup methods during - "rear mkrescue" because for most external backup methods only "rear - mkrescue" is used, cf. the section "BACKUP SOFTWARE INTEGRATION" in 'man - rear' (issue #2337). +- Moved backup/default/005_valid_backup_methods.sh to + prep/default/035_valid_backup_methods.sh to also check for valid backup + methods during "rear mkrescue" because for most external backup methods + only "rear mkrescue" is used, cf. the section "BACKUP SOFTWARE + INTEGRATION" in 'man rear' (issue #2337). - o Changes to Data Protector branding, adjustment required to support the - new Data Protector 10.x agent with Secure Socket Communication. Fixed - some problems related to SessionID format and problems with object names - like host type vs. file system backup (issue #2335). +- Changes to Data Protector branding, adjustment required to support the new + Data Protector 10.x agent with Secure Socket Communication. Fixed some + problems related to SessionID format and problems with object names like + host type vs. file system backup (issue #2335). - o Hard-coded root home directory string ("/root") was replaced with - $ROOT_HOME_DIR that is the eeal home directory of root user (issue # - 2334). +- Hard-coded root home directory string ("/root") was replaced with + $ROOT_HOME_DIR that is the eeal home directory of root user (issue #2334). - o Make USE_DHCLIENT and USE_STATIC_NETWORKING more fail-safe in ReaR +- Make USE_DHCLIENT and USE_STATIC_NETWORKING more fail-safe in ReaR recovery system (issue #2325). - o For FDR backups, if the path for $FDRUPSTREAM_DATA_PATH does not exist, - then create it. This change is FDR specific for s390 and is only in - effect if ZVM_NAMING is "Y" (issue #2320). +- For FDR backups, if the path for $FDRUPSTREAM_DATA_PATH does not exist, + then create it. This change is FDR specific for s390 and is only in effect + if ZVM_NAMING is "Y" (issue #2320). - o Overhauled rescue/default/010_merge_skeletons.sh: Made - 010_merge_skeletons.sh behave more reliably. Now it errors out when - things really went wrong. Now is is also supported that both a $skel_dir +- Overhauled rescue/default/010_merge_skeletons.sh: Made + 010_merge_skeletons.sh behave more reliably. Now it errors out when things + really went wrong. Now is is also supported that both a $skel_dir directory (e.g. usr/share/rear/skel/default/) plus a $skel_dir.tar.gz (e.g. usr/share/rear/skel/default.tar.gz) exist and then both get copied into the recovery system, first the directory and then the tar.gz so that via the tar.gz files from the $skel_dir directory could be overwritten if needed. (issues #2307 #2317) - o Added $FDRUPSTREAM_DATA_PATH/rear to $COPY_AS_IS_EXCLUDE_FDRUPSTREAM +- Added $FDRUPSTREAM_DATA_PATH/rear to $COPY_AS_IS_EXCLUDE_FDRUPSTREAM (issue #2318). - o Added documentation about possible (dirty) workarounds for needed - bind9-export libraries for CentOS 7.7 and 8.0 which is somewhat related - to Rubrik-CDM (issues #2266 #2284). +- Added documentation about possible (dirty) workarounds for needed + bind9-export libraries for CentOS 7.7 and 8.0 which is somewhat related to + Rubrik-CDM (issues #2266 #2284). - o Enhanced recovery system BIOS boot default settings for USB and ISO: For +- Enhanced recovery system BIOS boot default settings for USB and ISO: For OUTPUT=ISO the user can now explicitly specify what to boot by default when booting the ISO on BIOS systems via ISO_DEFAULT="boothd0" to boot from the first disk and ISO_DEFAULT="boothd1" to boot from the second disk. For OUTPUT=USB the user can now explicitly specify what to boot by - default when booting the disk on BIOS systems via USB_BIOS_BOOT_DEFAULT= - "boothd0" to boot from the first disk. The default USB_BIOS_BOOT_DEFAULT= - "" boots the second disk. (issues #2276 #2303) + default when booting the disk on BIOS systems via + USB_BIOS_BOOT_DEFAULT="boothd0" to boot from the first disk. The default + USB_BIOS_BOOT_DEFAULT="" boots the second disk. (issues #2276 #2303) - o Remove outdated /root/rear-DATE-TIMESTAMP.log symlinks. Before creating +- Remove outdated /root/rear-DATE-TIMESTAMP.log symlinks. Before creating the current symlink to the current log file test all files that match the - file name pattern of such symlinks and remove those already existing - (old) symlinks where its now outdated symlink target is the current log - file (issue #2301). + file name pattern of such symlinks and remove those already existing (old) + symlinks where its now outdated symlink target is the current log file + (issue #2301). - o Initial preliminary first basic support for IBM Z architecture "s390x" - (a.k.a. "z Systems", formerly "System/390", simply called "s390") : +- Initial preliminary first basic support for IBM Z architecture "s390x" + (a.k.a. "z Systems", formerly "System/390", simply called "s390"): Currently primarily for Red Hat and SUSE enterprise systems so that - interested users can try out early how far things work in their - particular IBM Z environments and provide feedback preferably plus needed - fixes, adaptions, and enhancements to ReaR upstream at https://github.com - /rear/rear (issues #2137 #2142). + interested users can try out early how far things work in their particular + IBM Z environments and provide feedback preferably plus needed fixes, + adaptions, and enhancements to ReaR upstream at + https://github.com/rear/rear (issues #2137 #2142). - o Improved 'is_multipath_path' function to be more fail safe that now tests +- Improved 'is_multipath_path' function to be more fail safe that now tests (via "multipath -l" output) if multipath is used. Additionally have 'lsblk' output as disklayout.conf header comments to make it easier to understand the values in the subsequent entries. Furthermore added 'xdd' to the PROGS array because sometimes a tool to display binary files is required in the recovery system. (issue #2298). - o Error out during "rear mkrescue" in case of insufficient LVM tools: - Overhauled layout/save/GNU/Linux/220_lvm_layout.sh where now the exit - code of the "lvm ..." calls are checked and it errors out if one fails in +- Error out during "rear mkrescue" in case of insufficient LVM tools: + Overhauled layout/save/GNU/Linux/220_lvm_layout.sh where now the exit code + of the "lvm ..." calls are checked and it errors out if one fails in particular to avoid that entries in disklayout.conf are missing or broken when too old LVM tools are used that do not support the needed options of - those "lvm ..." calls. Also in layout/save/default/ - 950_verify_disklayout_file.sh a simple test was added to verify that the - 'lvm...' entries in disklayout.conf look syntactically correct. Furthermore - in 06-layout-configuration.adoc in the "Disk layout file syntax" section - the exact same syntax for the "lvm..." entries is now listed as the headers - of the "lvm..." entries in disklayout.conf are. (issues #2259 #2291) - - o Fixed missing delete_dummy_partitions_and_resize_real_ones calls: Use - same code as for disks for multipath devices (these are just regular - disks), including MBR erasing and partition creation and cleanup : + those "lvm ..." calls. Also in + layout/save/default/950_verify_disklayout_file.sh a simple test was added + to verify that the 'lvm...' entries in disklayout.conf look syntactically + correct. Furthermore in 06-layout-configuration.adoc in the "Disk layout + file syntax" section the exact same syntax for the "lvm..." entries is now + listed as the headers of the "lvm..." entries in disklayout.conf are. + (issues #2259 #2291) + +- Fixed missing delete_dummy_partitions_and_resize_real_ones calls: Use same + code as for disks for multipath devices (these are just regular disks), + including MBR erasing and partition creation and cleanup: create_multipath() cannot call create_disk() because create_disk() verifies that the device is a block device, which it isn't with multipath because it's a symlink to a device mapper instead. Make sure new code handling re-creation of partitions and cleanup is used because Software Raid can have partitions since these are virtual disks. (issue #2281) - o Moved 'route' back from REQUIRED_PROGS to PROGS. ReaR uses 'ip route' +- Moved 'route' back from REQUIRED_PROGS to PROGS. ReaR uses 'ip route' everywhere so 'route' is not actually required. As 'route' is no longer always installed nowadays, having it in REQUIRED_PROGS lets ReaR falsely error out on such systems (issues #1961 #1652). - o Update 990_verify_rootfs.sh: Use a FDRUPSTREAM-specific LD_LIBRARY_PATH - to find FDR libraries (issue #2296). +- Update 990_verify_rootfs.sh: Use a FDRUPSTREAM-specific LD_LIBRARY_PATH to + find FDR libraries (issue #2296). - o Do not run 'ldd' on untrusted files to mitigate possible ldd security +- Do not run 'ldd' on untrusted files to mitigate possible ldd security issues because some versions of ldd may directly execute the file (see "man ldd") which happens as user 'root' during "rear mkrescue". The new - TRUSTED_FILE_OWNERS user config array contains user names that are - trusted owners of files where RequiredSharedObjects calls ldd (cf. - COPY_AS_IS) and where a ldd test is run inside the recovery system that - tests all binaries for 'not found' libraries. Furthermore use '2>>/dev/ - $DISPENSABLE_OUTPUT_DEV' at more places to avoid that the "rear -D - mkrescue" log file size would grow from about 5 MiB to about 17 MiB so + TRUSTED_FILE_OWNERS user config array contains user names that are trusted + owners of files where RequiredSharedObjects calls ldd (cf. COPY_AS_IS) and + where a ldd test is run inside the recovery system that tests all binaries + for 'not found' libraries. Furthermore use + '2>>/dev/$DISPENSABLE_OUTPUT_DEV' at more places to avoid that the "rear + -D mkrescue" log file size would grow from about 5 MiB to about 17 MiB so that now that log file size even shrinked to about 2 MiB. (issue #2279) - o Initial draft implementation of the new 'mountonly' workflow to use ReaR +- Initial draft implementation of the new 'mountonly' workflow to use ReaR as rescue system, therein mount the filesystems of the target system so - that one can manually repair it. This is described in doc/user-guide/ - 04-scenarios.adoc (issue #2247). + that one can manually repair it. This is described in + doc/user-guide/04-scenarios.adoc (issue #2247). - o Feature RAWDISK and OPALPBA improvements: RAWDISK: include additional - Grub modules from /boot/grub (and /boot/grub2) which had formerly been - missing. OPALPBA: improve Plymouth boot animation on Ubuntu, provide - integration capabilities for other distros. RAWDISK: add support for - distros which use 'grub2' naming. (issue #2275) +- Feature RAWDISK and OPALPBA improvements: RAWDISK: include additional Grub + modules from /boot/grub (and /boot/grub2) which had formerly been missing. + OPALPBA: improve Plymouth boot animation on Ubuntu, provide integration + capabilities for other distros. RAWDISK: add support for distros which use + 'grub2' naming. (issue #2275) - o Improved check for missing libraries in 990_verify_rootfs.sh so that now +- Improved check for missing libraries in 990_verify_rootfs.sh so that now also libraries are checked that are no executables plus skipped the ldd test for firmware files (issue #2279). - o Update default.conf: More explanatory comment in default.conf how +- Update default.conf: More explanatory comment in default.conf how COPY_AS_IS versus LIBS, PROGS, and REQUIRED_PROGS are meant to be used (issue #2278). - o Accommodate logs from multiple FDR/Upstream services. Users may run +- Accommodate logs from multiple FDR/Upstream services. Users may run multiple FDR/Upstream services on a single machine. This code change - allows ReaR to copy FDR/Upstream logs from the recovery environment to - the restored system for all running services. Previously the code was + allows ReaR to copy FDR/Upstream logs from the recovery environment to the + restored system for all running services. Previously the code was expecting to find only one set of logs and reports in the FDR/Upstream installation directory. For several years now, logs and reports have been stored in a new location ($FDRUPSTREAM_DATA_PATH), so we check here instead (issue #2251). - o Update format-workflow.sh: Set EXIT_FAIL_MESSAGE=0 before exiting in - "rear format - -help" to avoid the "rear format failed, check ...rear...log - for details" message that is pointless in this case. +- Update format-workflow.sh: Set EXIT_FAIL_MESSAGE=0 before exiting in "rear + format - -help" to avoid the "rear format failed, check ...rear...log for + details" message that is pointless in this case. - o Added alternative for 'poweroff.com' that is 'poweroff.c32' on RHEL8 +- Added alternative for 'poweroff.com' that is 'poweroff.c32' on RHEL8 (issue #2238). - o Removed unnecessary line that sets NSRSERVER in layout/save/NSR/default/ - 650_check_iso_recoverable.sh which prevented setting NSRSERVER in - local.conf (issue #2162). +- Removed unnecessary line that sets NSRSERVER in + layout/save/NSR/default/650_check_iso_recoverable.sh which prevented + setting NSRSERVER in local.conf (issue #2162). - o Enhanced default AUTOEXCLUDE_PATH=( /media /run /mnt /tmp ) plus - explanatory comment in default.conf how AUTOEXCLUDE_PATH works (issues # - 2239 #2261). +- Enhanced default AUTOEXCLUDE_PATH=( /media /run /mnt /tmp ) plus + explanatory comment in default.conf how AUTOEXCLUDE_PATH works (issues + #2239 #2261). - o Use is_true() for AUTOEXCLUDE_MULTIPATH and cleanup of the +- Use is_true() for AUTOEXCLUDE_MULTIPATH and cleanup of the get_partition_number function that now errors out with BugError in cases that are likely a bug in ReaR (issues #2245 #2260). - o Do not abort in case of invalid user input but retry if possible. When +- Do not abort in case of invalid user input but retry if possible. When input values can be (syntactically) validated there is now a retry loop - until the user provided valid input (or intentionally aborts) except - those cases where it was not possible to implement validation when it is - unknown what is valid for third-party tools (issue #2253). - - o Removed '2>/dev/null' redirection where possible: Some general cleanup - for ReaR 2.6. in all scripts by removing '2>/dev/null' where it makes - sense and replacing '&>/dev/null' by '1>/dev/null' where it makes sense - or also removing '&>/dev/null' where that seems to be better (issue # - 1395). By the way verify/TSM/default/ - 390_request_point_in_time_restore_parameters.sh and verify/NBU/default/ - 390_request_point_in_time_restore_parameters.sh have been somewhat - overhauled but both scripts still abort 'rear recover' when the user did - an invalid date or time input (issue #2253). - - o Fix including of multipath disks in backup: The AUTOEXCLUDE_DISKS logic + until the user provided valid input (or intentionally aborts) except those + cases where it was not possible to implement validation when it is unknown + what is valid for third-party tools (issue #2253). + +- Removed '2>/dev/null' redirection where possible: Some general cleanup for + ReaR 2.6 in all scripts by removing '2>/dev/null' where it makes sense and + replacing '&>/dev/null' by '1>/dev/null' where it makes sense or also + removing '&>/dev/null' where that seems to be better (issue #1395). By the + way verify/TSM/default/390_request_point_in_time_restore_parameters.sh and + verify/NBU/default/390_request_point_in_time_restore_parameters.sh have + been somewhat overhauled but both scripts still abort 'rear recover' when + the user did an invalid date or time input (issue #2253). + +- Fix including of multipath disks in backup: The AUTOEXCLUDE_DISKS logic traverses filesystems and tries to determine the associated underlying disks that should be kept. When having a filesystem fs:/A that is present - on a multipath device MP and the root filesystem fs:/ that is present on - a normal disk D, the code first established that fs:/A depends on fs:/ + on a multipath device MP and the root filesystem fs:/ that is present on a + normal disk D, the code first established that fs:/A depends on fs:/ resulting in marking D as used and then short-circuited processing of multipath devices. This resulted in MP not being considered as a used device and removed from the backup. The problem is fxed by removing the short-circuit logic in find_disk_and_multipath() so multipath devices get processed too unless AUTOEXCLUDE_MULTIPATH is true (issue #2236). - o Improve handling of partition label type for multipath disks: When +- Improve handling of partition label type for multipath disks: When recording information about a multipath disk, ReaR did not store information about its partition label type in the 'multipath' entry in - disklayout.conf. The recovery code tried to automatically detect the - label type using a heuristic that depends on GPT partition names. The - logic would incorrectly detect the device as having the MBR label type - instead of GPT if one of the partition names was exactly "primary", - "extended" or "logical". The problem is solved by explicitly storing the - partition label type for multipath devices (same as is already done for - normal disks) so that the syntax of the 'multipath' entries in - disklayout.conf is changed and that new syntax is now documented in doc/ - user-guide/06-layout-configuration.adoc The recovery code is accordingly - updated to utilize this information (issue #2234). - - o In doc/user-guide/16-Rubrik-CDM.adoc updated links to SUSE documentation: - Replaced the outdated link https://www.suse.com/documentation/sle-ha-12/ - book_sleha/data/cha_ha_rear.html that gets redirected to a not really - helpful standard fallback page by the currently valid URL for the most - recent SLE-HA-15-SP1 https://documentation.suse.com/sle-ha/15-SP1/html/ - SLE-HA-all/cha-ha-rear.html because meanwhile the SUSE documentation was - moved to its own server. The more general and more comprehensive - documentation URL https://en.opensuse.org/SDB:Disaster_Recovery is kept - and listed topmost (issue #2255). - - o Added explanatory comments to default.conf about backup restore that may - fail for things like BACKUP_PROG_COMPRESS_OPTIONS=("-zstd") due to - missing zstd binaries in the ReaR recovery system probably also for -lzip - -lzma -lzop (issue #2241). - - o New BACKUP=CDM method for Rubrik Cloud Data Management (CDM) see doc/ - user-guide/16-Rubrik-CDM.adoc (issues #2248 #2249). - - o In verify/NETFS/default/050_start_required_nfs_daemons.sh all kind of '2> - /dev/null' (i.e. also '&>/dev/null') are removed (so that '&>/dev/null' + disklayout.conf. The recovery code tried to automatically detect the label + type using a heuristic that depends on GPT partition names. The logic + would incorrectly detect the device as having the MBR label type instead + of GPT if one of the partition names was exactly "primary", "extended" or + "logical". The problem is solved by explicitly storing the partition label + type for multipath devices (same as is already done for normal disks) so + that the syntax of the 'multipath' entries in disklayout.conf is changed + and that new syntax is now documented in + doc/user-guide/06-layout-configuration.adoc The recovery code is + accordingly updated to utilize this information (issue #2234). + +- In doc/user-guide/16-Rubrik-CDM.adoc updated links to SUSE documentation: + Replaced the outdated link + https://www.suse.com/documentation/sle-ha-12/book_sleha/data/cha_ha_rear.html + that gets redirected to a not really helpful standard fallback page by the + currently valid URL for the most recent SLE-HA-15-SP1 + https://documentation.suse.com/sle-ha/15-SP1/html/SLE-HA-all/cha-ha-rear.html + because meanwhile the SUSE documentation was moved to its own server. The + more general and more comprehensive documentation URL + https://en.opensuse.org/SDB:Disaster_Recovery is kept and listed topmost + (issue #2255). + +- Added explanatory comments to default.conf about backup restore that may + fail for things like BACKUP_PROG_COMPRESS_OPTIONS=("-zstd") due to missing + zstd binaries in the ReaR recovery system probably also for -lzip -lzma + -lzop (issue #2241). + +- New BACKUP=CDM method for Rubrik Cloud Data Management (CDM) see + doc/user-guide/16-Rubrik-CDM.adoc (issues #2248 #2249). + +- In verify/NETFS/default/050_start_required_nfs_daemons.sh all kind of + '2>/dev/null' (i.e. also '&>/dev/null') are removed (so that '&>/dev/null' is replaced by '1>/dev/null') because in general '2>/dev/null' is unhelpful because it needlessly suppresses error messages in the log that would be helpful to see when something fails (issues #2250 #1395). - o Update 06-layout-configuration.adoc: Changed section title from +- Update 06-layout-configuration.adoc: Changed section title from "Including/Excluding components" to only "Excluding components". The latter avoids possible misunderstanding that there would be a config variable to explicitly include something but "including" here describes only to disable autoecludes (issue #2229). - o Update default.conf: At AUTOEXCLUDE_DISKS removed the comment Explicitly - excluding/including devices is generally a safer option because it is not - acually helpful and even misleading because there is currently no config - variable to explicitly include devices (issue #2229). +- Update default.conf: At AUTOEXCLUDE_DISKS removed the comment + Explicitly excluding/including devices is generally a safer option because + it is not acually helpful and even misleading because there is currently + no config variable to explicitly include devices (issue #2229). - o Do not keep the build dir when ReaR is run noninteractively by additional - special values for the KEEP_BUILD_DIR config variable, see its - desctiption in default.conf (issue #2218). +- Do not keep the build dir when ReaR is run noninteractively by additional + special values for the KEEP_BUILD_DIR config variable, see its desctiption + in default.conf (issue #2218). - o Fix LVM2 thin pool recreation logic / use of vgcfgrestore is broken: +- Fix LVM2 thin pool recreation logic / use of vgcfgrestore is broken: Removing forcibly (with '-force' passed twice) seems to work for now. But our use of vgcfgrestore is probably not appropriate at all. It works by chance. Typically, it works only for Linear volumes, and won't probably for Caches and Raid hierarchies or when there are existing Snapshots on the system. The only proper solution is perhaps stop relying on vgcfgrestore at all, but then we are not capable of restoring volume - groups and logical volumes with all properties from original system - (issue #2222). + groups and logical volumes with all properties from original system (issue + #2222). - o Use mountpoint instead of mount | grep (issue #2225). +- Use mountpoint instead of mount | grep (issue #2225). - o Updated default.conf: Replaced ARRAY=( "${ARRAY[@]}" additional elements - ) with simpler and more fail safe ARRAY+=( additional elements ) (related - to issues #2223 #2220). +- Updated default.conf: Replaced ARRAY=( "${ARRAY[@]}" additional elements ) + with simpler and more fail safe ARRAY+=( additional elements ) (related to + issues #2223 #2220). - o Append to the CLONE_USERS and CLONE_GROUPS arrays by using += instead of +- Append to the CLONE_USERS and CLONE_GROUPS arrays by using += instead of expanding the previous value to an empty element which then causes problems later (issues #2223 #699). - o Update 900_clone_users_and_groups.sh: Skip empty user and group values +- Update 900_clone_users_and_groups.sh: Skip empty user and group values (issue #2220). - o New use-case for BLOCKCLONE backup method for complex LUKS-encrypted +- New use-case for BLOCKCLONE backup method for complex LUKS-encrypted filesystems by usinng the "multiple backups" approach with a second BLOCKCLONE phase with a dd image that contains the encrypted filesystem where unmounting the encrypted filesystem before creating its image and before restoring it has to be done via the new config variable BLOCKCLONE_TRY_UNMOUNT (issue #2200). - o Fixed when ReaR does not detect an uncompressed kernel that is named +- Fixed when ReaR does not detect an uncompressed kernel that is named 'vmlinux' (instead of 'vmlinuz'). - o Added $BACKUP_OPTIONS to the line with mount_url in 800_copy_to_tftp.sh +- Added $BACKUP_OPTIONS to the line with mount_url in 800_copy_to_tftp.sh and 810_create_pxelinux_cfg.sh otherwise in case of OUTPUT=PXE mounting the NFS will fail. - o UEFI: Search for Grub2 modules in /usr/lib/grub/x86_64-efi and not in / - boot. On Fedora and RHEL systems, Grub2 UEFI modules live in /usr/lib/ - grub/x86_64-efi, not /boot, unless grub2-install is executed, but +- UEFI: Search for Grub2 modules in /usr/lib/grub/x86_64-efi and not in + /boot. On Fedora and RHEL systems, Grub2 UEFI modules live in + /usr/lib/grub/x86_64-efi, not /boot, unless grub2-install is executed, but executing this tool is not needed with UEFI. On SUSE systems, Grub2 UEFI modules also live in /usr/lib/grub/x86_64-efi but there is also a copy in /boot, so it's not needed searching in /boot at all. Additionally, only - UEFI modules should be looked for, so /boot cannot be searched but only / - boot/grub2/x86_64-efi (similarly /usr/lib/grub/x86_64-efi, not just /usr/ - lib/grub*), otherwise we could get some false positives on dual boot + UEFI modules should be looked for, so /boot cannot be searched but only + /boot/grub2/x86_64-efi (similarly /usr/lib/grub/x86_64-efi, not just + /usr/lib/grub*), otherwise we could get some false positives on dual boot systems (UEFI + Legacy), since modules for Legacy will also match, which is wrong (issues #2199 #2001). - o RAWDISK and TCG Opal 2 Self-Encrypting Disks: Add Secure Boot support +- RAWDISK and TCG Opal 2 Self-Encrypting Disks: Add Secure Boot support (issue #2166). - o Avoid that disklayout.conf contains duplicate 'lvmvol' lines: Create the +- Avoid that disklayout.conf contains duplicate 'lvmvol' lines: Create the 'lvmvol' lines commented out when multiple segments exist for a given LV. This is not an issue unless Migration Mode is used. In such case, using 'lvcreate' commands already does best effort and loses LV information - (issues #2194 #2187). Additionally do not overload the kval variable - (that is intended for passing options to lvcreate) by adding extra keys - to it, which are not supported by lvcreate. Introduce another variable - infokval for this purpose and print those unsupported and purely - informational keys only in comments (issue #2196). - - o Apply layout mappings also to other relevant files via layout/prepare/ - default/320_apply_mappings.sh In migration mode apply the disk layout - mappings not only to disklayout.conf but also to other files that are - also used to migrate the disk layout, namely VAR_DIR/layout/config/df.txt - and /etc/rear/rescue.conf Additionally re-read the - BTRFS_SUBVOLUME_SLES_SETUP variable from /etc/rear/rescue.conf because - its value could have been migrated to a new value (issue #2181). - - o In the function apply_layout_mappings (therein in its "step 3") treat + (issues #2194 #2187). Additionally do not overload the kval variable (that + is intended for passing options to lvcreate) by adding extra keys to it, + which are not supported by lvcreate. Introduce another variable infokval + for this purpose and print those unsupported and purely informational keys + only in comments (issue #2196). + +- Apply layout mappings also to other relevant files via + layout/prepare/default/320_apply_mappings.sh In migration mode apply the + disk layout mappings not only to disklayout.conf but also to other files + that are also used to migrate the disk layout, namely + VAR_DIR/layout/config/df.txt and /etc/rear/rescue.conf Additionally + re-read the BTRFS_SUBVOLUME_SLES_SETUP variable from /etc/rear/rescue.conf + because its value could have been migrated to a new value (issue #2181). + +- In the function apply_layout_mappings (therein in its "step 3") treat leftover temporary replacement words (like _REAR1_) as an error only if they are in a non-comment line (issue #2183). - o When 420_autoresize_last_partitions.sh resizes the last partition on a - GPT disk it leaves 33 LBA blocks at the end of the disk so that - 420_autoresize_last_partitions.sh produces a fully correct - disklayout.conf file where no further automated adjustments by a - subsequent script like 100_include_partition_code.sh are done (issue # - 2182). +- When 420_autoresize_last_partitions.sh resizes the last partition on a GPT + disk it leaves 33 LBA blocks at the end of the disk so that + 420_autoresize_last_partitions.sh produces a fully correct disklayout.conf + file where no further automated adjustments by a subsequent script like + 100_include_partition_code.sh are done (issue #2182). - o Error out when DHCLIENT_BIN and DHCLIENT6_BIN are empty and USE_DHCLIENT +- Error out when DHCLIENT_BIN and DHCLIENT6_BIN are empty and USE_DHCLIENT is set (issue #2184). - o In build/default/990_verify_rootfs.sh skip the ldd test for kernel - modules because in general running ldd on kernel modules does not make - sense and sometimes running ldd on kernel modules causes needless errors - because sometimes that segfaults (issue #2177). +- In build/default/990_verify_rootfs.sh skip the ldd test for kernel modules + because in general running ldd on kernel modules does not make sense and + sometimes running ldd on kernel modules causes needless errors because + sometimes that segfaults (issue #2177). - o Bacula: Fixed 'bconsole' prompt not displaying at all when doing a - recover (issue #2173). +- Bacula: Fixed 'bconsole' prompt not displaying at all when doing a recover + (issue #2173). - o Fixed that the EFI Boot Manager cannot be installed when the system has a - NVMe SSD, because the device name is resolved as /dev/nvme0n1p rather - than /dev/nvme0n1 so for NVMe devices the trailing 'p' in the Disk value - as in /dev/nvme0n1p that is derived from /dev/nvme0n1p1 needs to be - stripped to get /dev/nvme0n1 (issues #2160 #1564). +- Fixed that the EFI Boot Manager cannot be installed when the system has a + NVMe SSD, because the device name is resolved as /dev/nvme0n1p rather than + /dev/nvme0n1 so for NVMe devices the trailing 'p' in the Disk value as in + /dev/nvme0n1p that is derived from /dev/nvme0n1p1 needs to be stripped to + get /dev/nvme0n1 (issues #2160 #1564). - o Changed double quotes (") to single quotes(') in documentation related to +- Changed double quotes (") to single quotes(') in documentation related to BORG_PASSPHRASE variable. Double quotes will not avoid expanding of - variable when '$' sign is used inside pass phrase string, which can lead - to unsuccessful opening of Borg archive (issue #2205). + variable when'$' sign is used inside pass phrase string, which can lead to + unsuccessful opening of Borg archive (issue #2205). - o More fail safe BACKUP_PROG_CRYPT_KEY handling: Use double quotes +- More fail safe BACKUP_PROG_CRYPT_KEY handling: Use double quotes "$BACKUP_PROG_CRYPT_KEY" so that the BACKUP_PROG_CRYPT_KEY value can contain spaces. Escape special regexp characters in the BACKUP_PROG_CRYPT_KEY value when it is used as a regexp in grep or sed. Use single quotes BACKUP_PROG_CRYPT_KEY='my_passphrase' in the - documentation examples so that the BACKUP_PROG_CRYPT_KEY value can - contain special characters (except single quote). Recommend in - default.conf to not use special characters in the BACKUP_PROG_CRYPT_KEY - value to be to be on the safe side against things breaking in unexpected - weird ways when certain code in ReaR is not yet safe against arbitrary - special characters in values (issues #2157 #1372). - - o Avoid that the BACKUP_PROG_CRYPT_KEY value appears in a log file in - particular when 'rear' is run in debugscript mode where 'set -x' is set - by redirecting STDERR to /dev/null for those confidential commands that - use the BACKUP_PROG_CRYPT_KEY value, cf. the comment of the UserInput - function how to keep things confidential when 'rear' is run in - debugscript mode (issues #2155 #2156). - - o Fix when recovery fails on LUKS-encrypted filesystem using simple - password because only the first 2 parameters in /etc/crypttab are - mandatory (issue #2151). - - o Generic enhancements for IBM Z (s390) to support 'dasd' disks and SUSE + documentation examples so that the BACKUP_PROG_CRYPT_KEY value can contain + special characters (except single quote). Recommend in default.conf to not + use special characters in the BACKUP_PROG_CRYPT_KEY value to be to be on + the safe side against things breaking in unexpected weird ways when + certain code in ReaR is not yet safe against arbitrary special characters + in values (issues #2157 #1372). + +- Avoid that the BACKUP_PROG_CRYPT_KEY value appears in a log file in + particular when 'rear' is run in debugscript mode where 'set -x' is set by + redirecting STDERR to /dev/null for those confidential commands that use + the BACKUP_PROG_CRYPT_KEY value, cf. the comment of the UserInput function + how to keep things confidential when 'rear' is run in debugscript mode + (issues #2155 #2156). + +- Fix when recovery fails on LUKS-encrypted filesystem using simple password + because only the first 2 parameters in /etc/crypttab are mandatory (issue + #2151). + +- Generic enhancements for IBM Z (s390) to support 'dasd' disks and SUSE specific enhancements to install during "rear recover" the special GRUB2 - plus ZIPL bootloader on SLES12 and later via the new finalize/SUSE_LINUX/ - s390/660_install_grub2_and_zipl.sh script (issues #2137 #2150). - - o Cleanup of the OUTPUT=RAMDISK code. Overhauled 900_copy_ramdisk.sh as - architecture independent new script output/RAMDISK/default/ - 900_copy_ramdisk.sh that replaces output/RAMDISK/Linux-i386/ - 900_copy_ramdisk.sh and overhauled conf/templates/ - RESULT_usage_RAMDISK.txt plus better description of OUTPUT=RAMDISK and - RESULT_FILES in default.conf (issue #2148). - - o Enable creation of non consecutive partitions (issues #2081 #1793 #1771 # - 1681). - - o The new SUSE-specific script finalize/SUSE_LINUX/i386/675_install_shim.sh - calls 'shim-install' inside chroot on the recreated target system and - uses boot/grub2/grub.cfg as input to recreate two needed (identical) EFI - files /boot/grub2/x86_64-efi/core.efi and /boot/efi/EFI/opensuse/ - grubx64.efi (issue #2116). - - o Fixed that output/default/950_copy_result_files.sh falsely errors out - when an output method OUTPUT=FOO is used where there is no matching usr/ - share/rear/conf/templates/RESULT_usage_FOO.txt because when it is missing - it is no sufficient reason to error out. Furthermore now + plus ZIPL bootloader on SLES12 and later via the new + finalize/SUSE_LINUX/s390/660_install_grub2_and_zipl.sh script (issues + #2137 #2150). + +- Cleanup of the OUTPUT=RAMDISK code. Overhauled 900_copy_ramdisk.sh as + architecture independent new script + output/RAMDISK/default/900_copy_ramdisk.sh that replaces + output/RAMDISK/Linux-i386/900_copy_ramdisk.sh and overhauled + conf/templates/RESULT_usage_RAMDISK.txt plus better description of + OUTPUT=RAMDISK and RESULT_FILES in default.conf (issue #2148). + +- Enable creation of non consecutive partitions (issues #2081 #1793 #1771 + #1681). + +- The new SUSE-specific script finalize/SUSE_LINUX/i386/675_install_shim.sh + calls 'shim-install' inside chroot on the recreated target system and uses + boot/grub2/grub.cfg as input to recreate two needed (identical) EFI files + /boot/grub2/x86_64-efi/core.efi and /boot/efi/EFI/opensuse/grubx64.efi + (issue #2116). + +- Fixed that output/default/950_copy_result_files.sh falsely errors out when + an output method OUTPUT=FOO is used where there is no matching + usr/share/rear/conf/templates/RESULT_usage_FOO.txt because when it is + missing it is no sufficient reason to error out. Furthermore now 950_copy_result_files.sh is skipped when RESULT_FILES is empty because that means there is no actual output and then it is not needed to only copy VERSION README and LOGFILE to the output location (issue #2147). @@ -911,74 +2160,73 @@ Version 2.5 (May 2019) Abstract -New features, bigger enhancements, and possibly backward incompatible -changes: +New features, bigger enhancements, and possibly backward incompatible changes: - o Enhancements to better support mmcblk/eMMC disks: An "eMMC" device could +- Enhancements to better support mmcblk/eMMC disks: An "eMMC" device could be not only one single disk but actually consist of several 'disk' type block devices for example the actually usable disk /dev/mmcblk0 (with its - partitions like /dev/mmcblk0p1 and /dev/mmcblk0p2) plus special - additional disks on the same eMMC device like /dev/mmcblk0boot0 and /dev/ - mmcblk0boot1 and /dev/mmcblk0rpmb (issue #2087). + partitions like /dev/mmcblk0p1 and /dev/mmcblk0p2) plus special additional + disks on the same eMMC device like /dev/mmcblk0boot0 and /dev/mmcblk0boot1 + and /dev/mmcblk0rpmb (issue #2087). - o Now there is in default.conf MODULES=( 'all_modules' ) which means that +- Now there is in default.conf MODULES=( 'all_modules' ) which means that now by default all kernel modules get included in the recovery system (issue #2041). Usually this is required when migrating to different - hardware. Additionally it makes the recovery system better prepared when + hardware. Additionaly it makes the recovery system better prepared when this or that additional kernel module is needed, e.g. to ensure a USB keyboard is usable in the recovery system (issue #1870) or to ensure data on external medium (e.g. iso9660) can be read (issue #1202). Furthermore this is helpful to be on the safe side against possibly missing dependant kernel modules that are not automatically found (issue #1355). The - drawback of MODULES=( 'all_modules' ) is that it makes the recovery - system (and its ISO image) somewhat bigger (see issue #2041 for some - numbers). With MODULES=() the old behaviour can be still specified. There - is a minor backward incompatible change: Before the user had to specify - in etc/rear/local.conf MODULES=( "${MODULES[@]}" 'moduleX' 'moduleY' ) to - get some specific modules included in addition to the ones via an empty + drawback of MODULES=( 'all_modules' ) is that it makes the recovery system + (and its ISO image) somewhat bigger (see issue #2041 for some numbers). + With MODULES=() the old behaviour can be still specified. There is a minor + backward incompatible change: Before the user had to specify in + etc/rear/local.conf MODULES=( "${MODULES[@]}" 'moduleX' 'moduleY' ) to get + some specific modules included in addition to the ones via an empty MODULES=() but now the user must specify MODULES=( 'moduleX' 'moduleY' ) for that because with "${MODULES[@]}" the new default value 'all_modules' would be kept which would trigger that all modules get included so that - now MODULES=( "${MODULES[@]}" 'moduleX' 'moduleY' ) includes all kernel - modules in the recovery system which includes in particular 'moduleX' and - 'moduleY' so that things still work but with a bigger recovery system. - For details see the MODULES description in default.conf. - - o The new verify script layout/save/default/950_verify_disklayout_file.sh - verifies the disklayout.conf file that is created by "rear mkrescue/ - mkbackup". Currently only some very basic verification is implemented: It - verifies that the 'disk' entries look syntactically correct (only basic - value type testing), the 'part' entries look syntactically correct (only - basic value type testing), the 'part' entries specify consecutive - partitions. The latter is needed to make ReaR more fail-safe in case of - sparse partition schemes (i.e. when there are non-consecutive partitions) - because currently "rear recover" fails when there are non-consecutive - partitions (issue #1681). In general verification of the created - disklayout.conf should help to avoid failures when it is too late (i.e. - when "rear recover" fails). It is better to error out early during "rear - mkrescue/mkbackup". It may happen that layout/save/default/ - 950_verify_disklayout_file.sh falsely lets "rear mkrescue/mkbackup" error - out because of false alarm. The immediate workaround for the user in such - cases is to remove that script or skip what it does by adding a 'return - 0' command at its very beginning. - - o Basic support for EFISTUB booting: Via the new config variable EFI_STUB + now `MODULES=( "${MODULES[@]}" 'moduleX' 'moduleY' )` includes + all kernel modules in the recovery system which includes in particular + 'moduleX' and 'moduleY' so that things still work but with a bigger + recovery system. For details see the MODULES description in default.conf. + +- The new verify script layout/save/default/950_verify_disklayout_file.sh + verifies the disklayout.conf file that is created by "rear + mkrescue/mkbackup". Currently only some very basic verification is + implemented: It verifies that the 'disk' entries look syntactically + correct (only basic value type testing), the 'part' entries look + syntactically correct (only basic value type testing), the 'part' entries + specify consecutive partitions. The latter is needed to make ReaR more + fail-safe in case of sparse partition schemes (i.e. when there are + non-consecutive partitions) because currently "rear recover" fails when + there are non-consecutive partitions (issue #1681). In general + verification of the created disklayout.conf should help to avoid failures + when it is too late (i.e. when "rear recover" fails). It is better to + error out early during "rear mkrescue/mkbackup". It may happen that + layout/save/default/950_verify_disklayout_file.sh falsely lets "rear + mkrescue/mkbackup" error out because of false alarm. The immediate + workaround for the user in such cases is to remove that script or skip + what it does by adding a 'return 0' command at its very beginning. + +- Basic support for EFISTUB booting: Via the new config variable EFI_STUB (see default.conf) the user can (and if needed must) specify that the recreated system should boot via EFISTUB. If EFI_STUB is specified but some boot loader like GRUB2 or ELILO is used on the original system, the recreated system gets migrated to boot (only) via EFISTUB. - o The whole 'rear dump' output format need to be changed to improve it to +- The whole 'rear dump' output format need to be changed to improve it to clearly distinguish array elements. - o Now during "rear mkrescue/mkbackup" md5sums are created for all regular - files in in the recovery system and stored as /md5sums.txt in the - recovery system. During recovery system startup it verifies those - md5sums. Via the new config variable EXCLUDE_MD5SUM_VERIFICATION (see - default.conf) the user can specify what files should be excluded from - being verified to avoid errors on "false positives". +- Now during "rear mkrescue/mkbackup" md5sums are created for all regular + files in in the recovery system and stored as /md5sums.txt in the recovery + system. During recovery system startup it verifies those md5sums. Via the + new config variable EXCLUDE_MD5SUM_VERIFICATION (see default.conf) the + user can specify what files should be excluded from being verified to + avoid errors on "false positives". - o GRUB2 installation on x86 and ppc64le architecture was completely +- GRUB2 installation on x86 and ppc64le architecture was completely rewritten and enhanced by the new config variable GRUB2_INSTALL_DEVICES (see default.conf) so that now the user can specify what he wants if needed and in MIGRATION_MODE disk mappings are applied when devices in @@ -986,151 +2234,149 @@ changes: Details (mostly in chronological order - newest topmost): - o In packaging/rpm/rear.spec reactivated 'BuildRoot' and 'defattr' because - both are required for building 'rear' RPM packages on SLES 11 and RHEL 5 - / CentOS 5 via the openSUSE Build Service and removed /etc/cron.d/rear - and related things (issues #2135 #1855 #1856 #1908 #1892) +- In packaging/rpm/rear.spec reactivated 'BuildRoot' and 'defattr' because + both are required for building 'rear' RPM packages on SLES 11 and RHEL 5 / + CentOS 5 via the openSUSE Build Service and removed /etc/cron.d/rear and + related things (issues #2135 #1855 #1856 #1908 #1892) - o In layout/prepare/default/420_autoresize_last_partitions.sh continue with - the next disk if the current one has no partitions otherwise the "Find - the last partition for the current disk" code fails (issue #2134) +- In layout/prepare/default/420_autoresize_last_partitions.sh continue with + the next disk if the current one has no partitions otherwise the "Find the + last partition for the current disk" code fails (issue #2134) - o Improved handling of broken symlinks inside the recovery system: Relative +- Improved handling of broken symlinks inside the recovery system: Relative symbolic links are now properly handled. Directories, which are link targets, are not copied into the recovery system but a meaningful hint is printed that COPY_AS_IS can be used for that (issues #2129 #2130 #2131) - o In layout/prepare/GNU/Linux/135_include_btrfs_subvolumes_generic_code.sh +- In layout/prepare/GNU/Linux/135_include_btrfs_subvolumes_generic_code.sh fixed 'btrfs subvolume set-default' command for older versions of 'btrfsprogs' where that command requires both arguments 'subvolid' and 'path' (issue #2119) - o For backup NBU: Exclude the whole '/usr/openv/netbackup/logs' directory +- For backup NBU: Exclude the whole '/usr/openv/netbackup/logs' directory instead of only the files in that directory (issue #2132) - o Fixed layout mapping error when TCG Opal 2 self-encrypting disks were - present but had to be excluded due to a non-existent disk during - recovery. This exclusion was not handled properly (issue #2126) +- Fixed layout mapping error when TCG Opal 2 self-encrypting disks were + present but had to be excluded due to a non-existent disk during recovery. + This exclusion was not handled properly (issue #2126) - o In build/default/995_md5sums_rootfs.sh also exclude all files with a +- In build/default/995_md5sums_rootfs.sh also exclude all files with a trailing '~' in their name because those are also excluded when the recovery system initrd is made by pack/GNU/Linux/900_create_initramfs.sh (issue #2127) - o Suppressed unwanted "Welcome to Relax-and-Recover ..." etc/motd messages +- Suppressed unwanted "Welcome to Relax-and-Recover ..." etc/motd messages from 'chroot $ROOTFS_DIR /bin/bash -login ...' calls that appear in the log file or in the stdout of the chroot call when the output is further processes by 'grep' by redirecting stdin of the chroot call to /dev/null because no input is needed (issues #2120 #2125) - o Replace RULE_FILES with a global UDEV_NET_MAC_RULE_FILES in default.conf +- Replace RULE_FILES with a global UDEV_NET_MAC_RULE_FILES in default.conf (issues #2074 #2123) - o Added comment to default.conf that tells when KEEP_BUILD_DIR is +- Added comment to default.conf that tells when KEEP_BUILD_DIR is automatically set to true (issue #2121) - o Added /usr/openv/netbackup/sec/at/lib/ to NBU_LD_LIBRARY_PATH in +- Added /usr/openv/netbackup/sec/at/lib/ to NBU_LD_LIBRARY_PATH in default.conf (issues #2105 #2122) - o Simplified awk constructs in 320_include_uefi_env.sh into using plain - grep and a bash array to avoid inexplicable wrong behaviour in some cases - that is somehow related to the nullglob bash option together with - different kind of awk (issues #2095 #2115) +- Simplified awk constructs in 320_include_uefi_env.sh into using plain grep + and a bash array to avoid inexplicable wrong behaviour in some cases that + is somehow related to the nullglob bash option together with different + kind of awk (issues #2095 #2115) - o For older systems (e.g. like SLES11) where /dev is no mountpoint in the +- For older systems (e.g. like SLES11) where /dev is no mountpoint in the recovery system we first mount TARGET_FS_ROOT/dev as 'tmpfs' and then we - copy all /dev contents from the recovery system into TARGET_FS_ROOT/dev - to make all recovery system /dev contents available at TARGET_FS_ROOT/dev + copy all /dev contents from the recovery system into TARGET_FS_ROOT/dev to + make all recovery system /dev contents available at TARGET_FS_ROOT/dev (which are needed therein for things like "chroot TARGET_FS_ROOT mkinitrd") but only as long as the recovery system runs. On the rebooted target system its pristine /dev will be there. This is basically what finalize/default/100_populate_dev.sh had done but now without dirty remainders on the user's target system disk (issue #2113). - o Do not copy symlink targets in /proc/ /sys/ /dev/ or /run/ into the ReaR - recovery system. For example on SLES11 /lib/udev/devices/core is a - symlink to /proc/kcore so that "rear mkrescue" basically hangs up while - copying /proc/kcore because it is huge (issue #2112) +- Do not copy symlink targets in /proc/ /sys/ /dev/ or /run/ into the ReaR + recovery system. For example on SLES11 /lib/udev/devices/core is a symlink + to /proc/kcore so that "rear mkrescue" basically hangs up while copying + /proc/kcore because it is huge (issue #2112) - o Avoid needless things when there is more than one disk: Avoid tot go into +- Avoid needless things when there is more than one disk: Avoid tot go into MIGRATION_MODE in any case when there is more than one disk. Avoid that - GRUB2 gets needlessly installed two times on the same device (issue # - 2108) + GRUB2 gets needlessly installed two times on the same device (issue #2108) - o Fixed disk device name in efibootmgr call for eMMC devices: For eMMC - devices the trailing 'p' in the disk device name (as in /dev/mmcblk0p - that is derived from /dev/mmcblk0p1) needs to be stripped (to get /dev/ - mmcblk0), otherwise the efibootmgr call fails because of a wrong disk +- Fixed disk device name in efibootmgr call for eMMC devices: For eMMC + devices the trailing 'p' in the disk device name (as in /dev/mmcblk0p that + is derived from /dev/mmcblk0p1) needs to be stripped (to get + /dev/mmcblk0), otherwise the efibootmgr call fails because of a wrong disk device name (issue #2103) - o For Ubuntu 18.x use /run/systemd/resolve/resolv.conf as /etc/resolv.conf +- For Ubuntu 18.x use /run/systemd/resolve/resolv.conf as /etc/resolv.conf in the recovery system: Basically the /etc/resolv.conf symlink target and /lib/systemd/resolv.conf contain only the systemd-resolved stub resolver - "nameserver 127.0.0.53" and only /run/systemd/resolve/resolv.conf - contains a real nameserver (issues #2018 #2101) + "nameserver 127.0.0.53" and only /run/systemd/resolve/resolv.conf contains + a real nameserver (issues #2018 #2101) - o When mktemp needs to be called with a TEMPLATE call it with sufficent - XXXXXXXXXX in the TEMPLATE, otherwise use the mktemp default (issue # - 2092) +- When mktemp needs to be called with a TEMPLATE call it with sufficent + XXXXXXXXXX in the TEMPLATE, otherwise use the mktemp default (issue #2092) - o LPAR/PPC64 bootlist was incorrectly set when having multiple 'prep' +- LPAR/PPC64 bootlist was incorrectly set when having multiple 'prep' partitions: Use the specific right syntax for array expansion of the boot_list array (issues #2096 #2097 #2098 #1068) - o Ensure that the Error function results a direct and complete exit of the +- Ensure that the Error function results a direct and complete exit of the whole running 'rear' program even if the Error function was called from a (possibly nested) subshell in a sourced script: Now the Error function terminates all descendant processes of MASTER_PID except MASTER_PID and - the current (subshell) process that runs the Error function and when - Error was called from a subshell it finally exits its own subshell so - that when the Error function finished only MASTER_PID is still running - and finally MASTER_PID exits as usual via the DoExitTasks function - (issues #2088 #2089 #2099) + the current (subshell) process that runs the Error function and when Error + was called from a subshell it finally exits its own subshell so that when + the Error function finished only MASTER_PID is still running and finally + MASTER_PID exits as usual via the DoExitTasks function (issues #2088 #2089 + #2099) - o Ignore special additional disks on eMMC devices named "rpmb" and "boot": +- Ignore special additional disks on eMMC devices named "rpmb" and "boot": Now the extract_partitions() function skips device nodes on eMMC devices like /dev/mmcblk0rpmb or /dev/mmcblk0boot0 and /dev/mmcblk0boot1 because ReaR wrongly recognized those 'disk' type block devices as if they were 'part' type block devices, i.e. those are no partitions, but special additional disks on the eMMC device (issue #2087) - o Updated the OPALPBA workflow: Set USE_RESOLV_CONF='no' as networking is - not required/available in the PBA. Avoid copying in the entire /etc/ - alternatives directory as its links could pull in lots of unwanted stuff, - which is not required in rescue systems. Clean up plymouth/unlock service - startup (issue #2083) +- Updated the OPALPBA workflow: Set USE_RESOLV_CONF='no' as networking is + not required/available in the PBA. Avoid copying in the entire + /etc/alternatives directory as its links could pull in lots of unwanted + stuff, which is not required in rescue systems. Clean up plymouth/unlock + service startup (issue #2083) - o Network: Record permanent mac address when device is enslaved in a Team, +- Network: Record permanent mac address when device is enslaved in a Team, or else /etc/mac-addresses will record broken information. Use "ethtool -P" as the preferred method to retrieve the MAC address. Otherwise fall back to other methods, which may lead to some invalid MAC address when using Teams (issues #1954 #2065) - o Added 'net-tools' to Debian dependencies as required for 'route' command +- Added 'net-tools' to Debian dependencies as required for 'route' command which belongs to the REQUIRED_PROGS (issue #2082) - o Added an additional separated new btrfs_subvolumes_setup_generic() +- Added an additional separated new btrfs_subvolumes_setup_generic() function to recreate all mounted Btrfs subvolumes in a more generic way. The old btrfs_subvolumes_setup function was renamed into btrfs_subvolumes_setup_SLES and this one is called as fallback to be - backward compatible. For both btrfs_subvolumes_setup implementations it - is individually configurable which one is used for which btrfs device via - the new config variables BTRFS_SUBVOLUME_SLES_SETUP and + backward compatible. For both btrfs_subvolumes_setup implementations it is + individually configurable which one is used for which btrfs device via the + new config variables BTRFS_SUBVOLUME_SLES_SETUP and BTRFS_SUBVOLUME_GENERIC_SETUP. Currently it is not documented because it is work in progress where arbitrary further changes will happen (e.g. the current btrfs_subvolumes_setup_generic function makes diskrestore.sh fail on older systems where 'btrfs subvolume set-default' needs two arguments) - so one has to inspect the current code and comments in the layout/prepare - /GNU/Linux/13X_include_... scripts to see how things currently work (issues - #2067 #2079 #2080 #2084 #2085) + so one has to inspect the current code and comments in the + layout/prepare/GNU/Linux/13X_include_... scripts to see how things currently + work (issues #2067 #2079 #2080 #2084 #2085) - o Fixed SSH root login on the recovery system with some configurations: On - Ubuntu 18.04 with OpenSSH 7.6, /etc/ssh/sshd_config contains - commented-out lines for 'PermitRootLogin' and other options. This fix - makes sure that settings changed for sshd in the ReaR recovery system - will be real, not comments (issue #2070) +- Fixed SSH root login on the recovery system with some configurations: On + Ubuntu 18.04 with OpenSSH 7.6, /etc/ssh/sshd_config contains commented-out + lines for 'PermitRootLogin' and other options. This fix makes sure that + settings changed for sshd in the ReaR recovery system will be real, not + comments (issue #2070) - o RAWDISK output: Improved device partition detection (e.g. Ubuntu 18.04). +- RAWDISK output: Improved device partition detection (e.g. Ubuntu 18.04). On Ubuntu 18.04, it has been observed that after creating a loop device and creating a properly sized VFAT file system >250 MB on it, after mounting the file system size was actually just 30 MB. Reason: The @@ -1140,343 +2386,339 @@ Details (mostly in chronological order - newest topmost): opportunity to detect partitions. If the option is not available, a traditional losetup call will be used as a fallback (issue #2071) - o OPALPBA output fix: Do not include any PBA into another PBA. A TCG Opal +- OPALPBA output fix: Do not include any PBA into another PBA. A TCG Opal pre-boot authentication (PBA) system is a minimal operating system constructed by ReaR to unlock self-encrpyting disks before the regular OS takes over. Before this PR, it could happen that a previously created PBA - was included in a subsequently created PBA, causing it to be - unnecessarily large (issue ##2072) + was included in a subsequently created PBA, causing it to be unnecessarily + large (issue ##2072) - o Now there is in default.conf MODULES=( 'all_modules' ) which means that +- Now there is in default.conf MODULES=( 'all_modules' ) which means that now by default all kernel modules get included in the recovery system (issues #2041 #1870 #1202 #1355) - o New verify script layout/save/default/950_verify_disklayout_file.sh to +- New verify script layout/save/default/950_verify_disklayout_file.sh to verify disklayout.conf that was created by "rear mkrescue/mkbackup" (issues #2060 #1681) - o Refresh udev with trigger before activating multipath (issue #2064): - Ensure that all information from multipath devices is updated by udev - into /sys before activating multipath. This helps to mitigate certain - kind of issues when something wrong in the SAN zoning configuration - (issues #2002 #2016 #2019) +- Refresh udev with trigger before activating multipath (issue #2064): + Ensure that all information from multipath devices is updated by udev into + /sys before activating multipath. This helps to mitigate certain kind of + issues when something wrong in the SAN zoning configuration (issues #2002 + #2016 #2019) - o For backup NSR: It is of additional use to not skip the retrieval of the +- For backup NSR: It is of additional use to not skip the retrieval of the filesystems even in NSR_CLIENT_MODE so that this is now also done in NSR_CLIENT_MODE: Due to saving the save sets filesystem information in $VAR_DIR/recovery/nsr_paths within the recovery image one is able to - retrieve/read this information during a recovery process i.e. for - advising the EMC networker server team to recover the appropriate + retrieve/read this information during a recovery process i.e. for advising + the EMC networker server team to recover the appropriate filesystem(-structure) from the backups beeing made (issue #2058) - o Skip patching absolute symlinks during finalize stage (issue #2055). That +- Skip patching absolute symlinks during finalize stage (issue #2055). That does not actually fix issue #1338 but for now it should at least avoid patching wrong files. Furthermore do no longer create udev rules in the recreated system that have not been there. This way one can avoid that ReaR creates udev rules that are created and maintained by systemd/udev - like /etc/udev/rules.d/70-persistent-net.rules when one excludes such - udev rules from being restored from the backup or by moving them away via + like /etc/udev/rules.d/70-persistent-net.rules when one excludes such udev + rules from being restored from the backup or by moving them away via BACKUP_RESTORE_MOVE_AWAY_FILES (issue #770) - o Now /proc /sys /dev and /run are bind-mounted into TARGET_FS_ROOT at the +- Now /proc /sys /dev and /run are bind-mounted into TARGET_FS_ROOT at the beginning of the finalize stage via the new 110_bind_mount_proc_sys_dev_run.sh script and existing code in various finalize scripts for mounting /proc /sys /dev and things like that was removed and the finalize scripts were adapted and renumbered as needed (issues #2045 #2035) - o Added eno-fix.rules to RULE_FILES for LAN interface MAC address changes - to the ens-style LAN interface names instead of the older eno-type LAN - names and aligned RULE_FILES content in the involved scripts (issue # - 2046) +- Added eno-fix.rules to RULE_FILES for LAN interface MAC address changes to + the ens-style LAN interface names instead of the older eno-type LAN names + and aligned RULE_FILES content in the involved scripts (issue #2046) - o Fixed 58-start-dhclient.sh script to make it work reliably with multiple +- Fixed 58-start-dhclient.sh script to make it work reliably with multiple network interfaces (issue #2038) - o Borg backup restore enhancements and fixes: Added checks whether we can +- Borg backup restore enhancements and fixes: Added checks whether we can read Borg archive, user can enable progress display via BORGBACKUP_SHOW_PROGRESS="yes", corrected mounting of USB device when using Borg (issues #2029 #2037) - o Fix for GRUB2 EFI modules search directory location: Instead of looking +- Fix for GRUB2 EFI modules search directory location: Instead of looking for GRUB2 modules only in /boot also find them in /usr/lib/grub*, where GRUB2 modules are normally installed by default (issue #2039) - o Basic support for EFISTUB booting plus documentation (issues #1942 #2030) +- Basic support for EFISTUB booting plus documentation (issues #1942 #2030) - o Multipath optimizations: Optimized get_device_name() by calling "dmsetup +- Multipath optimizations: Optimized get_device_name() by calling "dmsetup info" only once, and for "dm" devices only. Removed collecting output of /sys/class/fc_transport since it can be very slow and is not used (issues #2020 #2034) - o Suppress dispensable 'set -x' debug output unless called with +- Suppress dispensable 'set -x' debug output unless called with '-debugscripts x': A noticeable part (25% and more) of the 'set -x' - debugscripts output is usually of no interest and therefore such output - is suppressed by default (e.g. when rear is called with '-D') unless rear - is called with '-debugscripts x' where the full debugscripts output is - still there as it was before (issue #2024) + debugscripts output is usually of no interest and therefore such output is + suppressed by default (e.g. when rear is called with '-D') unless rear is + called with '-debugscripts x' where the full debugscripts output is still + there as it was before (issue #2024) - o Cleaned up the Docker specific exclude part in 230_filesystem_layout.sh: +- Cleaned up the Docker specific exclude part in 230_filesystem_layout.sh: Determine docker_root_dir only once and try to be safer against possibly crippled Docker installations (e.g. timeout 'docker info') and be safe against empty docker_root_dir (otherwise all mountpoints would match the empty string and we would would skip all mountpoints) and show possible errors to the user in any case (issues #1989 #2021) - o Improved setup of /etc/resolv.conf in the recovery system: In case of - static networking setup in the recovery system a plain traditional /etc/ - resolv.conf file with an entry of a remote 'nameserver +- Improved setup of /etc/resolv.conf in the recovery system: In case of + static networking setup in the recovery system a plain traditional + /etc/resolv.conf file with an entry of a remote 'nameserver DNS.server.IP.address' is needed. It cannot work when /etc/resolv.conf contains only loopback IP addresses (which happens when the stub resolver systemd-resolved is used) or when there is no nameserver entry so that "rear mkrescue/mkbackup" errors out in this case. In contrast when USE_DHCLIENT is ture (e.g. when DHCP is used on the original system) then - during recovery system startup /etc/resolv.conf will be generated by /bin - /dhclient-script so that it does not matter what its content was before. - For special cases the user can specify what he wants via the new + during recovery system startup /etc/resolv.conf will be generated by + /bin/dhclient-script so that it does not matter what its content was + before. For special cases the user can specify what he wants via the new USE_RESOLV_CONF variable (issues #2015 #2018 #2076) - o Improved 'rear dump' output to clearly distinguish array elements. The +- Improved 'rear dump' output to clearly distinguish array elements. The whole 'rear dump' output format was changed. Now it shows normal string - variables as STRING="string of words" and arrays as a beautified/ - simplified 'declare -p' output as ARRAY=("first element" "second element" - ... ) unless in debug mode where the plain 'declare -p' output is shown - like declare -- STRING="string of words" and declare -a ARRAY=([0]="first - element" [1]="second element" ... ) that is needed for complicated array - elements that contain brackets. Furthermore now the 'rear dump' output - can be directly sourced (issue #2014) - - o Error out when we cannot make a bootable EFI image of GRUB2 which is + variables as STRING="string of words" and arrays as a + beautified/simplified 'declare -p' output as + ARRAY=("first element" "second element" ... ) unless in debug mode where + the plain 'declare -p' output is shown like + declare -- STRING="string of words" and + declare -a ARRAY=([0]="first element" [1]="second element" ... ) that is + needed for complicated array elements that contain brackets. Furthermore + now the 'rear dump' output can be directly sourced (issue #2014) + +- Error out when we cannot make a bootable EFI image of GRUB2 which is required when UEFI is used (issue 2013) - o Allow non-interactive authentication with rsync by using +- Allow non-interactive authentication with rsync by using BACKUP_RSYNC_OPTIONS to specify the "-password-file=/full/path/to/file" rsync option (issue #2011) - o Add possibility for user to specify whether or not show Borg stats at the +- Add possibility for user to specify whether or not show Borg stats at the end of backup session (issue #2008) - o Adaption for Fedora 29: grub2-efi-x64-modules does not contain linuxefi +- Adaption for Fedora 29: grub2-efi-x64-modules does not contain linuxefi module (issue #2001) - o Let the user optionally specify mkfs.xfs options if needed to recreate - XFS filesystems with different options than before (e.g. in - MIGRATION_MODE because of different replacement hardware) via the new - MKFS_XFS_OPTIONS config variable (issue #2005) +- Let the user optionally specify mkfs.xfs options if needed to recreate XFS + filesystems with different options than before (e.g. in MIGRATION_MODE + because of different replacement hardware) via the new MKFS_XFS_OPTIONS + config variable (issue #2005) - o Network interface renaming: Automatically map device to its new name when +- Network interface renaming: Automatically map device to its new name when device is found on the system with same MAC address but different name (issue #2004) - o It is no BugError when neither getty nor agetty are avaiable. Such a case +- It is no BugError when neither getty nor agetty are avaiable. Such a case it is an Error because the user must have the programs in REQUIRED_PROGS installed on his system - o Fix for 'error:unrecognized number' when booting ISO on PPC: Issue is +- Fix for 'error:unrecognized number' when booting ISO on PPC: Issue is caused by incorrect, according to PAPR specification, bootinfo entity parsing code in GRUB2 (issue #1978) - o Fixed that in recovery system DHCP client did not iterate through all +- Fixed that in recovery system DHCP client did not iterate through all network interfaces. It incorrectly took only the first one. If the interface that is intended for recovery was not the first one, it had failed (issue #1986) - o Automatically exclude BUILD_DIR from the backup. When TMPDIR was - specified to something not in /tmp, BUILD_DIR was not automatically - excluded from the backup (issue #1993) +- Automatically exclude BUILD_DIR from the backup. When TMPDIR was specified + to something not in /tmp, BUILD_DIR was not automatically excluded from + the backup (issue #1993) - o Support ISOs bigger than 4GiB with OUTPUT=ISO by including the udf kernel +- Support ISOs bigger than 4GiB with OUTPUT=ISO by including the udf kernel module into the recovery system when mkisofs or genisoimage is used (not needed for xorrisoifs) and enable creating an hybrid iso9660/udf DVD (issue #1836) - o Cleaned up how KERNEL_FILE is set: Now the KERNEL_FILE variable is set in +- Cleaned up how KERNEL_FILE is set: Now the KERNEL_FILE variable is set in the 'prep' stage only by the new prep/GNU/Linux/400_guess_kernel.sh that replaces the old pack/GNU/Linux/400_guess_kernel.sh and all the various - different 300_copy_kernel.sh scripts (except output/USB/Linux-i386/ - 830_copy_kernel_initrd.sh) are removed because the 300_copy_kernel.sh - scripts had also only set the KERNEL_FILE variable in various ways. - Additionally it errors out in various output stage scripts when kernel or - initrd are missing and it shows aligned error messages in those cases to - the user. Finally it errors out when the file does not exist or is a - broken symlink. (issues #1851 #1983 #1985) - - o Enhanced and more robust exclude of vfat filesystem (ESP) from SELinux + different 300_copy_kernel.sh scripts (except + output/USB/Linux-i386/830_copy_kernel_initrd.sh) are removed because the + 300_copy_kernel.sh scripts had also only set the KERNEL_FILE variable in + various ways. Additionally it errors out in various output stage scripts + when kernel or initrd are missing and it shows aligned error messages in + those cases to the user. Finally it errors out when the file does not + exist or is a broken symlink. (issues #1851 #1983 #1985) + +- Enhanced and more robust exclude of vfat filesystem (ESP) from SELinux relabeling during recover process (issue #1977) - o Fixed NBU (NetBackup) not working since ReaR 2.4: With NetBackup, too - many binaries were included, causing the recovery system verification to - fail or print error messages. A new NBU_LD_LIBRARY_PATH variable is used - during verification. The RequiredSharedOjects() function was fixed to not - list the left part of the ldd mapping when there is a right part: Some - NetBackup libraries have a mapping such as "/lib/ld64.so => /lib64/ - ld-linux-x86-64.so.2", causing the RequiredSharedOjects() function to - print "/lib/ld64.so" which does not resolve (issue #1974) +- Fixed NBU (NetBackup) not working since ReaR 2.4: With NetBackup, too many + binaries were included, causing the recovery system verification to fail + or print error messages. A new NBU_LD_LIBRARY_PATH variable is used during + verification. The RequiredSharedOjects() function was fixed to not list + the left part of the ldd mapping when there is a right part: Some + NetBackup libraries have a mapping such as "/lib/ld64.so => + /lib64/ld-linux-x86-64.so.2", causing the RequiredSharedOjects() function + to print "/lib/ld64.so" which does not resolve (issue #1974) - o Add /etc/ca-certificates directory to recovery system (issue #1971) +- Add /etc/ca-certificates directory to recovery system (issue #1971) - o Skip 630_run_efibootmgr.sh when UEFI_BOOTLOADER is empty or not a regular +- Skip 630_run_efibootmgr.sh when UEFI_BOOTLOADER is empty or not a regular file and determine the ESP mountpoint from UEFI_BOOTLOADER and use $TARGET_FS_ROOT/boot/efi as fallback plus corrected logic whether or not NOBOOTLOADER is set empty (issues #1942 #1945) - o Missing libraries in recovery system caused executables to fail because +- Missing libraries in recovery system caused executables to fail because libraries were skipped from copying when their library path was a substring of another already copied library, for example /path/to/lib was skipped when /other/path/to/lib was already copied (issue #1975) - o Check for carriage return in local/site/rescue.conf files (issue #1965) +- Check for carriage return in local/site/rescue.conf files (issue #1965) - o Specific required programs are added to REQUIRED_PROGS depending on what +- Specific required programs are added to REQUIRED_PROGS depending on what component keywords are used in disklayout.conf (issue #1963) - o Cleanup and enhancement of REQUIRED_PROGS and PROGS checks (issue #1962) +- Cleanup and enhancement of REQUIRED_PROGS and PROGS checks (issue #1962) - o Moved PROGS from GNU/Linux.conf to default.conf REQUIRED_PROGS (issue # - 892) +- Moved PROGS from GNU/Linux.conf to default.conf REQUIRED_PROGS (issue + #892) - o Record permanent mac address when device is enslaved in a bond, or else / - etc/mac-addresses will record broken information (issue #1954) +- Record permanent mac address when device is enslaved in a bond, or else + /etc/mac-addresses will record broken information (issue #1954) - o For Slackware Linux: Added automatic OS vendor and version detection. +- For Slackware Linux: Added automatic OS vendor and version detection. Added logic to search for kernel files when the standard ELILO kernel install paths come up empty. Added legacy LILO bootloader support. - o Enhanced and cleaned up 985_fix_broken_links.sh to find symlinks in the +- Enhanced and cleaned up 985_fix_broken_links.sh to find symlinks in the recovery system where the link target is outside of the recovery system and in such cases the missing link target gets copied into the recovery system plus more verbose error reporting to the user (issue #1638) - o Fixed that the FindStorageDrivers function failed on kernels with no +- Fixed that the FindStorageDrivers function failed on kernels with no modules. When MODULES=( 'no_modules' ) is set FindStorageDrivers() is now skipped (issue #1941) - o Increased the default USB_UEFI_PART_SIZE from 200 to 400 MiB (issue # - 1205) +- Increased the default USB_UEFI_PART_SIZE from 200 to 400 MiB (issue #1205) - o Enhanced rescue/GNU/Linux/310_network_devices.sh for Bonding and Teaming +- Enhanced rescue/GNU/Linux/310_network_devices.sh for Bonding and Teaming when it is configured with LACP. Bonding should not be simplified in this - case plus added code for non-simplified Teaming (issue #1926). - Furthermore simplified/hardened the tests/setup1 and tests/setup2 - testbeds. + case plus added code for non-simplified Teaming (issue #1926). Furthermore + simplified/hardened the tests/setup1 and tests/setup2 testbeds. - o In 500_clone_keyboard_mappings.sh avoid running find in the whole tree of +- In 500_clone_keyboard_mappings.sh avoid running find in the whole tree of filesystems which takes ages (issue #1906) - o Let 100_include_partition_code.sh error out when the parted command is - not there (issue #1933) +- Let 100_include_partition_code.sh error out when the parted command is not + there (issue #1933) - o Suppress bash stderr messages like 'unary operator expected' or 'integer +- Suppress bash stderr messages like 'unary operator expected' or 'integer expression expected' where a particular 'test' intentionally also fails this way (issue #1931) - o Show errors to the user via LogPrintError instead of only LogPrint - because LogPrint outputs only in case of verbose mode (issue #1914) +- Show errors to the user via LogPrintError instead of only LogPrint because + LogPrint outputs only in case of verbose mode (issue #1914) - o Avoid dangling symlinks when copying ReaR config files into the ReaR +- Avoid dangling symlinks when copying ReaR config files into the ReaR recovery system by copying the content of the symlink target via 'cp -L' (issue #1923) - o Avoid 'nullglob' pitfalls in 250_populate_efibootimg.sh and generally +- Avoid 'nullglob' pitfalls in 250_populate_efibootimg.sh and generally overhalued that script (issue #1921) - o Check that CONFIG_DIR is a directory (issue #1815) +- Check that CONFIG_DIR is a directory (issue #1815) - o Avoid bash error messages like: "eq: unary operator expected" when using +- Avoid bash error messages like: "eq: unary operator expected" when using older versions of xfsprogs (issue #1915) - o Added exclusion of zram devices from device mapping in default.conf. By +- Added exclusion of zram devices from device mapping in default.conf. By default zram devices are not to be mapped, exactly as it is done for ramdisk and loop devices (issue #1916) - o Fixed a non-working awk command in the generate_layout_dependencies() - function that falsely also matched commented '#btrfsmountedsubvol' - entries in disklayout.conf with an egrep command that is more in line - with how it had worked before (issue #1497) +- Fixed a non-working awk command in the generate_layout_dependencies() + function that falsely also matched commented '#btrfsmountedsubvol' entries + in disklayout.conf with an egrep command that is more in line with how it + had worked before (issue #1497) - o Fixed and enhanced NETFS+tar backup pipe exit code handling (issue #1913) +- Fixed and enhanced NETFS+tar backup pipe exit code handling (issue #1913) - o Moved the functionality of the recovery system setup script +- Moved the functionality of the recovery system setup script 67-check-by-label-cdrom.sh into the mount_url function 'iso' case plus additional enhancements there with a user dialog if things are not o.k. and removed the no longer needed 67-check-by-label-cdrom.sh (issues #1893 #1891 #326) - o Fixed that on LPAR/PPC64 PowerVM the boot devices order list was +- Fixed that on LPAR/PPC64 PowerVM the boot devices order list was incorrectly set (via 'bootlist') when having multiple 'prep' partitions. Now handling of multiple 'prep' partitions was added plus enhanced handling of multiple prep partitions and multipath (issue #1886) - o Fixed and enhanced the get_disk_size and get_block_size functions so that +- Fixed and enhanced the get_disk_size and get_block_size functions so that now by default blockdev is used (if exists) to retrieve the size of the - disk and its block size and compute partition start using 512 bytes - blocks (this is hardcoded in the Linux kernel) to fix wrong partition - information when a disk has 4K block size (issue #1884) + disk and its block size and compute partition start using 512 bytes blocks + (this is hardcoded in the Linux kernel) to fix wrong partition information + when a disk has 4K block size (issue #1884) - o Print multipath device name during "rear recover" when "firendly_name" +- Print multipath device name during "rear recover" when "firendly_name" option is off (issue #1889) - o Now the Error function shows some last messages of the last sourced - script to the user (issues #1877 #1875) +- Now the Error function shows some last messages of the last sourced script + to the user (issues #1877 #1875) - o Duplicity: Misc improvements (issues #1876 #1879 #1882) +- Duplicity: Misc improvements (issues #1876 #1879 #1882) - o Added code to recognize persistent LAN interface and manipulate +- Added code to recognize persistent LAN interface and manipulate KERNEL_CMDLINE (issue #1874) - o Initial tentative support for OBDR on ppc64le (issue #1868) +- Initial tentative support for OBDR on ppc64le (issue #1868) - o Wait for systemd-udevd to avoid broken pipe error in +- Wait for systemd-udevd to avoid broken pipe error in 40-start-udev-or-load-modules.sh (issue #1832) - o Aviod duplicate UUID in boot menuentry when snapper is used (issue #1871) +- Aviod duplicate UUID in boot menuentry when snapper is used (issue #1871) - o Added choice to confirm identical layout mapping only once plus disabling +- Added choice to confirm identical layout mapping only once plus disabling MIGRATION_MODE (issue #1857) - o Verify md5sums of files in recovery system (issues #1859 #1895) +- Verify md5sums of files in recovery system (issues #1859 #1895) - o Fedora28: syslinux needs libcom32.c32 to boot from HD and missing +- Fedora28: syslinux needs libcom32.c32 to boot from HD and missing ldlinux.c32 and libutil.c32 prevents PXE booting (issues #1861, #1866) - o Add support for Slackware UEFI/USB (issues #1853, #1863) +- Add support for Slackware UEFI/USB (issues #1853, #1863) - o RAWDISK output portability improvements (issue #1846) +- RAWDISK output portability improvements (issue #1846) - o Fixed, simplified, and enhanced GRUB2 installation on x86 and ppc64le +- Fixed, simplified, and enhanced GRUB2 installation on x86 and ppc64le architecture (issues #1828, #1845, #1847, #1437) Version 2.4 (June 2018) Abstract -New features, bigger enhancements, and possibly backward incompatible -changes: +New features, bigger enhancements, and possibly backward incompatible changes: - o Major rework and changed default behaviour how ReaR behaves in migration +- Major rework and changed default behaviour how ReaR behaves in migration mode when partitions can or must be resized to fit on replacement disks with different size. The new default behaviour is that only the partition end value of the last partition on a disk (and therefore its partition size) may get changed if really needed but no partition start value gets changed to avoid changes of the partition alignment. The new - 420_autoresize_last_partitions script implements the new behaviour and - the old 400_autoresize_disks was renamed into - 430_autoresize_all_partitions to still provide the old behaviour if that - is explicitly requested by the user but the old behaviour may result - unexpected changes of arbitrary partitions on a disk. The new config - variables AUTORESIZE_PARTITIONS AUTORESIZE_EXCLUDE_PARTITIONS - AUTOSHRINK_DISK_SIZE_LIMIT_PERCENTAGE + 420_autoresize_last_partitions script implements the new behaviour and the + old 400_autoresize_disks was renamed into 430_autoresize_all_partitions to + still provide the old behaviour if that is explicitly requested by the + user but the old behaviour may result unexpected changes of arbitrary + partitions on a disk. The new config variables AUTORESIZE_PARTITIONS + AUTORESIZE_EXCLUDE_PARTITIONS AUTOSHRINK_DISK_SIZE_LIMIT_PERCENTAGE AUTOINCREASE_DISK_SIZE_THRESHOLD_PERCENTAGE determine how ReaR behaves in migration mode when partitions can or must be resized. With AUTORESIZE_PARTITIONS='yes' the old behaviour is done. With AUTORESIZE_PARTITIONS='no' no partition is resized by ReaR. With the - default AUTORESIZE_PARTITIONS='' at most the last active partition on - each active disk gets resized but only if really needed which also - depends on the settings of the other config variables above. For details - see default.conf and the two 'autoresize' scripts. For some examples see + default AUTORESIZE_PARTITIONS='' at most the last active partition on each + active disk gets resized but only if really needed which also depends on + the settings of the other config variables above. For details see + default.conf and the two 'autoresize' scripts. For some examples see https://github.com/rear/rear/pull/1733 - o Network setup was completely reworked to support bonding, bridges, vlans +- Network setup was completely reworked to support bonding, bridges, vlans and teaming. There is a full rewrite of the 310_network_devices.sh script generating network interfaces for use during ReaR rescue/recovery system networking setup via the 60-network-devices.sh script. It also handles @@ -1484,149 +2726,147 @@ changes: when the administrator uses bonding plus bridges plus vlans as well as teaming. - o Initial (limited) support for certain ARM based hardware. It should work +- Initial (limited) support for certain ARM based hardware. It should work with Raspberry Pis, most TI and Allwinner devices. There are two ARM - specific BOOTLOADER variable values where 'ARM-ALLWINNER' is for - Allwinner devices that will backup and restore the 2nd stage bootloader - versus plain 'ARM' which is a dummy that does nothing so that on - Raspberry Pi and most TI devices you need to include the first FAT - partition (with the MLO or bootcode.bin) in your backup. + specific BOOTLOADER variable values where 'ARM-ALLWINNER' is for Allwinner + devices that will backup and restore the 2nd stage bootloader versus plain + 'ARM' which is a dummy that does nothing so that on Raspberry Pi and most + TI devices you need to include the first FAT partition (with the MLO or + bootcode.bin) in your backup. - o Simplified and enhanced TSM restore plus first draft of TSM backup. +- Simplified and enhanced TSM restore plus first draft of TSM backup. - o EMC Avamar support has been added. +- EMC Avamar support has been added. - o Duplicity backup has been seriously enhanced. +- Duplicity backup has been seriously enhanced. - o Support for TCG Opal 2-compliant self-encrypting disks and RAWDISK - output. +- Support for TCG Opal 2-compliant self-encrypting disks and RAWDISK output. - o YUM+backup adds the ability to backup and restore files to the YUM - method. +- YUM+backup adds the ability to backup and restore files to the YUM method. Details (mostly in chronological order): - o Copy backup restore log into recreated system (issue #1803) +- Copy backup restore log into recreated system (issue #1803) - o Sesam integration: add sesam bin directory to LD_LIBRARY_PATH (issue # - 1817) +- Sesam integration: add sesam bin directory to LD_LIBRARY_PATH (issue + #1817) - o ReaR recovery fails when the OS contains a Thin Pool/Volume (issues # - 1380, #1806) +- ReaR recovery fails when the OS contains a Thin Pool/Volume (issues #1380, + #1806) - o Make SLES12-GA/SP0 btrfs recovery work again (issues #1796, #1813) +- Make SLES12-GA/SP0 btrfs recovery work again (issues #1796, #1813) - o Verify if dm-X is a partition before adding to sysfs_paths (issue #1805) +- Verify if dm-X is a partition before adding to sysfs_paths (issue #1805) - o Do not start multipathd when not needed (issue #1804) +- Do not start multipathd when not needed (issue #1804) - o Better way to get multiapth partion name (issue #1802) +- Better way to get multiapth partion name (issue #1802) - o Exclude multipath device that does not have mounted fs (issue #1801) +- Exclude multipath device that does not have mounted fs (issue #1801) - o Do not print each files restores by TSM in main output (issue #1797) +- Do not print each files restores by TSM in main output (issue #1797) - o man page BACKUP SOFTWARE INTEGRATION update (issues #1788, #1791) +- man page BACKUP SOFTWARE INTEGRATION update (issues #1788, #1791) - o In the DRLM specific function drlm_import_runtime_config() solve problem +- In the DRLM specific function drlm_import_runtime_config() solve problem with some variables loading config from DRLM (issue #1794) - o Better describe NON_FATAL_BINARIES_WITH_MISSING_LIBRARY in default.conf +- Better describe NON_FATAL_BINARIES_WITH_MISSING_LIBRARY in default.conf (issues #1792, #1693) - o Introduced KEYMAPS_DEFAULT_DIRECTORY and KEYMAP variables, see the +- Introduced KEYMAPS_DEFAULT_DIRECTORY and KEYMAP variables, see the default.conf file (issues #1781, #1787) - o Fix multipath partition replacement and multipath partition naming (issue +- Fix multipath partition replacement and multipath partition naming (issue #1765) - o Skip LUKS encrypted disks when guessing bootloader (issue #1779) +- Skip LUKS encrypted disks when guessing bootloader (issue #1779) - o First draft of TSM backup (issue #1348) +- First draft of TSM backup (issue #1348) - o HP RAID code was updated as the new executable is now called as "ssacli" +- HP RAID code was updated as the new executable is now called as "ssacli" (issue #1760) - o Exclude docker file systems from layout (issue #1749) +- Exclude docker file systems from layout (issue #1749) - o Added migation mode confirmation at beginning of finalize stage (issue # - 1758) +- Added migation mode confirmation at beginning of finalize stage (issue + #1758) - o Show descendant processes PIDs with their commands in the log via pstree +- Show descendant processes PIDs with their commands in the log via pstree or ps as fallback (issues #1755, #1756) - o Check for 'Hah!IdontNeedEFI' GUID number for a GPT BIOS boot partition +- Check for 'Hah!IdontNeedEFI' GUID number for a GPT BIOS boot partition (issues #1752, #1754, #1780) - o Fixed invalid reported return code (always 0) upon NBU restoration - failure (issue #1751) +- Fixed invalid reported return code (always 0) upon NBU restoration failure + (issue #1751) - o Run exit tasks code with default bash flags and options (issues #700, # - 1747, #1748) +- Run exit tasks code with default bash flags and options (issues #700, + #1747, #1748) - o Major rework and changed default behaviour regarding - AUTORESIZE_PARTITIONS (issues #102, #1731, #1733, #1746) +- Major rework and changed default behaviour regarding AUTORESIZE_PARTITIONS + (issues #102, #1731, #1733, #1746) - o YUM+backup adds the ability to backup and restore files to the YUM method +- YUM+backup adds the ability to backup and restore files to the YUM method (issues #1464, #1740) - o Add dbus user and group by default (issues #1710, #1743) +- Add dbus user and group by default (issues #1710, #1743) - o Add gsk libs to TSM_LD_LIBRARY_PATH (issue #1744) +- Add gsk libs to TSM_LD_LIBRARY_PATH (issue #1744) - o Use 'grub2-install -no-nvram' on PowerNV system (issue #1742) +- Use 'grub2-install -no-nvram' on PowerNV system (issue #1742) - o We can use chronyd as time syncing mechanism now as well (issue #1739) +- We can use chronyd as time syncing mechanism now as well (issue #1739) - o Trace and fix broken symbolic links in rootfs (issues #1638, #1734) +- Trace and fix broken symbolic links in rootfs (issues #1638, #1734) - o Borg Backup can now use USB disk as well as backup storage area (issue # - 1730) +- Borg Backup can now use USB disk as well as backup storage area (issue + #1730) - o Initial support for ARM (issue #1662) +- Initial support for ARM (issue #1662) - o Improve the network parameters on the Linux Kernel command line (issue # - 1725) +- Improve the network parameters on the Linux Kernel command line (issue + #1725) - o Clean termination of descendant processes (issues #1712, #1720) +- Clean termination of descendant processes (issues #1712, #1720) - o Simpler and more fail-safe SLE btrfs-example.conf files (issues #1714, # - 1716) +- Simpler and more fail-safe SLE btrfs-example.conf files (issues #1714, + #1716) - o Use a fallback to get interface state using the 'carrier' status (issues +- Use a fallback to get interface state using the 'carrier' status (issues #1701, #1719) - o Fix duplicity backup (issue #1695) +- Fix duplicity backup (issue #1695) - o Include Bareos plugin directory to make bareos-fd start reliably (issues +- Include Bareos plugin directory to make bareos-fd start reliably (issues #1692, #1708) - o Again support GPT partition names with blanks (issues #212, #1563, #1706) +- Again support GPT partition names with blanks (issues #212, #1563, #1706) - o Improvements around Borg Backup (issues #1698, #1700) +- Improvements around Borg Backup (issues #1698, #1700) - o Automatically add 'missing' devices to MD arrays with not enough physical +- Automatically add 'missing' devices to MD arrays with not enough physical devices upon restore (issue #1697) - o Network setup was completely reworked to support bonding, bridges, vlans +- Network setup was completely reworked to support bonding, bridges, vlans and teaming (issue #1574) - o Fixed restore backup when BACKUP_INTEGRITY_CHECK=1 (issue #1685) +- Fixed restore backup when BACKUP_INTEGRITY_CHECK=1 (issue #1685) - o Support TCG Opal 2-compliant self-encrypting disks and RAWDISK output +- Support TCG Opal 2-compliant self-encrypting disks and RAWDISK output (issue #1659) - o Add EMC Avamar backup (issues #1621, #1677, #1687) +- Add EMC Avamar backup (issues #1621, #1677, #1687) - o Avoid falsely detected changed layout for 'rear checklayout' (issues # - 1657, #1658, #1673) +- Avoid falsely detected changed layout for 'rear checklayout' (issues + #1657, #1658, #1673) - o Simplified TSM dsmc restore and improved TSM connection test (issues # - 1534, #1643, #1645) +- Simplified TSM dsmc restore and improved TSM connection test (issues + #1534, #1643, #1645) - o Duplicity with duply waits forever (issues #1664, #1672) +- Duplicity with duply waits forever (issues #1664, #1672) - o Duplicity: Add Support for NETFS URLs (issues #1554, #1665, #1668, #1669) +- Duplicity: Add Support for NETFS URLs (issues #1554, #1665, #1668, #1669) Many minor fixes (too many to list them all - use 'git log' to view them). A big thank you to all contributors as without you it would be impossible to @@ -1638,80 +2878,79 @@ Abstract New features and bigger enhancements: - o First steps towards running Relax-and-Recover unattended in general. +- First steps towards running Relax-and-Recover unattended in general. Several user dialogs that had been implemented in ReaR via the bash builtin 'read' or the bash 'select' keyword are now implemented via the new UserInput function. The UserInput function proceeds with a default input value after a timeout so that it is now possible to let ReaR run unattended with its default behaviour. Additionally one can predefine an automated input value for each particular UserInput function call so that - it is now also possible for the user to predefine what ReaR should do - when running unattended. For details see the USER_INPUT_... config - variables in default.conf. Currently not all user dialogs use the - UserInput function so that this or that user dialog needs to be adapted - when it is reported to us via our issue tracker. In contrast when - programs that are called by ReaR work interactively (e.g. third-party - backup tools that show user dialogs or password prompts) the program call - itself must be adapted to run unattended (if possible), see the section - 'It should be possible to run ReaR unattended' in our https://github.com/ - rear/rear/wiki/Coding-Style Wiki article. - - o SSH support in the ReaR rescue/recovery system was overhauled. By default + it is now also possible for the user to predefine what ReaR should do when + running unattended. For details see the USER_INPUT_... config variables in + default.conf. Currently not all user dialogs use the UserInput function so + that this or that user dialog needs to be adapted when it is reported to + us via our issue tracker. In contrast when programs that are called by + ReaR work interactively (e.g. third-party backup tools that show user + dialogs or password prompts) the program call itself must be adapted to + run unattended (if possible), see the section 'It should be possible to + run ReaR unattended' in our https://github.com/rear/rear/wiki/Coding-Style + Wiki article. + +- SSH support in the ReaR rescue/recovery system was overhauled. By default it is now secure which means the recovery system is free of SSH secrets. Individual settings can be specified via the SSH_FILES, SSH_UNPROTECTED_PRIVATE_KEYS, and SSH_ROOT_PASSWORD config variables (for details see default.conf). - o Improved verification of the ReaR rescue/recovery system contents. Now +- Improved verification of the ReaR rescue/recovery system contents. Now during 'rear mkrescue/mkbackup' there is a verification step where 'ldd' tests for each program/binary and library in the recovery system whether or not its required binaries/libraries can be found in the recovery system. - o Improved autodetection during 'rear recover' when disks on the - replacement hardware seem to not match compared to what there was on the - original system so that ReaR is now more fail-safe against recreating on - a possibly wrong disk. +- Improved autodetection during 'rear recover' when disks on the replacement + hardware seem to not match compared to what there was on the original + system so that ReaR is now more fail-safe against recreating on a possibly + wrong disk. Possibly backward incompatible changes: - o In addition to STDERR now also STDOUT is redirected into the ReaR log - file. Accordingly all output of programs that are called by ReaR is now - in the log file so that the log file content is more complete and there - is no longer unintended verbose information from programs on the terminal - where ReaR was lauched. On the other hand this means when programs prompt - via STDOUT to get some user input (e.g. a program prompts for a user - confirmation under this or that circumstances) the program's STDOUT - prompt is no longer visible to the user when the program was not called - properly in the particular ReaR script as described in the section 'What - to do with stdin, stdout, and stderr' in our https://github.com/rear/rear - /wiki/Coding-Style Wiki article. We tried to fix as many program calls as - possible but it is impossible (with reasonable effort / with a reasonable - amount of time) to check all program calls in all ReaR scripts so that - this or that unnoticed program call will need to be fixed when it is - reported to us via our issue tracker. - - o SSH support in the ReaR rescue/recovery system is now secure by default. - There are no longer private SSH keys in the recovery system by default - and a RSA key is generated from scratch when starting sshd during - recovery system startup. Accordingly it does no longer work by default to - use SSH in the recovery system via the SSH keys that exist on the - original system. To get SSH keys included in the recovery system use the - SSH_FILES and SSH_UNPROTECTED_PRIVATE_KEYS config variables (for details - see default.conf). - - o Verification of required binaries/libraries in the ReaR rescue/recovery +- In addition to STDERR now also STDOUT is redirected into the ReaR log + file. Accordingly all output of programs that are called by ReaR is now in + the log file so that the log file content is more complete and there is no + longer unintended verbose information from programs on the terminal where + ReaR was lauched. On the other hand this means when programs prompt via + STDOUT to get some user input (e.g. a program prompts for a user + confirmation under this or that circumstances) the program's STDOUT prompt + is no longer visible to the user when the program was not called properly + in the particular ReaR script as described in the section 'What to do with + stdin, stdout, and stderr' in our + https://github.com/rear/rear/wiki/Coding-Style Wiki article. We tried to + fix as many program calls as possible but it is impossible (with + reasonable effort / with a reasonable amount of time) to check all program + calls in all ReaR scripts so that this or that unnoticed program call will + need to be fixed when it is reported to us via our issue tracker. + +- SSH support in the ReaR rescue/recovery system is now secure by default. + There are no longer private SSH keys in the recovery system by default and + a RSA key is generated from scratch when starting sshd during recovery + system startup. Accordingly it does no longer work by default to use SSH + in the recovery system via the SSH keys that exist on the original system. + To get SSH keys included in the recovery system use the SSH_FILES and + SSH_UNPROTECTED_PRIVATE_KEYS config variables (for details see + default.conf). + +- Verification of required binaries/libraries in the ReaR rescue/recovery system. By default it is now fatal when 'ldd' reports a 'not found' - library for any file in a /bin/ or /sbin/ directory in the recovery - system so that now 'rear mkrescue/mkbackup' may fail where it had - (blindly) worked before. In particular third-party backup tools sometimes - use their libraries via unexpected ways which can cause 'false alarm' by - the 'ldd' test. With the new config variable - NON_FATAL_BINARIES_WITH_MISSING_LIBRARY one can specify for which files a - 'not found' library should be considered as 'false alarm' (for details - see default.conf). - - o Improved MIGRATION_MODE autodetection when the disk layout looks + library for any file in a /bin/ or /sbin/ directory in the recovery system + so that now 'rear mkrescue/mkbackup' may fail where it had (blindly) + worked before. In particular third-party backup tools sometimes use their + libraries via unexpected ways which can cause 'false alarm' by the 'ldd' + test. With the new config variable NON_FATAL_BINARIES_WITH_MISSING_LIBRARY + one can specify for which files a 'not found' library should be considered + as 'false alarm' (for details see default.conf). + +- Improved MIGRATION_MODE autodetection when the disk layout looks ambiguous. Now 'rear recover' switches by default more often into MIGRATION_MODE where manual disk layout configuration happens via several user dialogs so that by default 'rear recover' shows more often user @@ -1720,27 +2959,27 @@ Possibly backward incompatible changes: Details (mostly in chronological order): - o Use /etc/os-release and /etc/system-release before falling back to +- Use /etc/os-release and /etc/system-release before falling back to lsb_release check in function SetOSVendorAndVersion (issues #1611, #731) - o Make BACKUP_URL=iso for mkrescue and mkbackuponly no longer fatal (issue +- Make BACKUP_URL=iso for mkrescue and mkbackuponly no longer fatal (issue #1613) - o Add ntpdate support (issue #1608) +- Add ntpdate support (issue #1608) - o Fix for XFS file system recreation code. In xfsprogs >= 4.7 log section +- Fix for XFS file system recreation code. In xfsprogs >= 4.7 log section sunit=0 is considered invalid (issue #1603) - o Changed the macro fedora_release into fedora in the rear.spec file (issue +- Changed the macro fedora_release into fedora in the rear.spec file (issue #1192 and bz1419512) - o Borg backup as back end now displays progress, when ReaR is launched in +- Borg backup as back end now displays progress, when ReaR is launched in verbose mode (issue #1594) - o Better MIGRATION_MODE autodetection (pull request #1593 related to issue +- Better MIGRATION_MODE autodetection (pull request #1593 related to issue #1271) - o With the new config variable NON_FATAL_BINARIES_WITH_MISSING_LIBRARY the +- With the new config variable NON_FATAL_BINARIES_WITH_MISSING_LIBRARY the user can specify what programs where the 'ldd' test reports 'not found' libraries are non-fatal so that those programs in the recovery system do not lead to an Error abort of "rear mkrescue/mkbackup". This is a generic @@ -1749,104 +2988,104 @@ Details (mostly in chronological order): libraries like https://github.com/rear/rear/issues/1533 (for TSM) and https://github.com/rear/rear/pull/1560 (for FDR/Upstream). - o Add a NSR_CLIENT_MODE to the backup method NSR (issue #1584) +- Add a NSR_CLIENT_MODE to the backup method NSR (issue #1584) - o Let /bin/ldd detect *.so with relative paths (issue #1560) +- Let /bin/ldd detect *.so with relative paths (issue #1560) - o Add support for Bridge Interfaces(issue #1570). Usually, virtual +- Add support for Bridge Interfaces(issue #1570). Usually, virtual interfaces are skipped, but for Bridges to work, we consider Bridges as physical interfaces, because the Bridge interface holds the IP address, not the physical interface attached to the Bridge. This patch enables those configurations: - - Bridge over simple Ethernet - - Bridge over Bond - - Bridge over Vlan interface - o Use UserInput in some more usual places to improve that 'rear recover' - can run unattended in migration mode (issues #1573, #1399) - o Error out for OUTPUT_URL=null together with OUTPUT=USB (issue #1571) + - Bridge over simple Ethernet + - Bridge over Bond + - Bridge over Vlan interface + +- Use UserInput in some more usual places to improve that 'rear recover' can + run unattended in migration mode (issues #1573, #1399) + +- Error out for OUTPUT_URL=null together with OUTPUT=USB (issue #1571) - o Added/updated paths for FDR/Upstream 4.0 because FDR/Upstream 4.0 - includes changes to some file paths (issue #1559) +- Added/updated paths for FDR/Upstream 4.0 because FDR/Upstream 4.0 includes + changes to some file paths (issue #1559) - o Fix copying kernel modules when module aliases are present (issue #1567) +- Fix copying kernel modules when module aliases are present (issue #1567) - o Netbackup agents not automatically started on RHEL 7 (issue #1523) +- Netbackup agents not automatically started on RHEL 7 (issue #1523) - o Forbid mkrescue and mkbackuponly for iso backup scheme (issues #1547, # - 1548) +- Forbid mkrescue and mkbackuponly for iso backup scheme (issues #1547, + #1548) - o Only support OpenSSH 3.1 and later for SSH setup (issue #1530) +- Only support OpenSSH 3.1 and later for SSH setup (issue #1530) - o Implemented USB_DEVICE_FILESYSTEM_LABEL (issue #1535) +- Implemented USB_DEVICE_FILESYSTEM_LABEL (issue #1535) - o Split network-functions.sh into DHCP setup and general ReaR functions +- Split network-functions.sh into DHCP setup and general ReaR functions (issue #1517) - o Avoid leaking unprotected SSH private key files onto rescue medium - (issues #1512, #1513) +- Avoid leaking unprotected SSH private key files onto rescue medium (issues + #1512, #1513) - o Improve cryptographic security and user-friendliness for LUKS volumes +- Improve cryptographic security and user-friendliness for LUKS volumes (issue #1493) - o Improve ReaR network migration (issues #1605, #1510, #1399) +- Improve ReaR network migration (issues #1605, #1510, #1399) - o Improved encrypted password detection in 500_ssh.sh (issue #1503) +- Improved encrypted password detection in 500_ssh.sh (issue #1503) - o Several code improvements in the way libraries are detected and copied +- Several code improvements in the way libraries are detected and copied (issues #1521, #1502, #1494) - o Allow btrfsmountedsubvol to be excluded via EXCLUDE_RECREATE (issue # - 1497) +- Allow btrfsmountedsubvol to be excluded via EXCLUDE_RECREATE (issue #1497) - o Avoid recreation of non-existing btrfs subvolumes (issue #1496) +- Avoid recreation of non-existing btrfs subvolumes (issue #1496) - o Add automatically some important kernel parameters to KERNEL_CMDLINE +- Add automatically some important kernel parameters to KERNEL_CMDLINE (issue #1495) - o Cleanup and simplified default input for the UserInput function (issue # - 1498) +- Cleanup and simplified default input for the UserInput function (issue + #1498) - o Avoid systemd log messages about multiple disk partitions with identical - name that happened because ReaR used a static 'rear-noname' for - originally unnamed partitions. Original gpt disk partitions may be - unnamed but parted requires a name for each gpt partition. Now ReaR uses - the basename of the partition device path (e.g. sda1, sda2) for - originally unnamed partitions (issue #1483) +- Avoid systemd log messages about multiple disk partitions with identical + name that happened because ReaR used a static 'rear-noname' for originally + unnamed partitions. Original gpt disk partitions may be unnamed but parted + requires a name for each gpt partition. Now ReaR uses the basename of the + partition device path (e.g. sda1, sda2) for originally unnamed partitions + (issue #1483) - o Enhanced how the ssh user is copied into the recovery system and improved +- Enhanced how the ssh user is copied into the recovery system and improved detection of the ssh user (issue #1489) - o Use meaningful variable for automated UserInput. Enforce calling - UserInput with a UserInput ID so that automated UserInput is always - possible for the user. Use and enforce uppercase letters in UserInput IDs - because the resulting variable names are meant as user config variables - (issue #1473) +- Use meaningful variable for automated UserInput. Enforce calling UserInput + with a UserInput ID so that automated UserInput is always possible for the + user. Use and enforce uppercase letters in UserInput IDs because the + resulting variable names are meant as user config variables (issue #1473) - o Fix UEFI tools integration (issues #1477, #1478) +- Fix UEFI tools integration (issues #1477, #1478) - o Added a new YUM backup method which will recreate the system by - installing it from scratch via installing RPM packages. The YUM backup - method uses the yum package manager in the same manner as the ZYPPER - backup method (issue #1464) +- Added a new YUM backup method which will recreate the system by installing + it from scratch via installing RPM packages. The YUM backup method uses + the yum package manager in the same manner as the ZYPPER backup method + (issue #1464) - o BACKUP_PROG_OPTIONS used to be a string variable, turn it into an array +- BACKUP_PROG_OPTIONS used to be a string variable, turn it into an array (issue #1475) - o Now CLONE_ALL_USERS_GROUPS always extends the CLONE_USERS and - CLONE_GROUPS arrays (issues #1471, #1464) +- Now CLONE_ALL_USERS_GROUPS always extends the CLONE_USERS and CLONE_GROUPS + arrays (issues #1471, #1464) - o PXE code improvements (issue #1466) +- PXE code improvements (issue #1466) - o Several improvements in the multipath code to allow full migration - (issues #1449, #1450) +- Several improvements in the multipath code to allow full migration (issues + #1449, #1450) - o Several improvements on the TSM code (issues #1539, #1461, #1452) +- Several improvements on the TSM code (issues #1539, #1461, #1452) - o Activate btrfs filesystem creation with uuid. Recent btrfs version - finally added this option (issue #1463) +- Activate btrfs filesystem creation with uuid. Recent btrfs version finally + added this option (issue #1463) - o By default a directories_permissions_owner_group file is created that +- By default a directories_permissions_owner_group file is created that saves permissions, owner, and group of basic directories plus symbolic link names and link targets of basic directories. Those basic directories are the currently used mountpoints (except some unwanted "noise" from all @@ -1856,627 +3095,629 @@ Details (mostly in chronological order): directories and symlinks that are still missing in his particular environment (issue #1459) - o Fix for cryptsetup hang on cryptsetup luksOpen ... when dmsetup is not +- Fix for cryptsetup hang on cryptsetup luksOpen ... when dmsetup is not present in recovery system (issue #1458) - o Enhance MOUNTPOINTS_TO_RESTORE into DIRECTORIES_TO_CREATE (issue #1455) +- Enhance MOUNTPOINTS_TO_RESTORE into DIRECTORIES_TO_CREATE (issue #1455) - o Several improvements for ppc64le/ppc64 arch. Systemd automatic serial +- Several improvements for ppc64le/ppc64 arch. Systemd automatic serial console detection, lilo and yaboot improvements (issue #1446, #1442) - o Introduction of UserInput in 300_map_disks.sh (issues 1399 and 1431) +- Introduction of UserInput in 300_map_disks.sh (issues 1399 and 1431) - o Adapt /etc/motd when 'rear recover' is running to avoid the additional - 'Run rear recover to restore your system' message that only makes sense - as long as 'rear recover' was not ever started (issue 1433) +- Adapt /etc/motd when 'rear recover' is running to avoid the additional + 'Run rear recover to restore your system' message that only makes sense as + long as 'rear recover' was not ever started (issue 1433) - o SLES12 with btrfs but without snapshots failed to recreate/mount btrfs FS - /subvolumes during recovery (issue #1036) +- SLES12 with btrfs but without snapshots failed to recreate/mount btrfs + FS/subvolumes during recovery (issue #1036) - o Adapt chrp-boot option when xorrisofs is used. Xorrisofs use - -chrp-boot-part option to generate PPC boot while mkisofs use -chrp - (issue #1430) +- Adapt chrp-boot option when xorrisofs is used. Xorrisofs use + -chrp-boot-part option to generate PPC boot while mkisofs use -chrp (issue + #1430) Version 2.2 (July 2017) - o Let the get_disk_size() function retry several times to be more fail-safe +- Let the get_disk_size() function retry several times to be more fail-safe when udev needs some time until device files appear. This introduces the new generic helper function retry_command() plus the new config variables REAR_SLEEP_DELAY and REAR_MAX_RETRIES. For details see default.conf and lib/layout-functions.sh (issue #1370) - o ReaR failed to continue due incorrect check of presence of USB device +- ReaR failed to continue due incorrect check of presence of USB device (REAR-000) in /proc/mounts, despite foregoing script (060_mount_NETFS_path.sh) did mounting of this device (issue #1415) - o Add missing privilege separation dir to start sshd on Debian 9 (issue # - 1381) +- Add missing privilege separation dir to start sshd on Debian 9 (issue + #1381) - o Redirect rsync verbose output to backup log (issue #1387) +- Redirect rsync verbose output to backup log (issue #1387) - o Load storage controllers in rescue system in same order as on host system +- Load storage controllers in rescue system in same order as on host system (issue #1384): - - Copy over modules from initrd on SUSE LINUX - - Prevent sorting of MODULES_LOAD array to keep the order intended - o Add /yaboot to ISO_FILE when running SUSE ppc64 (issue #1414) - o Add SSL cert directories so we can interact with Google Cloud Storage + - Copy over modules from initrd on SUSE LINUX + - Prevent sorting of MODULES_LOAD array to keep the order intended + +- Add /yaboot to ISO_FILE when running SUSE ppc64 (issue #1414) + +- Add SSL cert directories so we can interact with Google Cloud Storage (issue #1402) - o Modified the "unattended" into "automatic" with ISO_DEFAULT required for +- Modified the "unattended" into "automatic" with ISO_DEFAULT required for automated recovery tests (issue #1397) - o Use the original fds when ReaR was launched (which are now saved as fd6, +- Use the original fds when ReaR was launched (which are now saved as fd6, fd7, and fd8 for stdin, stdout, and stderr respectively) for actually - intended user input and user output. To keep backward compatible - behaviour all old deprecated usage of '>&8' is converted into '>/dev/ - null' but /dev/null usage in general should be cleaned up later (issues # - 887, #1395) + intended user input and user output. To keep backward compatible behaviour + all old deprecated usage of '>&8' is converted into '>/dev/null' but + /dev/null usage in general should be cleaned up later (issues #887, #1395) - o Added new generic UserInput and UserOutput plus LogUserOutput functions +- Added new generic UserInput and UserOutput plus LogUserOutput functions that are intended to replace current user input functionality that calls select or read directly. For the next ReaR version 2.3 it is planned to - also redirect stdout into the log file in addition to stderr (issues # - 885, #1366, #1398, #1399) + also redirect stdout into the log file in addition to stderr (issues #885, + #1366, #1398, #1399) - o The 'make rpm' now relies on 'make srpm' which creates the src.rpm - package first. This src.rpm package can then be easily copied to another - computer to rebuild a rpm package from it without needed the sources - itself (or git checkout) (issue #1389) +- The 'make rpm' now relies on 'make srpm' which creates the src.rpm package + first. This src.rpm package can then be easily copied to another computer + to rebuild a rpm package from it without needed the sources itself (or git + checkout) (issue #1389) - o Replaced some perl regexp with grep native extended regexp (issue #1376) +- Replaced some perl regexp with grep native extended regexp (issue #1376) - o Introducing SECURE_BOOT_BOOTLOADER variable in default.conf. This - variable should enable users booting with Secure Boot, to use whatever - custom signed boot loader they like, and removes hard coded entry - 'shim.efi' from the ReaR code (issue #1374) +- Introducing SECURE_BOOT_BOOTLOADER variable in default.conf. This variable + should enable users booting with Secure Boot, to use whatever custom + signed boot loader they like, and removes hard coded entry 'shim.efi' from + the ReaR code (issue #1374) - o Enhanced and cleaned up making ISO on POWER (ppc64/ppc64le). Now the +- Enhanced and cleaned up making ISO on POWER (ppc64/ppc64le). Now the backup can be stored in the ISO (via BACKUP_URL=iso...) and even multiple ISOs work on POWER now (issues #697, #1383) - o Create multipath.conf only during migration (from non-multipath to +- Create multipath.conf only during migration (from non-multipath to multipath), and always copy /etc/multipath/bindings to the TARGET_FS_ROOT (issues #1382, #1393) Version 2.1 (June 2017) - o Support for Grub2 installation with software RAID1 on Linux on POWER +- Support for Grub2 installation with software RAID1 on Linux on POWER (ppc64/ppc64le) (issue #1369) - o REBUILD_INITRAMFS variable was introduced. The new default.conf setting +- REBUILD_INITRAMFS variable was introduced. The new default.conf setting REBUILD_INITRAMFS="yes" rebuilds the initramfs/initrd during "rear recover" to be more on the safe side. With REBUILD_INITRAMFS="" the old behaviour can still be specified (issue #1321) - o ISO_RECOVER_MODE=unattended mode (issue #1351) - required for automated +- ISO_RECOVER_MODE=unattended mode (issue #1351) - required for automated ReaR testing with OUTPUT=ISO - o MODULES variable supports now special values like 'all_modules', +- MODULES variable supports now special values like 'all_modules', 'loaded_modules', 'no_modules' (issues #1202, #1355) - o Include systemd/network to preserve "Predictable Network Interface Names" +- Include systemd/network to preserve "Predictable Network Interface Names" (issue #1349) - o Various improvements regarding multipath (issues #1190, #1309, #1310, # - 1311, #1314, #1315, #1324, #1325, #1328, #1329, #1344, #1346) +- Various improvements regarding multipath (issues #1190, #1309, #1310, + #1311, #1314, #1315, #1324, #1325, #1328, #1329, #1344, #1346) - o Show OUTPUT variables in rear dump (issue #1337) +- Show OUTPUT variables in rear dump (issue #1337) - o Added support for "grub PXE style" via PXE_CONFIG_GRUB_STYLE and +- Added support for "grub PXE style" via PXE_CONFIG_GRUB_STYLE and PXE_TFTP_IP on non x86 platform (issue #1339) - o Try 'wipefs -force' and use 'dd' as fallback to better clean up disk +- Try 'wipefs -force' and use 'dd' as fallback to better clean up disk partitions (issue #1327) - o Reorganized "finalize" scripts ordering and cleanup of the PPC bootloader +- Reorganized "finalize" scripts ordering and cleanup of the PPC bootloader installation (issue #1323) - o Avoid long default wait in 'dig' when DNS servers are not set (issue # - 1319) +- Avoid long default wait in 'dig' when DNS servers are not set (issue + #1319) - o Fail-safe calculations in partitioning code (issues #1269, #1307) +- Fail-safe calculations in partitioning code (issues #1269, #1307) - o Improved support on ppc/ppc64/ppc64le architectures (issues #1178, #1311, +- Improved support on ppc/ppc64/ppc64le architectures (issues #1178, #1311, #1313, #1322) - o Define hostname in both /etc/HOSTNAME and /etc/hostname in rescue image +- Define hostname in both /etc/HOSTNAME and /etc/hostname in rescue image (for Arch) (issue #1316) - o Rename network interface when MAC not present in udev (issue #1312) +- Rename network interface when MAC not present in udev (issue #1312) - o Added support for 'nano' editor (in addition to 'vi') (issues #1298, # - 1306) +- Added support for 'nano' editor (in addition to 'vi') (issues #1298, + #1306) - o mmcblk disk types are now supported (issues #1301, #1302) +- mmcblk disk types are now supported (issues #1301, #1302) - o NETFS_RESTORE_CAPABILITIES variable was introduced to restore file +- NETFS_RESTORE_CAPABILITIES variable was introduced to restore file capabilities in a proper way (issue #1283) - o Added required libs and files for 'curl' with HTTPs by default (issues # - 1267, #1279) +- Added required libs and files for 'curl' with HTTPs by default (issues + #1267, #1279) - o More precise XFS file system creation during rear recover (issues #1208, +- More precise XFS file system creation during rear recover (issues #1208, #1213, #1276) - o DRLM management and security improvements (issue #1252) +- DRLM management and security improvements (issue #1252) - o Improved BOOTLOADER support (issue #1242) +- Improved BOOTLOADER support (issue #1242) - o DRLM support for multiple backups via multiple config files (issue #1229) +- DRLM support for multiple backups via multiple config files (issue #1229) - o FIRMWARE_FILES support to exclude firmware files in rescue image to - reduce the size of image (issue #1216) +- FIRMWARE_FILES support to exclude firmware files in rescue image to reduce + the size of image (issue #1216) - o Enable SELinux in the rescue image for tar internal backup method if +- Enable SELinux in the rescue image for tar internal backup method if BACKUP_SELINUX_DISABLE=0 (issue #1215) - o BOOT_OVER_SAN is now fully supported (issues #1190, #1309, #1314, #1315, +- BOOT_OVER_SAN is now fully supported (issues #1190, #1309, #1314, #1315, #1325, #1329, #1344) - o NVME disks are now fully supported (issue #1191) +- NVME disks are now fully supported (issue #1191) - o Some initial basic support for new backup type ZYPPER was added (issues # - 1085, #1209) +- Some initial basic support for new backup type ZYPPER was added (issues + #1085, #1209) - o Finding UEFI boot loaders on non standard places (issues #1204, #1225, # - 1293) +- Finding UEFI boot loaders on non standard places (issues #1204, #1225, + #1293) - o The USB UEFI partition size USB_UEFI_PART_SIZE for kernel image has been +- The USB UEFI partition size USB_UEFI_PART_SIZE for kernel image has been increased from 100 to 200 MB (issue #1205) - o REAR_INITRD_COMPRESSION variable was introduced to specify initrd +- REAR_INITRD_COMPRESSION variable was introduced to specify initrd compression (e.g. 'lzma' for PPC64) (issues #1142, #1218, #1290) - o New backup type BLOCKCLONE was added to backup non-Linux partitions (e.g. - Windows NTFS partitions) (issues #1078, #1162, #1172, #1180) +- New backup type BLOCKCLONE was added to backup non-Linux partitions + (e.g. Windows NTFS partitions) (issues #1078, #1162, #1172, #1180) - o Bareos 16.2 is now supported (issue #1169) +- Bareos 16.2 is now supported (issue #1169) - o New USB_PARTITION_ALIGN_BLOCK_SIZE and USB_DEVICE_FILESYSTEM_PARAMS +- New USB_PARTITION_ALIGN_BLOCK_SIZE and USB_DEVICE_FILESYSTEM_PARAMS variables were added (issue #1217) - o Improved the USB backup selection menu during the recovery via USB (issue +- Improved the USB backup selection menu during the recovery via USB (issue #1166) - o USB_SUFFIX variable was introduced to align backup on USB with backup on +- USB_SUFFIX variable was introduced to align backup on USB with backup on NFS (issues #1164, #1160, #1145) - o Forbid incremental backup to work on BACKUP_URL=usb:// (issue #1146) +- Forbid incremental backup to work on BACKUP_URL=usb:// (issue #1146) - o The USB_DEVICE_PARTED_LABEL=gpt setting is now honered while formatting +- The USB_DEVICE_PARTED_LABEL=gpt setting is now honered while formatting the USB disk (issue #1153) Version 2.00 (January 2017) (Important Note) ReaR 2.00 introduced the 3-digits scripts instead of the -2-digits script. This means all scripts must begin with 3 digits, e.g. -010-my-script.sh instead of 10-my-script. Therefore, if you wrote your own -scripts make sure to renumber these. You could also use the make validate to -check this. +2-digits script. This means all scripts must begin with 3 digits, +e.g. 010-my-script.sh instead of 10-my-script. Therefore, if you wrote your +own scripts make sure to renumber these. You could also use the make validate +to check this. - o Bareos support: add missing directory /var/run/bareos in recovery system +- Bareos support: add missing directory /var/run/bareos in recovery system (issue #1148) - o Forbid BACKUP_URL=usb for BACKUP_TYPE=incremental/differential (issues # - 1141 and #1145) +- Forbid BACKUP_URL=usb for BACKUP_TYPE=incremental/differential (issues + #1141 and #1145) - o Improved and added new example configurations (issue #1068, #1058) +- Improved and added new example configurations (issue #1068, #1058) - o Modified/Improved the exit code messages of ReaR (issues #1089, #1133) +- Modified/Improved the exit code messages of ReaR (issues #1089, #1133) - o Fix documentation regarding OUTPUT_URL=null (issues #734, #1130) +- Fix documentation regarding OUTPUT_URL=null (issues #734, #1130) - o Better and fail safe progress messages while tar backup restore (issue # - 1116) +- Better and fail safe progress messages while tar backup restore (issue + #1116) - o Implement simulation mode with simulation with the workflows validate and +- Implement simulation mode with simulation with the workflows validate and shell (issue #1098) - o Update 11-multiple-backups.adoc : Multiple backups are in general not +- Update 11-multiple-backups.adoc: Multiple backups are in general not supported for BACKUP_TYPE=incremental or BACKUP_TYPE=differential (issues #1074 and #1123) - o Using RUNTIME_LOGFILE in all scripts as needed (issue #1119) +- Using RUNTIME_LOGFILE in all scripts as needed (issue #1119) - o New Backup method was added - BORG (issues #1030, #1037, #1046, #1048, # - 1118) +- New Backup method was added - BORG (issues #1030, #1037, #1046, #1048, + #1118) - o Multiple backups are now possible (issues #1088, #1102, #1096) - see the +- Multiple backups are now possible (issues #1088, #1102, #1096) - see the documentation page (New) - o Support partitioning and formatting huge USB devices (issue #1105) +- Support partitioning and formatting huge USB devices (issue #1105) - o Skip remount async when systemd is used (issue #1097) +- Skip remount async when systemd is used (issue #1097) - o Fixed and enhanced code for multiple ISOs (issue #1081) +- Fixed and enhanced code for multiple ISOs (issue #1081) - o BACKUP_TYPE=incremental and BACKUP_TYPE=differential were updated (issues +- BACKUP_TYPE=incremental and BACKUP_TYPE=differential were updated (issues #974, #1069) - o Added support for setting a UUID on XFS with enabled CRC (RHEL 7) (issue +- Added support for setting a UUID on XFS with enabled CRC (RHEL 7) (issue #1065) - o Fix for ISO not bootable for SLES11 ppc64 with root LVM (issue #1061) +- Fix for ISO not bootable for SLES11 ppc64 with root LVM (issue #1061) - o PXE booting enhancement with new style of uploading the boot files (issue +- PXE booting enhancement with new style of uploading the boot files (issue #193) - o Renumbering the ReaR scripts from 2-digits to 3-digits (issue #1051) +- Renumbering the ReaR scripts from 2-digits to 3-digits (issue #1051) - o Improved boot loader detection (issue #1038) +- Improved boot loader detection (issue #1038) Version 1.19.0 (October 2016) - o Save bootloader info from POWER architecture and rebuild initrd after +- Save bootloader info from POWER architecture and rebuild initrd after migration (issues #1029, #1031) - o Improved documentation and man page in general (issues #918, #930, #1004, +- Improved documentation and man page in general (issues #918, #930, #1004, #1007, #1008) - o New SLE12-SP2-btrfs-example.conf file because since SLES12-SP2 btrfs - quota setup for snapper via "snapper setup-quota" is needed (issue #999) +- New SLE12-SP2-btrfs-example.conf file because since SLES12-SP2 btrfs quota + setup for snapper via "snapper setup-quota" is needed (issue #999) - o Simplified reboot halt poweroff and shutdown in the rescue/recovery - system in case of systemd (issue #953) +- Simplified reboot halt poweroff and shutdown in the rescue/recovery system + in case of systemd (issue #953) - o If TSM parameters contain a dot, the dot is replaced by an underscore in +- If TSM parameters contain a dot, the dot is replaced by an underscore in the TSM_SYS variable names (issues #985 and #986) - o Check if /dev/disk/by-label/RELAXRECOVER exist (issues #979 and #326) +- Check if /dev/disk/by-label/RELAXRECOVER exist (issues #979 and #326) - o Added PRE_BACKUP_SCRIPT and POST_BACKUP_SCRIPT to be able to do custom +- Added PRE_BACKUP_SCRIPT and POST_BACKUP_SCRIPT to be able to do custom tasks in the mkbackup/mkbackuponly workflows (issue #977) - o Make TMPDIR work in compliance with Unix standards (issue #969) +- Make TMPDIR work in compliance with Unix standards (issue #969) - o USE_STATIC_NETWORKING now really overrides USE_DHCLIENT (issue #964) +- USE_STATIC_NETWORKING now really overrides USE_DHCLIENT (issue #964) - o Make it safe against wrong btrfs subvolumes on SLES12 (issues #963, #966) +- Make it safe against wrong btrfs subvolumes on SLES12 (issues #963, #966) - o Encrypted incremental backup cannot read the tar label (issue #952) +- Encrypted incremental backup cannot read the tar label (issue #952) - o Introduction of the NETWORKING_PREPARATION_COMMANDS variable to prepare +- Introduction of the NETWORKING_PREPARATION_COMMANDS variable to prepare network setup in the rescue/recovery system (issue #960) - o After migration fs_uuid for root partition was not changed in ELILO - config file /etc/elilo.conf (issue #956) +- After migration fs_uuid for root partition was not changed in ELILO config + file /etc/elilo.conf (issue #956) - o Clarified rear man page and default.conf file around BACKUP_URL=rsync: +- Clarified rear man page and default.conf file around BACKUP_URL=rsync: (issues #930 and #918) - o Make "rear recover" work with default btrfs on SLES12-SP2 (issue #944) +- Make "rear recover" work with default btrfs on SLES12-SP2 (issue #944) - o Dropped GRUB_SUPERUSER and GRUB_RESCUE_PASSWORD to avoid that GRUB_RESCUE +- Dropped GRUB_SUPERUSER and GRUB_RESCUE_PASSWORD to avoid that GRUB_RESCUE could change the behaviour of the GRUB2 bootloader in the currently running system in unexpected ways. With the new optional GRUB_RESCUE_USER setting GRUB_RESCUE works in compliance with the existing GRUB2 configuration (issues #938, #942) - o Bail out if not enough disk space for GRUB and GRUB2 rescue image (issue +- Bail out if not enough disk space for GRUB and GRUB2 rescue image (issue #913) - o Use BACKUP_PROG_COMPRESS_OPTIONS as an array so that one can use it to +- Use BACKUP_PROG_COMPRESS_OPTIONS as an array so that one can use it to provide more complex values (issue #904) - o Add /usr/lib/syslinux/bios to the search path for mbr.bin (issue #908) +- Add /usr/lib/syslinux/bios to the search path for mbr.bin (issue #908) - o Always load modules in /etc/modules (issue #905) +- Always load modules in /etc/modules (issue #905) - o Ask user for EFI partition size on USB disk (issue #849) +- Ask user for EFI partition size on USB disk (issue #849) - o Insure /etc/rear/mappings directory exists before doing a recovery (issue +- Insure /etc/rear/mappings directory exists before doing a recovery (issue #861) - o First steps for rescue/recovery system update support via +- First steps for rescue/recovery system update support via RECOVERY_UPDATE_URL (issue #841) - o NFS mount points are not recreated after a recover (issue #818) +- NFS mount points are not recreated after a recover (issue #818) - o Correcting ReaR return code handling in auto recover mode (issue #893) +- Correcting ReaR return code handling in auto recover mode (issue #893) - o Added NFSv4 support for security 'sys' only so far (issue #754) +- Added NFSv4 support for security 'sys' only so far (issue #754) - o Changed the usage of 'rpcinfo -p' a bit to have the same outcome of +- Changed the usage of 'rpcinfo -p' a bit to have the same outcome of different Linux flavours (issue #889) - o RSYNC: /boot/efi needs -relative rsync option (issue #871) +- RSYNC: /boot/efi needs -relative rsync option (issue #871) - o New variables added for Bareos: BAREOS_RESTORE_JOB and BAREOS_FILESET +- New variables added for Bareos: BAREOS_RESTORE_JOB and BAREOS_FILESET - o Multipath partition not found in rhel7.2 (issue #875) +- Multipath partition not found in rhel7.2 (issue #875) - o Adding support for ppc64le PowerNV (non-virtualized aka Bare-Metal) - (issue #863) +- Adding support for ppc64le PowerNV (non-virtualized aka Bare-Metal) (issue + #863) - o First steps to support new ftpfs BACKUP_URL scheme (issue #845) +- First steps to support new ftpfs BACKUP_URL scheme (issue #845) - o Clean up 'url_host()' (issue #856) +- Clean up 'url_host()' (issue #856) - o Fix that libaio (needed for multipath) could be missing in rescue/ - recovery system because libaio can be located in different directories - (issue #852) +- Fix that libaio (needed for multipath) could be missing in rescue/recovery + system because libaio can be located in different directories (issue #852) - o Improved the Relax-and-Recover menu for GRUB2 (issues #844, #849, #850) +- Improved the Relax-and-Recover menu for GRUB2 (issues #844, #849, #850) - o Check for valid BACKUP_URL schemes (issue #842) +- Check for valid BACKUP_URL schemes (issue #842) - o USB UEFI boot support (issue #831) +- USB UEFI boot support (issue #831) - o Mitigate the problem that btrfs subvolums are not restored by default via +- Mitigate the problem that btrfs subvolums are not restored by default via TSM (issue #833) - o Determine EFI virtual disk size automatically (issue #816) +- Determine EFI virtual disk size automatically (issue #816) - o ebiso image size is too small if BACKUP=TSM (issue #811) +- ebiso image size is too small if BACKUP=TSM (issue #811) - o Improving the logics around ebiso usage in UEFI mode (issue #801) +- Improving the logics around ebiso usage in UEFI mode (issue #801) - o Fix for wrong UUID in initrd for bootfs (issues #649 and #851) +- Fix for wrong UUID in initrd for bootfs (issues #649 and #851) Version 1.18.0 (March 2016) - o Support was added for NVME SSD type of disk devices (issue #787) +- Support was added for NVME SSD type of disk devices (issue #787) - o For LUKS added the password libraries (issue #679) +- For LUKS added the password libraries (issue #679) - o Script 99_sysreqs.sh was added to save the minimal system requirements +- Script 99_sysreqs.sh was added to save the minimal system requirements necessary for cloning a system in a remote DRP data center (issue #798) - o New 99_move_away_restored_files.sh to remove restored files after recover +- New 99_move_away_restored_files.sh to remove restored files after recover (issue #799) New array was introduced to make this - BACKUP_RESTORE_MOVE_AWAY_FILES=() - o Improved 40-start-udev-or-load-modules.sh script for better udevd - handling (issue #766) +- Improved 40-start-udev-or-load-modules.sh script for better udevd handling + (issue #766) - o Run ldconfig -X before dhclient gets started at boot time (issue #772) +- Run ldconfig -X before dhclient gets started at boot time (issue #772) - o Remove the "-c3" option fron rsyslogd start-up (issue #773) +- Remove the "-c3" option fron rsyslogd start-up (issue #773) - o Add example configuration for NetBackup Master/Media server +- Add example configuration for NetBackup Master/Media server - o Added backup capabilities; getcap and setcap are used to backup and +- Added backup capabilities; getcap and setcap are used to backup and restore (issue #771) - o Correct bash syntax so ReaR is compatible with bash v3 and v4 (issue # - 765, #767) +- Correct bash syntax so ReaR is compatible with bash v3 and v4 (issue #765, + #767) - o Added support for new backup method Novastor NovaBACKUP DC (BACKUP=NBKDC) +- Added support for new backup method Novastor NovaBACKUP DC (BACKUP=NBKDC) (issue #669) - o Code was improved to have network teaming support (issue #655) +- Code was improved to have network teaming support (issue #655) - o Example configuration to put backup and rescue image on same ISO image, +- Example configuration to put backup and rescue image on same ISO image, eg. DVD (issue #430) - o Improved the ReaR documentation +- Improved the ReaR documentation - o remove the noatime mount option for cifs mount (issue #752) +- remove the noatime mount option for cifs mount (issue #752) - o Replace option 'grep -P' to 'grep -E' due to SELinux errors (issues #565, +- Replace option 'grep -P' to 'grep -E' due to SELinux errors (issues #565, #737) - o Hidding the encryption key while doing the restore in the rear.log (issue +- Hidding the encryption key while doing the restore in the rear.log (issue #749) - o is_true function was to uniform the different ways of enable/disble +- is_true function was to uniform the different ways of enable/disble variable settings (issue #625) - o Added and use sysctl.conf; rescue mode should honor these settings (issue +- Added and use sysctl.conf; rescue mode should honor these settings (issue #748) - o The BACKUP_PROG_COMPRESS variable was not used during incremental backup +- The BACKUP_PROG_COMPRESS variable was not used during incremental backup (issue #743) - o prevent any other workflow in ReaR rescue mode then recover (issue #719) +- prevent any other workflow in ReaR rescue mode then recover (issue #719) - o Exclude Oracle ASM device directory from Rescue System (issue #721) +- Exclude Oracle ASM device directory from Rescue System (issue #721) - o SaveBashFlagsAndOptions and RestoreBashFlagsAndOptions in usr/share/rear/ - lib/framework-functions.sh were added (issue #700) +- SaveBashFlagsAndOptions and RestoreBashFlagsAndOptions in + usr/share/rear/lib/framework-functions.sh were added (issue #700) - o /mnt/local became a global variable TARGET_FS_ROOT (issue #708) +- /mnt/local became a global variable TARGET_FS_ROOT (issue #708) - o Copy rear.log from recovery into /var/log/rear/recovery/ directory after - a 'rear recover' (issue #706) +- Copy rear.log from recovery into /var/log/rear/recovery/ directory after a + 'rear recover' (issue #706) - o wipefs will be used when available (issue #649) +- wipefs will be used when available (issue #649) - o SAN related improvements with btrfs (issue #695) +- SAN related improvements with btrfs (issue #695) - o Support for shim.efi (UEFI booting) added (issue #702) +- Support for shim.efi (UEFI booting) added (issue #702) - o Added support for elilo (used by SLES 11/12) (issue #583, #691, #692, # - 693) +- Added support for elilo (used by SLES 11/12) (issue #583, #691, #692, + #693) - o Added the -debugscripts command line option (help-workflow) (issue #688) +- Added the -debugscripts command line option (help-workflow) (issue #688) - o Removed dosfslabel as required program for vfat UEFI boot partition - (issue #694) +- Removed dosfslabel as required program for vfat UEFI boot partition (issue + #694) - o Bareos team added BAREOS_FILESET and ISO_DEFAULT to default.conf (for +- Bareos team added BAREOS_FILESET and ISO_DEFAULT to default.conf (for automated DR tests executed by Bareos team; issues #686, #719) - o Fix getty/agetty with upstart (issue #685) +- Fix getty/agetty with upstart (issue #685) - o New SLE11-SLE12-SAP-HANA-UEFI-example.conf (issue #683) +- New SLE11-SLE12-SAP-HANA-UEFI-example.conf (issue #683) - o usr/share/rear/conf/examples/SLE12-SP1-btrfs-example.conf added as an +- usr/share/rear/conf/examples/SLE12-SP1-btrfs-example.conf added as an example configuration file - o Added support for the SUSE specific gpt_sync_mbr partitioning scheme +- Added support for the SUSE specific gpt_sync_mbr partitioning scheme (issue #544) - o Improved btrfs snapshot support with SLES 12 (issue #556) +- Improved btrfs snapshot support with SLES 12 (issue #556) - o Unload scsi_debug driver in recovery mode for RHEL 7.1 (issue #626) +- Unload scsi_debug driver in recovery mode for RHEL 7.1 (issue #626) - o Saved the current mount points and permissions; in order to improve and +- Saved the current mount points and permissions; in order to improve and avoid missing mount points after recovery (issue #619) - o NSR servername not defined causing ReaR to hang (issue #637) +- NSR servername not defined causing ReaR to hang (issue #637) - o Removed mingetty as a required package (issue #661) +- Removed mingetty as a required package (issue #661) - o Adding -scrollprompt=no to dsmc query in script verify/TSM/default/ - 40_verify_tsm.sh (issue #667) +- Adding -scrollprompt=no to dsmc query in script + verify/TSM/default/40_verify_tsm.sh (issue #667) - o Fixed a bug around USB_DEVICE and OUTPUT_URL mis-match (issue #579) +- Fixed a bug around USB_DEVICE and OUTPUT_URL mis-match (issue #579) - o grub support for ppc64 (issue #673) +- grub support for ppc64 (issue #673) - o grub2 supported was added for ppc64 (issue #672) +- grub2 supported was added for ppc64 (issue #672) - o ppc64le support was added into the rear.spec (issue #665) +- ppc64le support was added into the rear.spec (issue #665) - o Network code partially rewritten to improve teaming (issue #662) +- Network code partially rewritten to improve teaming (issue #662) - o Changed default value of USE_CFG2HTML from 1 to empty (means do not run +- Changed default value of USE_CFG2HTML from 1 to empty (means do not run cfg2html by default) (issue #643) - o Move the 50_selinux_autorelabel.sh script to the default location so it +- Move the 50_selinux_autorelabel.sh script to the default location so it gets picked up by all backup methods. This was required for RHEL 7 (issue #650) - o Check via NSR if the ISO image is not obsolete (issue #653) +- Check via NSR if the ISO image is not obsolete (issue #653) - o Added ebiso support within ReaR (required for UEFI booting with SLES 11 & +- Added ebiso support within ReaR (required for UEFI booting with SLES 11 & 12 (issue #657) - o FDR/Upstream (BACKUP=FDRUPSTREAM) (New) (issue #659) +- FDR/Upstream (BACKUP=FDRUPSTREAM) (New) (issue #659) Version 1.17.2 (August 2015) - o Several fixed required to the Debian packaging rules needed so it builds +- Several fixed required to the Debian packaging rules needed so it builds correctly on OBS - o Fixed the NTP startup script (issue #641) +- Fixed the NTP startup script (issue #641) - o Fixed the vfat label issue (issue #647) +- Fixed the vfat label issue (issue #647) - o Improved the DUPLICITY method with finding all required libraries and +- Improved the DUPLICITY method with finding all required libraries and python scripts - o Added the /run directory to the list of recreating missing directories +- Added the /run directory to the list of recreating missing directories (issue #619) - o Fix issue with USB disk and rsync as internal backup program (issue #645) +- Fix issue with USB disk and rsync as internal backup program (issue #645) - o Fix rsync restore: -anchored invalid rsync option (issue #642) +- Fix rsync restore: -anchored invalid rsync option (issue #642) - o A new variable was introduced NSR_POOLNAME (issue #640) +- A new variable was introduced NSR_POOLNAME (issue #640) - o Replaced almost all temporary files from /tmp/ to $TMP_DIR/ (issue #607) +- Replaced almost all temporary files from /tmp/ to $TMP_DIR/ (issue #607) Related to security recommendations for Fedora and RHEL: - - https://bugzilla.redhat.com/show_bug.cgi?id=1239009 (f22) - - https://bugzilla.redhat.com/show_bug.cgi?id=1238843 (rhel 7.2) - o Move nfs-client from depends to recommends in Debian control file (issue + + - https://bugzilla.redhat.com/show_bug.cgi?id=1239009 (f22) + - https://bugzilla.redhat.com/show_bug.cgi?id=1238843 (rhel 7.2) + +- Move nfs-client from depends to recommends in Debian control file (issue #633) - o In packaging/rpm/rear.spec replaced "BuildArch: noarch" with +- In packaging/rpm/rear.spec replaced "BuildArch: noarch" with "ExclusiveArch: %ix86 x86_64 ppc ppc64" that should tell the user that ReaR is known to work only on %ix86 x86_64 ppc ppc64 and removed "Requires: yaboot" for ppc ppc64 because that is the default installed bootloader on ppc ppc64 (issues #629 and #631) - o Support the Oracle Linux 6 ksplice module (issue #605) +- Support the Oracle Linux 6 ksplice module (issue #605) - o In script 27_hpraid_layout.sh added the missing - HPSSACLI_BIN_INSTALLATION_DIR variable to the COPY_AS_IS array (issue # - 630) +- In script 27_hpraid_layout.sh added the missing + HPSSACLI_BIN_INSTALLATION_DIR variable to the COPY_AS_IS array (issue + #630) - o Modified the packaging Makefile and rules for debian to fix the failing +- Modified the packaging Makefile and rules for debian to fix the failing OBS Debian builds (issue #604) - o Syslinux version > 5.00 is now supported (ISO and USB output) - works on +- Syslinux version > 5.00 is now supported (ISO and USB output) - works on Debian 8, Ubuntu 15.04 (issue #624) - o Bail out when syslinux/modules are not found in lib/ - bootloader-functions.sh (issues #467 and #596) You could also define a - variable SYSLINUX_MODULES_DIR if ReaR cannot find it automatically - (should not be necessary) +- Bail out when syslinux/modules are not found in + lib/bootloader-functions.sh (issues #467 and #596) You could also define a + variable SYSLINUX_MODULES_DIR if ReaR cannot find it automatically (should + not be necessary) - o Support was added for SLES11 on PPC64 hardware (issues #616 and #628) +- Support was added for SLES11 on PPC64 hardware (issues #616 and #628) - o Support was added for new hardware - PPC64LE - RHEL and Ubuntu (issue # - 627) +- Support was added for new hardware - PPC64LE - RHEL and Ubuntu (issue + #627) - o FIX the hashed password (SSH_ROOT_PASSWORD variable) and added a missing +- FIX the hashed password (SSH_ROOT_PASSWORD variable) and added a missing library libfreeblpriv3 (issue #560) - o Insert a 3 seconds sleep after a volume group restauration to give udevd +- Insert a 3 seconds sleep after a volume group restauration to give udevd or systemd-udevd time to create needed devices (issue #608 and #617) - o Variable MANUAL_INCLUDE=YES has been introduced to work with array +- Variable MANUAL_INCLUDE=YES has been introduced to work with array BACKUP_PROG_INCLUDE (issue #597) - o Add new variable NSR_DEFAULT_POOL_NAME (defaulting to Default) to use a +- Add new variable NSR_DEFAULT_POOL_NAME (defaulting to Default) to use a different default pool name. Renamed the RETENTION_TIME variable to NSR_RETENTION_TIME (issue #640) - o ReaR website shows the user guide which is part of the ReaR software +- ReaR website shows the user guide which is part of the ReaR software (linked to GitHub) - o new document 10-integrating-external-backup.adoc which explains the steps +- new document 10-integrating-external-backup.adoc which explains the steps to take for a new backup integration into ReaR - o All AsciiDoc documentation changed extention from .txt to .adoc +- All AsciiDoc documentation changed extention from .txt to .adoc Version 1.17.1 (June 2015) - o Removed the plain password in the logs (and output) coming from +- Removed the plain password in the logs (and output) coming from BACKUP_PROG_CRYPT_KEY to avoid crib (issue #568) - o Mount vfat file system without special mount options seems to work much +- Mount vfat file system without special mount options seems to work much better then with these options in recovery mode, therefore, we do not use these anymore (especially for /boot/efi) (issue #576) - o Elilo support has been added for SLES (not fully tested yet) - issue #583 +- Elilo support has been added for SLES (not fully tested yet) - issue #583 - o Grub2 rescue menu has been added (enable this feature with GRUB_RESCUE=y) - - issue #589 +- Grub2 rescue menu has been added (enable this feature with + GRUB_RESCUE=y) - issue #589 - o splitted script 31_include_uefi_tools.sh in two pieces: +- splitted script 31_include_uefi_tools.sh in two pieces: 31_include_uefi_tools.sh: to grab the UEFI tools (as long as /boot/efi is - mounted), and 32_include_uefi_env.sh: to dig deeper into the - configuration when UEFI is active (related to issue #214) + mounted), and 32_include_uefi_env.sh: to dig deeper into the configuration + when UEFI is active (related to issue #214) This is necessary to have to UEFI tools on SLES 11/12 where we cannot - create an UEFI bootable ISO image. We must boot in BIOS mode, and need - the UEFI tools to make the system bootable over UEFI. + create an UEFI bootable ISO image. We must boot in BIOS mode, and need the + UEFI tools to make the system bootable over UEFI. - o It is now possible to format an USB disk with a vfat and ext3 partition +- It is now possible to format an USB disk with a vfat and ext3 partition (for UEFI booting) - issue #593 rear -v format -- --efi /dev/ - o Simplified the code for ext* fs and added StopIfError calls to prevent +- Simplified the code for ext* fs and added StopIfError calls to prevent divide by zero during recovery (issue #598) - o Syslinux version >6 requires some special handling due to splitting up - the package (Ubuntu 15.04) (issue #584) +- Syslinux version >6 requires some special handling due to splitting up the + package (Ubuntu 15.04) (issue #584) - o Debian 8 support added with ISO booting with latest syslinux release as +- Debian 8 support added with ISO booting with latest syslinux release as well (issues #596, #599 and #600) - o Changed the behavior of SSH_ROOT_PASSWORD - now saved as MD5 hash +- Changed the behavior of SSH_ROOT_PASSWORD - now saved as MD5 hash password, but backwards compatibility is still respected (issue #560) - o For EMC NetWorker server/client we added some exclude items to +- For EMC NetWorker server/client we added some exclude items to COPY_AS_IS_EXCLUDE_NSR (issue #571) - o Removed the Warning message from main usr/sbin/rear script as it was +- Removed the Warning message from main usr/sbin/rear script as it was misleading (issues #563 and #564) - o output/ISO/Linux-i386/80_create_isofs.sh: make sure ISO_FILES[@] are +- output/ISO/Linux-i386/80_create_isofs.sh: make sure ISO_FILES[@] are copied to isofs directory (issue #569) System and Software Requirements Relax-and-Recover works on GNU/Linux kernel with version 2.6 and higher. For -lower kernel versions Relax-and-Recover cannot be used, and for these -systems, mkcdrec is still a good alternative. +lower kernel versions Relax-and-Recover cannot be used, and for these systems, +mkcdrec is still a good alternative. As Relax-and-Recover has been solely written in the bash language we need the bash shell which is standard available on all GNU/Linux based systems. The @@ -2490,9 +3731,9 @@ after every successful DR test please and mail us the results. Choosing the best compression algorithm -The default backup program with Relax-and-Recover is (BACKUP_PROG=tar) GNU -tar and the default compression used with tar is gzip. However, is using gzip -the best choice? We have done some tests and published the results. See +The default backup program with Relax-and-Recover is (BACKUP_PROG=tar) GNU tar +and the default compression used with tar is gzip. However, is using gzip the +best choice? We have done some tests and published the results. See Relax-and-Recover compression tests Support @@ -2507,58 +3748,57 @@ as a service (not free of charge). Supported and Unsupported Operating Systems -We try to keep our wiki page Test Matrix rear 2.6 up-to-date with feedback we +We try to keep our wiki page Test Matrix ReaR 2.7 up-to-date with feedback we receive from the community. -ReaR-2.6 is supported on the following Linux based operating systems: +ReaR 2.7 is supported on the following Linux based operating systems: - o Fedora 29, 30, 31, and 32 - o RHEL 6, 7, and 8 - o CentOS 6, 7, and 8 - o Scientific Linux 6 and 7 - o SLES 12 and 15 - o openSUSE Leap 15.x - o Debian 8, and 9 - o Ubuntu 16, 17, and 18 +- Fedora 29, 30, 31, and 32 +- RHEL 6, 7, and 8 +- CentOS 6, 7, and 8 +- Scientific Linux 6 and 7 +- SLES 12 and 15 +- openSUSE Leap 15.x +- Debian 8, and 9 +- Ubuntu 16, 17, and 18 -ReaR-2.6 dropped official support for the following Linux based operating +ReaR 2.7 dropped official support for the following Linux based operating systems: - o Fedora < 29 - o RHEL < 6 - o CentOS < 6 - o Scientific Linux < 6 - o SLES < 12 - o openSUSE Leap 42.x and before (i.e. openSUSE <= 13) - o openSUSE Tumbleweed - o Debian < 8 - o Ubuntu < 16 - -Usually ReaR-2.6 should also work on newer versions of the above listed +- Fedora < 29 +- RHEL < 6 +- CentOS < 6 +- Scientific Linux < 6 +- SLES < 12 +- openSUSE Leap 42.x and before (i.e. openSUSE <= 13) +- openSUSE Tumbleweed +- Debian < 8 +- Ubuntu < 16 + +Usually ReaR 2.7 should also work on newer versions of the above listed supported Linux based operating systems but sometimes arbitrary failures can -happen when software that is used by ReaR (like partitioning tools, -filesystem tools, bootloader tools, ISO image creating tools, networking -tools, and so on) changes in not fully backward compatible ways or when there -are innovations of the basic system (like kernel, storage, bootloader, init, +happen when software that is used by ReaR (like partitioning tools, filesystem +tools, bootloader tools, ISO image creating tools, networking tools, and so +on) changes in not fully backward compatible ways or when there are +innovations of the basic system (like kernel, storage, bootloader, init, networking, and so on) that are not yet supported by ReaR. -In theory ReaR-2.6 should work on openSUSE Tumbleweed but in practice +In theory ReaR 2.7 should work on openSUSE Tumbleweed but in practice arbitrary failures could happen at any time because the Tumbleweed distribution is a pure rolling release version of openSUSE containing the -latest stable versions of all software (cf. https://en.opensuse.org/ -Portal:Tumbleweed) so arbitrary changes of any software are possible at any -time that could arbitrarily break how ReaR works. +latest stable versions of all software +(cf. https://en.opensuse.org/Portal:Tumbleweed) so arbitrary changes of any +software are possible at any time that could arbitrarily break how ReaR works. -ReaR-2.6 may still work for SLES 11 and openSUSE Leap 42.x or even earlier +ReaR 2.7 may still work for SLES 11 and openSUSE Leap 42.x or even earlier openSUSE versions but it is no longer sufficiently well tested there so arbitrary regressions could appear. -ReaR 2.6, ReaR-2.5, and ReaR-2.4 (and probably also some earlier versions) -are known to no longer work reasonably well for the following Linux based -operating systems: +ReaR 2.7 and earlier versions are known to no longer work reasonably well for +the following Linux based operating systems: - o RHEL 5 (and probably also CentOS 5): See issue #1766 - o SLES 9 and 10: See issue #1842 +- RHEL 5 (and probably also CentOS 5): See issue #1766 +- SLES 9 and 10: See issue #1842 If you require support for unsupported Linux operating systems you must acquire a ReaR support contract. @@ -2568,25 +3808,25 @@ achieved with serious sponsoring. Supported and Unsupported Architectures -ReaR-2.6 is supported on: +ReaR 2.7 is supported on: - o Intel x86 type of processors - o AMD x86 type of processors - o PPC64 processors - o PPC64LE processors +- Intel x86 type of processors +- AMD x86 type of processors +- PPC64 processors +- PPC64LE processors -ReaR-2.6 may or may not work on: +ReaR 2.7 may or may not work on: - o Intel Itanium processors - o ARM type of processors - o IBM Z "s390x" type of processors +- Intel Itanium processors +- ARM type of processors +- IBM Z "s390x" type of processors -ReaR-2.6 does not support: +ReaR 2.7 does not support: - o old PPC (32bit) processors +- old PPC (32bit) processors If you feel the need to get a fully functional ReaR working on one of the -above mentioned type of processors please buy consultancy from one of our +above mentioned types of processors please buy consultancy from one of our official developers. Supported ReaR versions @@ -2596,26 +3836,26 @@ versions. If you have a problem we urge you to install the latest stable ReaR version or the development version (available on GitHub) before submitting an issue. -However, we do understand that it is not always possible to install on -hundreds of systems the latest version so we are willing to support previous -versions of ReaR if you buy a support contract. Why do we change our policy? -We cannot handle the big support requests anymore and we must give paid -projects priority, therefore, we urge our customers to buy a support contract -for one or more systems. You buy time with our core developers. +We understand that it is not always possible to install on hundreds of systems +the latest version so we are willing to support previous versions of ReaR when +you buy a support contract. We cannot handle the big amount of support +requests anymore on a voluntary base and we must give paid projects priority, +therefore, we urge our customers to buy a support contract for one or more +systems. You buy time with our core developers. Known Problems and Workarounds Issue Description: 'rear' package on Ubuntu 14.04 depends on isolinux package (which does not exist) - o Workaround: +- Workaround: Read the comments in issue #1403 Issue Description: tar -test-label is not supported on Centos 5 who have tar version 1.15 - o Workaround: +- Workaround: Read the comments in issue #1014 @@ -2624,7 +3864,7 @@ Issue Description: BACKUP=NSR on RHEL 6 could break yum Issue #387 describes a problem seen on RHEL 6 where when rear uses NSR and afterwards the link /lib64/libexpat.so.1 has been changed. - o Workaround: +- Workaround: So far there is no workaround for this issue. @@ -2635,15 +3875,15 @@ Using rear -v -c /etc/rear/mydir mkbackup works fine in production, but when you try (once booted from rescue image) rear -v -c /etc/rear/mydir recover it will fail. - o Workaround: +- Workaround: -The configuration files are copied to /etc/rear/ into the rescue image, so -you need to type: rear -v recover See issue #512 +The configuration files are copied to /etc/rear/ into the rescue image, so you +need to type: rear -v recover See issue #512 Issue Description: Is there a possibility to add btrfs subvolume to a rsync backup - o Workaround: +- Workaround: At present (release 1.18) there is no workaround in place. If you happen to know how this could be fixed then add your ideas to issue #417 @@ -2651,7 +3891,7 @@ know how this could be fixed then add your ideas to issue #417 Issue Description: UEFI ISO booting does not work on openSUSE 12.x, or SLES 11/12 - o Workaround: +- Workaround: At present (release 1.18.x and higher) genisoimage cannot produce ISO images that can boot via UEFI on an openSUSE distribution (and also SLES). However, @@ -2659,38 +3899,38 @@ use the ebiso package instead to create UEFI ISO images on SLES. Issue Description: System reconfiguration still has some weaknesses. - o this has to be tested before relying on it, there are too many unknowns +- this has to be tested before relying on it, there are too many unknowns involved so that we cannot guarantee anything in this area. It has been developed mostly as a P2V tool to migrate HP servers to VMware Vms - o hard disks need to be at least of the same size and amount as in the +- hard disks need to be at least of the same size and amount as in the original system, ATM this is a simple 1:1 mapping of old to new disks, there is no removal of RAID groups or merging of smaller disks onto a bigger one or making stuff smaller. - o any use of /dev/disk/by-path or /dev/disk/by-id is untested and will most - likely not work. In some cases Relax-and-Recover will print a warning, - but we are not able to detect all cases. Typically this leads to - unbootable systems or bad /etc/fstab files +- any use of /dev/disk/by-path or /dev/disk/by-id is untested and will most + likely not work. In some cases Relax-and-Recover will print a warning, but + we are not able to detect all cases. Typically this leads to unbootable + systems or bad /etc/fstab files Issue Description: If SELinux is not disabled during backup (variable BACKUP_SELINUX_DISABLE= in /etc/rear/local.conf) then we might see errors in the rear-$(hostname).log file such as: -tar: var/cache/yum/i386/15/updates/packages: Cannot setfilecon: No such file or directory + tar: var/cache/yum/i386/15/updates/packages: Cannot setfilecon: No such file or directory - o Workaround: +- Workaround: Make sure the BACKUP_URL destination understands extended attributes (CIFS is out of the question and NFS is problematic). When using local disks (or external USB devices) make sure the proper mount options are given in the BACKUP_OPTIONS variable, e.g.: -BACKUP_OPTIONS="rw,relatime,seclabel,user_xattr,acl,barrier=1,data=ordered" + BACKUP_OPTIONS="rw,relatime,seclabel,user_xattr,acl,barrier=1,data=ordered" -(TIP) BACKUP_SELINUX_DISABLE=1 variable has been introduced in the /usr/share -/rear/conf/default.conf file to disable SELinux while the backup is running -(default setting). +(TIP) BACKUP_SELINUX_DISABLE=1 variable has been introduced in the +/usr/share/rear/conf/default.conf file to disable SELinux while the backup is +running (default setting). Issue Description: ERROR: FindStorageDrivers called but STORAGE_DRIVERS is empty @@ -2699,8 +3939,7 @@ Above error message might be seen after a fresh installation of the GNU/Linux kernel. ReaR got confused between the running kernel version number and the actual fresh kernel available. - o Workaround: +- Workaround: Reboot your server before using ReaR again, which is a good practice anyway after upgrading the GNU/Linux kernel. - From 0a7b67d08721dff4c261e6f2a486b87cb3e3ec17 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 14 Jun 2022 16:47:57 +0200 Subject: [PATCH 135/215] In doc/rear-release-notes.txt added missing http... links --- doc/rear-release-notes.txt | 50 +++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index 4ce8013c80..be9cdeec3b 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -3,9 +3,9 @@ Release Notes for Relax-and-Recover version 2.7 This document contains the release notes for the open source project Relax-and-Recover. -Relax-and-Recover website +Relax-and-Recover website: http://relax-and-recover.org/ -GitHub project +GitHub project: https://github.com/rear/ This document is distributed with the following license: "Creative Commons Attribution-NoDerivs 3.0 Unported (CC BY-ND 3.0)". To read the license deed, @@ -71,8 +71,10 @@ functionality: - rsync over the network (BACKUP=RSYNC, BACKUP_PROG=rsync) - Multiple backup methods. See the documentation about Using Multiple Backups for Relax-and-Recover + https://github.com/rear/rear/blob/master/doc/user-guide/11-multiple-backups.adoc - Any partition (e.g. a Windows partition) via BACKUP=BLOCKCLONE. See the documention about BLOCKCLONE + https://github.com/rear/rear/blob/master/doc/user-guide/12-BLOCKCLONE.adoc - BACKUP=ZYPPER is for SUSE systems only (Experimental) - BACKUP=YUM is for RedHat architectures ony (Experimental) @@ -95,7 +97,7 @@ functionality: - Borg Backup (BACKUP=BORG) - Rubrik Cloud Data Management (BACKUP=CDM) -- Integrates with Disaster Recovery Linux Manager (DRLM) +- Integrates with Disaster Recovery Linux Manager (DRLM) http://drlm.org - Udev support (except for some really ancient udev versions) which is the base for many important features: @@ -182,7 +184,7 @@ that is used by Relax-and-Recover and for each change of your basic system you must re-validate that your disaster recovery procedure still works for you. The references pointing to fix #nr or issue #nr refer to our GitHub issues -tracker. +tracker https://github.com/rear/rear/issues Version 2.7 (June 2022) @@ -2889,11 +2891,12 @@ New features and bigger enhancements: running unattended. For details see the USER_INPUT_... config variables in default.conf. Currently not all user dialogs use the UserInput function so that this or that user dialog needs to be adapted when it is reported to - us via our issue tracker. In contrast when programs that are called by - ReaR work interactively (e.g. third-party backup tools that show user - dialogs or password prompts) the program call itself must be adapted to - run unattended (if possible), see the section 'It should be possible to - run ReaR unattended' in our https://github.com/rear/rear/wiki/Coding-Style + us via our issue tracker https://github.com/rear/rear/issues + In contrast when programs that are called by ReaR work interactively + (e.g. third-party backup tools that show user dialogs or password prompts) + the program call itself must be adapted to run unattended (if possible), + see the section 'It should be possible to run ReaR unattended' in our + https://github.com/rear/rear/wiki/Coding-Style Wiki article. - SSH support in the ReaR rescue/recovery system was overhauled. By default @@ -2929,7 +2932,8 @@ Possibly backward incompatible changes: fix as many program calls as possible but it is impossible (with reasonable effort / with a reasonable amount of time) to check all program calls in all ReaR scripts so that this or that unnoticed program call will - need to be fixed when it is reported to us via our issue tracker. + need to be fixed when it is reported to us via our issue tracker + https://github.com/rear/rear/issues - SSH support in the ReaR rescue/recovery system is now secure by default. There are no longer private SSH keys in the recovery system by default and @@ -3323,7 +3327,9 @@ to check this. #1118) - Multiple backups are now possible (issues #1088, #1102, #1096) - see the - documentation page (New) + documentation page + https://github.com/rear/rear/blob/master/doc/user-guide/11-multiple-backups.adoc + (New) - Support partitioning and formatting huge USB devices (issue #1105) @@ -3717,7 +3723,7 @@ System and Software Requirements Relax-and-Recover works on GNU/Linux kernel with version 2.6 and higher. For lower kernel versions Relax-and-Recover cannot be used, and for these systems, -mkcdrec is still a good alternative. +mkcdrec http://mkcdrec.sourceforge.net/ is still a good alternative. As Relax-and-Recover has been solely written in the bash language we need the bash shell which is standard available on all GNU/Linux based systems. The @@ -3735,6 +3741,7 @@ The default backup program with Relax-and-Recover is (BACKUP_PROG=tar) GNU tar and the default compression used with tar is gzip. However, is using gzip the best choice? We have done some tests and published the results. See Relax-and-Recover compression tests +http://www.it3.be/2013/09/16/NETFS-compression-tests/ Support @@ -3748,8 +3755,9 @@ as a service (not free of charge). Supported and Unsupported Operating Systems -We try to keep our wiki page Test Matrix ReaR 2.7 up-to-date with feedback we -receive from the community. +We try to keep our wiki page Test Matrix ReaR 2.7 +https://github.com/rear/rear/wiki/Test-Matrix-ReaR-2.7 +up-to-date with feedback we receive from the community. ReaR 2.7 is supported on the following Linux based operating systems: @@ -3850,18 +3858,19 @@ Issue Description: 'rear' package on Ubuntu 14.04 depends on isolinux package - Workaround: -Read the comments in issue #1403 +Read the comments in https://github.com/rear/rear/issues/1403 Issue Description: tar -test-label is not supported on Centos 5 who have tar version 1.15 - Workaround: -Read the comments in issue #1014 +Read the comments in https://github.com/rear/rear/issues/1014 Issue Description: BACKUP=NSR on RHEL 6 could break yum -Issue #387 describes a problem seen on RHEL 6 where when rear uses NSR and +https://github.com/rear/rear/issues/387 +describes a problem seen on RHEL 6 where when rear uses NSR and afterwards the link /lib64/libexpat.so.1 has been changed. - Workaround: @@ -3886,7 +3895,8 @@ backup - Workaround: At present (release 1.18) there is no workaround in place. If you happen to -know how this could be fixed then add your ideas to issue #417 +know how this could be fixed then add your ideas to +https://github.com/rear/rear/issues/417 Issue Description: UEFI ISO booting does not work on openSUSE 12.x, or SLES 11/12 @@ -3895,7 +3905,9 @@ Issue Description: UEFI ISO booting does not work on openSUSE 12.x, or SLES At present (release 1.18.x and higher) genisoimage cannot produce ISO images that can boot via UEFI on an openSUSE distribution (and also SLES). However, -use the ebiso package instead to create UEFI ISO images on SLES. +use the ebiso package +http://download.opensuse.org/repositories/Archiving:/Backup:/Rear/SLE_11_SP3/x86_64/ebiso-0.2.3-1.x86_64.rpm +instead to create UEFI ISO images on SLES. Issue Description: System reconfiguration still has some weaknesses. From 60e6d27a32c8b8c8ab5b5af571e097f5c6a198af Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 15 Jun 2022 15:35:00 +0200 Subject: [PATCH 136/215] Added Renaud Metrich and pcahyna to the AUTHORS section in doc/rear.8.adoc and replaced some 'please' therein with better wording and made it plain ASCII and then regenerated doc/rear.8 via 'asciidoctor -b manpage rear.8.adoc' so now also rear.8 is plain ASCII --- doc/rear.8 | 56 ++++++++++++++++++++++++++++++------------------- doc/rear.8.adoc | 39 ++++++++++++++++++++++++---------- 2 files changed, 63 insertions(+), 32 deletions(-) diff --git a/doc/rear.8 b/doc/rear.8 index 5c7bb42bb2..0ad3ad8d04 100644 --- a/doc/rear.8 +++ b/doc/rear.8 @@ -2,12 +2,12 @@ .\" Title: rear .\" Author: [see the "AUTHOR(S)" section] .\" Generator: Asciidoctor 2.0.10 -.\" Date: 2022-06-09 +.\" Date: 2022-06-15 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "REAR" "8" "2022-06-09" "\ \&" "\ \&" +.TH "REAR" "8" "2022-06-15" "\ \&" "\ \&" .ie \n(.g .ds Aq \(aq .el .ds Aq ' .ss \n[.ss] 0 @@ -133,7 +133,7 @@ mkbackup/mkrescue .sp \fBdump\fP .RS 4 -dump configuration and system information; please run this to verify +dump configuration and system information; run this to verify your setup .RE .sp @@ -437,8 +437,8 @@ Skip network devices setup by the rescue system start up scripts (overrides ip= .RE .SH "BACKUP SOFTWARE INTEGRATION" .sp -Currently Relax\-and\-Recover supports the following backup methods. Please -distinguish carefully between Relax\-and\-Recover support for 3rd party +Currently Relax\-and\-Recover supports the following backup methods. +There is a distinction between Relax\-and\-Recover support for 3rd party backup software and Relax\-and\-Recover internal backup methods. The latter also creates a backup of your data while the former will only integrate Relax\-and\-Recover with the backup software to restore the data with the @@ -847,12 +847,12 @@ Relax\-and\-Recover script components. /usr/share/rear/conf/default.conf .RS 4 Relax\-and\-Recover default values. Contains a complete set of parameters -and its explanation. Please do not edit or modify. Copy values to -\fIlocal.conf\fP or \fIsite.conf\fP instead. +and its explanation. Do not edit or modify things therein but +use \fIlocal.conf\fP or \fIsite.conf\fP for specific settings. .RE .SH "BUGS" .sp -Feedback is welcome, please report any issues or improvements to our +Feedback is welcome, please report issues or improvements to our issue\-tracker at: \c .URL "http://github.com/rear/issues/" "" "" .sp @@ -865,27 +865,41 @@ Relax\-and\-Recover comes with extensive documentation located in .sp The ReaR project was initiated in 2006 by Schlomo Schapiro (\c .URL "https://github.com/schlomo" "" ")" -and Gratien D\(cqhaese ( +and +Gratien D\(cqhaese (\c .URL "https://github.com/gdha" "" ")" -and has since then seen a lot of contributions by many authors. As ReaR deals with bare metal disaster recovery, there is a large amount of code that was contributed by owners and users of specialized hardware and software. Without their combined efforts and contributions ReaR would not be the universal Linux bare metal disaster recovery solution that it is today. -.sp -As time passed the project was lucky to get the support of additional developers to also help as maintainers: Dag Wieers (\c +and has since then seen a lot of contributions by many authors. +As ReaR deals with bare metal disaster recovery, there is a large amount of code +that was contributed by owners and users of specialized hardware and software. +Without their combined efforts and contributions ReaR would not be +the universal Linux bare metal disaster recovery solution that it is today. +.sp +As time passed the project was lucky to get the support of additional developers to also help as maintainers: +Dag Wieers (\c .URL "https://github.com/dagwieers" "" ")," -Jeroen Hoekx ( +Jeroen Hoekx (\c .URL "https://github.com/jhoekx" "" ")," -Johannes Meixner ( +Johannes Meixner (\c .URL "https://github.com/jsmeix" "" ")," -Vladimir Gozora ( -.URL "https://github.com/gozora" "" ")" -and Sébastien Chabrolles ( -.URL "https://github.com/schabrolles" "" ")." -We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. Please refer to the MAINTAINERS ( +Vladimir Gozora (\c +.URL "https://github.com/gozora" "" ")," +Sebastien Chabrolles (\c +.URL "https://github.com/schabrolles" "" ")," +Renaud Metrich (\c +.URL "https://github.com/rmetrich" "" ")" +and +"pcahyna" (\c +.URL "https://github.com/pcahyna" "" ")." +We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. +Refer to the MAINTAINERS (\c .URL "https://github.com/rear/rear/blob/master/MAINTAINERS" "" ")" file for the list of active and past maintainers. .sp -To see the full list of authors and their contributions please look at the git history (\c +To see the full list of authors and their contributions +look at the git history (\c .URL "https://github.com/rear/rear/graphs/contributors" "" ")." -We are very thankful to all authors and encourage anybody interested to take a look at our source code and to contribute what you find important. +We are very thankful to all authors and encourage anybody interested +to take a look at our source code and to contribute what you find important. .sp Relax\-and\-Recover is a collaborative process using Github at \c .URL "http://github.com/rear/" "" "" diff --git a/doc/rear.8.adoc b/doc/rear.8.adoc index 80afac41b7..4b5bd9dc71 100644 --- a/doc/rear.8.adoc +++ b/doc/rear.8.adoc @@ -93,7 +93,7 @@ the GNU General Public License at: http://www.gnu.org/licenses/gpl.html mkbackup/mkrescue *dump*:: - dump configuration and system information; please run this to verify + dump configuration and system information; run this to verify your setup *format*:: @@ -276,8 +276,8 @@ ip=192.168.100.2 nm=255.255.255.0 netdev=eth0 gw=192.168.100.1 Skip network devices setup by the rescue system start up scripts (overrides ip= nm= netdev= gw=). == BACKUP SOFTWARE INTEGRATION -Currently Relax-and-Recover supports the following backup methods. Please -distinguish carefully between Relax-and-Recover support for 3rd party +Currently Relax-and-Recover supports the following backup methods. +There is a distinction between Relax-and-Recover support for 3rd party backup software and Relax-and-Recover internal backup methods. The latter also creates a backup of your data while the former will only integrate Relax-and-Recover with the backup software to restore the data with the @@ -561,12 +561,12 @@ Relax-and-Recover script components. /usr/share/rear/conf/default.conf:: Relax-and-Recover default values. Contains a complete set of parameters -and its explanation. Please do not edit or modify. Copy values to -_local.conf_ or _site.conf_ instead. +and its explanation. Do not edit or modify things therein but +use _local.conf_ or _site.conf_ for specific settings. == BUGS -Feedback is welcome, please report any issues or improvements to our +Feedback is welcome, please report issues or improvements to our issue-tracker at: http://github.com/rear/issues/ Furthermore, we welcome pull requests via GitHub. @@ -577,11 +577,28 @@ _/usr/share/doc_. == AUTHORS -The ReaR project was initiated in 2006 by Schlomo Schapiro (https://github.com/schlomo) and Gratien D'haese (https://github.com/gdha) and has since then seen a lot of contributions by many authors. As ReaR deals with bare metal disaster recovery, there is a large amount of code that was contributed by owners and users of specialized hardware and software. Without their combined efforts and contributions ReaR would not be the universal Linux bare metal disaster recovery solution that it is today. - -As time passed the project was lucky to get the support of additional developers to also help as maintainers: Dag Wieers (https://github.com/dagwieers), Jeroen Hoekx (https://github.com/jhoekx), Johannes Meixner (https://github.com/jsmeix), Vladimir Gozora (https://github.com/gozora) and Sébastien Chabrolles (https://github.com/schabrolles). We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. Please refer to the MAINTAINERS (https://github.com/rear/rear/blob/master/MAINTAINERS) file for the list of active and past maintainers. - -To see the full list of authors and their contributions please look at the git history (https://github.com/rear/rear/graphs/contributors). We are very thankful to all authors and encourage anybody interested to take a look at our source code and to contribute what you find important. +The ReaR project was initiated in 2006 by Schlomo Schapiro (https://github.com/schlomo) and +Gratien D'haese (https://github.com/gdha) and has since then seen a lot of contributions by many authors. +As ReaR deals with bare metal disaster recovery, there is a large amount of code +that was contributed by owners and users of specialized hardware and software. +Without their combined efforts and contributions ReaR would not be +the universal Linux bare metal disaster recovery solution that it is today. + +As time passed the project was lucky to get the support of additional developers to also help as maintainers: +Dag Wieers (https://github.com/dagwieers), +Jeroen Hoekx (https://github.com/jhoekx), +Johannes Meixner (https://github.com/jsmeix), +Vladimir Gozora (https://github.com/gozora), +Sebastien Chabrolles (https://github.com/schabrolles), +Renaud Metrich (https://github.com/rmetrich) and +"pcahyna" (https://github.com/pcahyna). +We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. +Refer to the MAINTAINERS (https://github.com/rear/rear/blob/master/MAINTAINERS) file for the list of active and past maintainers. + +To see the full list of authors and their contributions +look at the git history (https://github.com/rear/rear/graphs/contributors). +We are very thankful to all authors and encourage anybody interested +to take a look at our source code and to contribute what you find important. Relax-and-Recover is a collaborative process using Github at http://github.com/rear/ From 7469cea1364de12ac5826418a95ab2154c775bfb Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Mon, 4 Apr 2022 19:20:35 +0200 Subject: [PATCH 137/215] Update Packit build versions Update Fedora using the fedora-all alias, to avoid having to bump the version in the future. https://packit.dev/docs/configuration/#aliases Add CentOS Stream. --- .packit.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.packit.yaml b/.packit.yaml index 04e65153b7..007bc575eb 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -3,10 +3,9 @@ jobs: - job: copr_build metadata: targets: - - fedora-29-x86_64 - - fedora-30-x86_64 - - fedora-31-x86_64 - - fedora-rawhide-x86_64 + - fedora-all + - centos-stream-8-x86_64 + - centos-stream-9-x86_64 trigger: pull_request specfile_path: packaging/rpm/rear.spec synced_files: From a5b69b716d7786961e9cfc84f7b5dec0899f32dd Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Mon, 16 May 2022 15:03:40 +0200 Subject: [PATCH 138/215] Configure Koji scratch builds in Packit Despite the name, production_build refers to Koji scratch builds. https://packit.dev/docs/configuration/#production_build --- .packit.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.packit.yaml b/.packit.yaml index 007bc575eb..0cbf6fd3ea 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -7,6 +7,13 @@ jobs: - centos-stream-8-x86_64 - centos-stream-9-x86_64 trigger: pull_request +- job: production_build + metadata: + scratch: True + targets: + - fedora-latest-stable + - epel-all + trigger: pull_request specfile_path: packaging/rpm/rear.spec synced_files: - .packit.yaml From 4d18bf9b25d2d15c9f184247b1cb6d98dd039c3f Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Thu, 9 Jun 2022 15:03:29 +0200 Subject: [PATCH 139/215] Add openSUSE Copr targets for Packit --- .packit.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.packit.yaml b/.packit.yaml index 0cbf6fd3ea..73787fbb39 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -6,6 +6,8 @@ jobs: - fedora-all - centos-stream-8-x86_64 - centos-stream-9-x86_64 + - opensuse-leap-15.3-x86_64 + - opensuse-tumbleweed-x86_64 trigger: pull_request - job: production_build metadata: From 5e3c97246fda077a827550339fe65f013230e6fa Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Thu, 16 Jun 2022 16:05:28 +0200 Subject: [PATCH 140/215] Avoid creating a /bin/vim symlink to vi The symlink is more confusing than helpful: if one has both vi and vim, and they are different (vi usually has less features), one expects to get the more featureful version by executing "vim", but one gets "vi". More importantly, in recent Fedora and RHEL, "vi" is a shell script that executes "vim" if found, so linking "vim" to "vi" leads to an infinite loop. --- usr/share/rear/build/GNU/Linux/005_create_symlinks.sh | 1 - .../Linux-i386/106_remove_files_copied_unconditionally.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/usr/share/rear/build/GNU/Linux/005_create_symlinks.sh b/usr/share/rear/build/GNU/Linux/005_create_symlinks.sh index df75e07d0f..55f25bef68 100644 --- a/usr/share/rear/build/GNU/Linux/005_create_symlinks.sh +++ b/usr/share/rear/build/GNU/Linux/005_create_symlinks.sh @@ -8,7 +8,6 @@ ln -sf $v bin/init $ROOTFS_DIR/init >&2 ln -sf $v bin $ROOTFS_DIR/sbin >&2 ln -sf $v bash $ROOTFS_DIR/bin/sh >&2 -ln -sf $v vi $ROOTFS_DIR/bin/vim >&2 ln -sf $v true $ROOTFS_DIR/bin/pam_console_apply >&2 # RH/Fedora with udev needs this ln -sf $v ../bin $ROOTFS_DIR/usr/bin >&2 ln -sf $v ../bin $ROOTFS_DIR/usr/sbin >&2 diff --git a/usr/share/rear/build/OPALPBA/Linux-i386/106_remove_files_copied_unconditionally.sh b/usr/share/rear/build/OPALPBA/Linux-i386/106_remove_files_copied_unconditionally.sh index abc982fc36..dcb2918ce3 100644 --- a/usr/share/rear/build/OPALPBA/Linux-i386/106_remove_files_copied_unconditionally.sh +++ b/usr/share/rear/build/OPALPBA/Linux-i386/106_remove_files_copied_unconditionally.sh @@ -5,7 +5,6 @@ # Remove symlinks whose targets have been excluded on the PBA system local symlinks_to_remove=( - bin/vim var/lib/rear ) From 7bd877dcc7d41f5dd8b88f684dff2223b4f5461d Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Thu, 16 Jun 2022 16:22:05 +0200 Subject: [PATCH 141/215] Add /usr/libexec/vi to the rescue ramdisk On recent Fedora and RHEL 9, vi is a shell script that calls vim or /usr/libexec/vi. If neither is installed, the vi command is not useful. One could add vim as well, but I am afraid that it would bloat the rescue ramdisk in case vim is compiled with a rich set of features. (Should there be a Fedora-specific configuration file? There is already SUSE_LINUX.conf and Ubuntu.conf .) --- usr/share/rear/conf/GNU/Linux.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/share/rear/conf/GNU/Linux.conf b/usr/share/rear/conf/GNU/Linux.conf index f39df4c83e..318aa45174 100644 --- a/usr/share/rear/conf/GNU/Linux.conf +++ b/usr/share/rear/conf/GNU/Linux.conf @@ -227,6 +227,12 @@ LIBS+=( ) COPY_AS_IS+=( /dev /etc/inputr[c] /etc/protocols /etc/services /etc/rpc /etc/termcap /etc/terminfo /lib*/terminfo /usr/share/terminfo /etc/netconfig /etc/mke2fs.conf /etc/*-release /etc/localtime /etc/magic /usr/share/misc/magic /etc/dracut.conf /etc/dracut.conf.d /usr/lib/dracut /sbin/modprobe.ksplice-orig /etc/sysctl.conf /etc/sysctl.d /etc/e2fsck.conf ) + +# Needed by vi on Fedora and derived distributions +# where vi is a shell script that executes /usr/libexec/vi +# see https://github.com/rear/rear/pull/2822 +COPY_AS_IS+=( /usr/libexec/vi ) + # Required by curl with https: # There are stored the distribution provided certificates # installed from packages, nothing confidential. From 9d6ce289938092834425cb1c27658500f68bc69f Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 22 Jun 2022 13:41:40 +0200 Subject: [PATCH 142/215] Use fail-safe 'yes' pipe for "lvm lvcreate" In layout/prepare/GNU/Linux/110_include_lvm_code.sh pipe as many 'y' as asked for into "lvm lvcreate" see https://github.com/rear/rear/issues/513 and https://github.com/rear/rear/issues/2820 --- .../prepare/GNU/Linux/110_include_lvm_code.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh b/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh index fd7f93bf44..0bd863acd7 100644 --- a/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh +++ b/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh @@ -287,13 +287,25 @@ create_lvmvol() { # so e.g. 'lvcreate -L 123456b -n LV VG' becomes 'lvcreate -l 100%FREE -n LV VG' fallbacklvopts="$( sed -e 's/-L [0-9b]*/-l 100%FREE/' <<< "$lvopts" )" + # In SLES11 "man lvcreate" does not show '-y' or '--yes' + # so we cannot use "lvm lvcreate -y ..." + # see https://github.com/rear/rear/issues/2820#issuecomment-1161934013 + # instead we input as many 'y' as asked for by "lvm lvcreate" + # see https://github.com/rear/rear/issues/513 + # and https://github.com/rear/rear/issues/2820 + # plus be safe against possible 'set -o pipefail' non-zero exit code of 'yes' via '( yes || true ) | ...' + # see https://github.com/rear/rear/issues/2820#issuecomment-1162804476 + # because 'yes' may get terminated by SIGPIPE when plain 'yes | ...' is used + # see https://github.com/rear/rear/issues/2820#issuecomment-1162772415 + # and suppress needless "yes: standard output: Broken pipe" stderr messages + # that appear at least with newer 'yes' in coreutils-8.32 in openSUSE Leap 15.3 cat >> "$LAYOUT_CODE" </dev/null || true ) | lvm lvcreate $lvopts $vg ; then LogPrintError "Failed to create LVM volume '$vg/$lvname' with lvcreate $lvopts $vg" - if lvm lvcreate $fallbacklvopts $vg <</dev/null || true ) | lvm lvcreate $fallbacklvopts $vg ; then LogPrintError "Created LVM volume '$vg/$lvname' using fallback options lvcreate $fallbacklvopts $vg" else LogPrintError "Also failed to create LVM volume '$vg/$lvname' with lvcreate $fallbacklvopts $vg" From 2d613ad19ef2c25b348c914ba9f9d91742ea06ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zaoral?= Date: Tue, 21 Jun 2022 14:32:25 +0200 Subject: [PATCH 143/215] Replace mkinitrd with dracut on Fedora and RHEL For a long time, mkinitrd was a wrapper script around dracut in Fedora and RHEL that maintained backwards compatibility with the real (but rather obsolete) mkinitrd. Dracut 054 removed the mkinitrd wrapper script [1] so now we have to execute dracut directly. Fortunately, ReaR did not use almost anything from mkinitrd's interface (with the exception of --with) so the transition is quite straightforward and should not introduce any regressions because dracut is the default initrd generator since RHEL 6. Without this patch the initrd regeneration in the finalize stage of recovery fails on RHEL 9 and Fedora 33 or newer. [1] https://github.com/dracutdevs/dracut/pull/1285 --- .../Fedora/i386/550_rebuild_initramfs.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh index 213c5c2da7..70187d6c3a 100644 --- a/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh +++ b/usr/share/rear/finalize/Fedora/i386/550_rebuild_initramfs.sh @@ -60,7 +60,7 @@ INITRD_MODULES="${OLD_INITRD_MODULES[*]} ${NEW_INITRD_MODULES[*]}" Log "New INITRD_MODULES='$INITRD_MODULES'" # Do not quote $INITRD_MODULES otherwise printf could not split words as separated arguments on separated lines: -WITH_INITRD_MODULES=$( printf '%s\n' $INITRD_MODULES | awk '{printf "--with=%s ", $1}' ) +WITH_INITRD_MODULES=$( printf '%s\n' $INITRD_MODULES | awk '{printf "--add-drivers=%s ", $1}' ) # Recreate any initrd or initramfs image under $TARGET_FS_ROOT/boot/ with new drivers # Images ignored: @@ -71,19 +71,19 @@ for INITRD_IMG in $( ls $TARGET_FS_ROOT/boot/initramfs-*.img $TARGET_FS_ROOT/boo # Do not use KERNEL_VERSION here because that is readonly in the rear main script: kernel_version=$( basename $( echo $INITRD_IMG ) | cut -f2- -d"-" | sed s/"\.img"// ) INITRD=$( echo $INITRD_IMG | egrep -o "/boot/.*" ) - LogPrint "Running mkinitrd..." - # Run mkinitrd directly in chroot without a login shell in between (see https://github.com/rear/rear/issues/862). - # We need the mkinitrd binary in the chroot environment i.e. the mkinitrd binary in the recreated system. - # Normally we would use a login shell like: chroot $TARGET_FS_ROOT /bin/bash --login -c 'type -P mkinitrd' + LogPrint "Running dracut..." + # Run dracut directly in chroot without a login shell in between (see https://github.com/rear/rear/issues/862). + # We need the dracut binary in the chroot environment i.e. the dracut binary in the recreated system. + # Normally we would use a login shell like: chroot $TARGET_FS_ROOT /bin/bash --login -c 'type -P dracut' # because otherwise there is no useful PATH (PATH is only /bin) so that 'type -P' won't find it # but we cannot use a login shell because that contradicts https://github.com/rear/rear/issues/862 # so that we use a plain (non-login) shell and set a (hopefully) reasonable PATH: - local mkinitrd_binary=$( chroot $TARGET_FS_ROOT /bin/bash -c 'PATH=/sbin:/usr/sbin:/usr/bin:/bin type -P mkinitrd' ) - # If there is no mkinitrd in the chroot environment plain 'chroot $TARGET_FS_ROOT' will hang up endlessly + local dracut_binary=$( chroot $TARGET_FS_ROOT /bin/bash -c 'PATH=/sbin:/usr/sbin:/usr/bin:/bin type -P dracut' ) + # If there is no dracut in the chroot environment plain 'chroot $TARGET_FS_ROOT' will hang up endlessly # and then "rear recover" cannot be aborted with the usual [Ctrl]+[C] keys. # Use plain $var because when var contains only blanks test "$var" results true because test " " results true: - if test $mkinitrd_binary ; then - if chroot $TARGET_FS_ROOT $mkinitrd_binary -v -f $WITH_INITRD_MODULES $INITRD $kernel_version ; then + if test $dracut_binary ; then + if chroot $TARGET_FS_ROOT $dracut_binary -v -f $WITH_INITRD_MODULES $INITRD $kernel_version ; then LogPrint "Updated initrd with new drivers for kernel $kernel_version." else LogPrint "WARNING: @@ -94,7 +94,7 @@ and decide yourself, whether the system will boot or not. fi else LogPrint "WARNING: -Cannot create initrd (found no mkinitrd in the recreated system). +Cannot create initrd (found no dracut in the recreated system). Check the recreated system (mounted at $TARGET_FS_ROOT) and decide yourself, whether the system will boot or not. " From ffd3a8f3fa4bc29e1022f3a8518e4b11fad8189d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zaoral?= Date: Wed, 22 Jun 2022 17:14:58 +0200 Subject: [PATCH 144/215] Recognise -b/--bios options in format workflow --- usr/share/rear/lib/format-workflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/format-workflow.sh b/usr/share/rear/lib/format-workflow.sh index c0509492fd..2698135814 100644 --- a/usr/share/rear/lib/format-workflow.sh +++ b/usr/share/rear/lib/format-workflow.sh @@ -19,7 +19,7 @@ WORKFLOW_format () { # Parse options # (do not use OPTS here because that is readonly in the rear main script): - format_workflow_opts="$( getopt -n "$PROGRAM format" -o "efhy" -l "efi,force,help,yes" -- "$@" )" + format_workflow_opts="$( getopt -n "$PROGRAM format" -o "befhy" -l "bios,efi,force,help,yes" -- "$@" )" if (( $? != 0 )) ; then LogPrintError "Use '$PROGRAM format -- --help' for more information." # TODO: Use proper exit codes cf. https://github.com/rear/rear/issues/1134 From 5e1ca54e119f016a366bbae6501ff48bec575e64 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 23 Jun 2022 12:17:01 +0200 Subject: [PATCH 145/215] Enhanced LogPrintError description comment In lib/_input-output-functions.sh enhanced LogPrintError description comment that when there was already a LogPrintError message then subsequent messages that are related to that LogPrintError message must also be shown to the user as LogPrintError messages, cf. https://github.com/rear/rear/pull/2828#issuecomment-1164225761 --- usr/share/rear/lib/_input-output-functions.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/rear/lib/_input-output-functions.sh b/usr/share/rear/lib/_input-output-functions.sh index 4e04ca21b5..fae162798b 100644 --- a/usr/share/rear/lib/_input-output-functions.sh +++ b/usr/share/rear/lib/_input-output-functions.sh @@ -493,6 +493,11 @@ function LogUserOutput () { # LogPrintError is also meant to show important "error-like" messages to the user # (for example when the user must decide if that means a real error in his case) # and other important messages that must appear on the user's terminal. +# In particular when there was already a LogPrintError message +# subsequent messages that are related to this LogPrintError message +# must also be shown to the user as LogPrintError messages +# to ensure the user gets them on his terminal regardless +# whether or not he launched 'rear' in verbose mode. # Messages that are no real error messages should not be prefixed with 'Warning: ' # cf. https://blog.schlomo.schapiro.org/2015/04/warning-is-waste-of-my-time.html function LogPrintError () { From 7e1410fa2c62fdd846f5d7c5a3585c68f0556cd6 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 24 Jun 2022 08:27:03 +0200 Subject: [PATCH 146/215] Set USB_DEVICE_PARTED_LABEL to match format-workflow.sh In format/USB/default/300_format_usb_disk.sh set USB_DEVICE_PARTED_LABEL to match format-workflow.sh which means when a format workflow option -b/--bios or -e/--efi was specified set USB_DEVICE_PARTED_LABEL to "msdos" or "gpt" accordingly and when no format workflow option -b/--bios or -e/--efi was specified it means hybrid boot supporting BIOS and UEFI by default so USB_DEVICE_PARTED_LABEL must be set to "gpt". --- .../format/USB/default/300_format_usb_disk.sh | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/usr/share/rear/format/USB/default/300_format_usb_disk.sh b/usr/share/rear/format/USB/default/300_format_usb_disk.sh index 14d870be9e..33c6bf0115 100644 --- a/usr/share/rear/format/USB/default/300_format_usb_disk.sh +++ b/usr/share/rear/format/USB/default/300_format_usb_disk.sh @@ -32,6 +32,25 @@ local current_partition_number=1 # current start byte of the next partition to add local current_partition_start_byte=$(( USB_PARTITION_ALIGN_BLOCK_SIZE * MiB_bytes )) +### Create partition table section + +# Initialize USB disk via "parted mklabel" (create partition table) +# When a format workflow option -b/--bios or -e/--efi was specified set USB_DEVICE_PARTED_LABEL accordingly +# cf. https://github.com/rear/rear/pull/2828#issuecomment-1164590100 +# and when no format workflow option -b/--bios or -e/--efi was specified +# then rear/lib/format-workflow.sh sets both FORMAT_BIOS and FORMAT_EFI to 'y' +# cf. https://github.com/rear/rear/commit/9591fbf77c0c12329738625fcb83bb1d9b419b51 +# to get hybrid boot supporting BIOS and UEFI from the same medium by default +# see https://github.com/rear/rear/pull/2705 +# so the ordering of the two settings below is crucial +# to ensure a GUID partition table is set up for hybrid boot: +is_true "$FORMAT_BIOS" && USB_DEVICE_PARTED_LABEL="msdos" +is_true "$FORMAT_EFI" && USB_DEVICE_PARTED_LABEL="gpt" +LogPrint "Creating partition table of type $USB_DEVICE_PARTED_LABEL on $RAW_USB_DEVICE" +if ! parted -s $RAW_USB_DEVICE mklabel $USB_DEVICE_PARTED_LABEL ; then + Error "Failed to create $USB_DEVICE_PARTED_LABEL partition table on $RAW_USB_DEVICE" +fi + # Flag for the partition wherefrom is booted which is the boot partition if exists # or the data partition as fallback when there is no boot partition: local boot_partition_flag="$USB_BOOT_PARTITION_FLAG" @@ -51,16 +70,6 @@ if ! test $boot_partition_flag ; then esac fi -### Create partition table section - -# Initialize USB disk via "parted mklabel" (create partition table) -# If not set use fallback value 'msdos' (same as the default value in default.conf): -test "msdos" = "$USB_DEVICE_PARTED_LABEL" -o "gpt" = "$USB_DEVICE_PARTED_LABEL" || USB_DEVICE_PARTED_LABEL="msdos" -LogPrint "Creating partition table of type $USB_DEVICE_PARTED_LABEL on $RAW_USB_DEVICE" -if ! parted -s $RAW_USB_DEVICE mklabel $USB_DEVICE_PARTED_LABEL ; then - Error "Failed to create $USB_DEVICE_PARTED_LABEL partition table on $RAW_USB_DEVICE" -fi - ### Create partitions section # in order: # * BIOS partition (GPT only) / partition gap aka empty space for MSDOS From ff5ae811efd0dcb201fb30890198510abfe85aa5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 24 Jun 2022 08:46:11 +0200 Subject: [PATCH 147/215] USB_DEVICE_PARTED_LABEL is no longer a user config variable In default.conf describe how USB_DEVICE_PARTED_LABEL is set via the format workflow option -b/--bios or -e/--efi and that it is no longer a user config variable. The description in default.conf is there to inform users who had set USB_DEVICE_PARTED_LABEL in etc/rear/local.conf that this no longer has an effect. --- usr/share/rear/conf/default.conf | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index dd92a51f7a..ffb32fca4c 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -968,14 +968,17 @@ ISO_RECOVER_MODE="" # (see also USB_DEVICE_FILESYSTEM_LABEL below): USB_DEVICE= # -# The partition type that is used when formatting a medium for use with ReaR via the format workflow. +# USB_DEVICE_PARTED_LABEL is the partition type (i.e. what is used for 'parted mklabel') +# that is used when formatting a medium for use with ReaR via the format workflow. # It can be 'msdos' to create a MBR partition table or 'gpt' to create a GUID partition table (GPT). -# When UEFI is used the format workflow will create a GUID partition table in any case. -# A MBR partition table limits the maximum usable storage space on the medium to 2TB. -# For larger medium sizes use 'gpt'. If you must use a MBR partition table on a medium > 2TB -# set USB_DEVICE_FILESYSTEM_PERCENTAGE appropriately so that what is used for ReaR does not exceed -# what works with a MBR partition table but then you cannot use the remaining space on the medium: -USB_DEVICE_PARTED_LABEL=msdos +# It is set depending on the format workflow option -b/--bios or -e/--efi +# so setting USB_DEVICE_PARTED_LABEL in etc/rear/local.conf has no effect. +# When a format workflow option -b/--bios or -e/--efi was specified +# USB_DEVICE_PARTED_LABEL is set to 'msdos' or 'gpt' accordingly. +# When no format workflow option -b/--bios or -e/--efi was specified +# it means hybrid boot supporting BIOS and UEFI by default +# so USB_DEVICE_PARTED_LABEL must be set to 'gpt' +# see https://github.com/rear/rear/issues/2698 # # The label that is set for the ReaR data partition via the format workflow. # That label must be used for settings like From 3f7f9120f3164ecc7a7e1e3776b21ef442a696bb Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 24 Jun 2022 09:40:50 +0200 Subject: [PATCH 148/215] Update 340_find_mbr_bin.sh Same 'case "$USB_DEVICE_PARTED_LABEL"' syntax as in format/USB/default/300_format_usb_disk.sh plus Error instead of StopIfError --- .../prep/USB/Linux-i386/340_find_mbr_bin.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh b/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh index ce4c88adc7..e113e38fb1 100644 --- a/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh +++ b/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh @@ -5,18 +5,17 @@ local mbr_image_file # Choose right MBR image file for right partition table type (issue #1153) case "$USB_DEVICE_PARTED_LABEL" in - "msdos") + (msdos) mbr_image_file="mbr.bin" - ;; - "gpt") + ;; + (gpt) mbr_image_file="gptmbr.bin" - ;; - *) - Error "USB_DEVICE_PARTED_LABEL is incorrectly set, please check your settings." - ;; + ;; + (*) + Error "USB_DEVICE_PARTED_LABEL='$USB_DEVICE_PARTED_LABEL' (neither 'msdos' nor 'gpt')" + ;; esac -SYSLINUX_MBR_BIN=$(find_syslinux_file $mbr_image_file) +SYSLINUX_MBR_BIN=$( find_syslinux_file $mbr_image_file ) -[[ -s "$SYSLINUX_MBR_BIN" ]] -StopIfError "Could not find file '$mbr_image_file'. Syslinux version 3.08 or newer is required, 4.x prefered!" +test -s "$SYSLINUX_MBR_BIN" || Error "Could not find SYSLINUX MBR image file '$mbr_image_file' (at least SYSLINUX 3.08 is required, 4.x preferred)" From be7c1374a8fbffdfc0973ebc0facc0b5822d5e56 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 24 Jun 2022 10:07:21 +0200 Subject: [PATCH 149/215] Update default.conf In default.conf have USB_DEVICE_PARTED_LABEL=gpt to match the new default behaviour of the format workflow and describe that when the format workflow option -b/--bios was used the matching USB_DEVICE_PARTED_LABEL=msdos must be set in etc/rear/local.conf otherwise things would go wrong in prep/USB/Linux-i386/340_find_mbr_bin.sh --- usr/share/rear/conf/default.conf | 42 ++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index ffb32fca4c..416d2c9fe5 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -969,16 +969,22 @@ ISO_RECOVER_MODE="" USB_DEVICE= # # USB_DEVICE_PARTED_LABEL is the partition type (i.e. what is used for 'parted mklabel') -# that is used when formatting a medium for use with ReaR via the format workflow. +# that is used when formatting a medium for use with ReaR via the format workflow and +# when SYSLINUX/EXTLINUX is used as booloader used for the USB medium (see USB_BOOTLOADER below). # It can be 'msdos' to create a MBR partition table or 'gpt' to create a GUID partition table (GPT). -# It is set depending on the format workflow option -b/--bios or -e/--efi -# so setting USB_DEVICE_PARTED_LABEL in etc/rear/local.conf has no effect. +# It is set depending on the format workflow option -b/--bios or -e/--efi as follows: # When a format workflow option -b/--bios or -e/--efi was specified # USB_DEVICE_PARTED_LABEL is set to 'msdos' or 'gpt' accordingly. # When no format workflow option -b/--bios or -e/--efi was specified # it means hybrid boot supporting BIOS and UEFI by default # so USB_DEVICE_PARTED_LABEL must be set to 'gpt' # see https://github.com/rear/rear/issues/2698 +# so we set 'gpt' as default here because USB_DEVICE_PARTED_LABEL +# is needed in prep/USB/Linux-i386/340_find_mbr_bin.sh +# which means when the format workflow option -b/--bios was used +# the matching USB_DEVICE_PARTED_LABEL=msdos must be set in etc/rear/local.conf +# otherwise things would go wrong in prep/USB/Linux-i386/340_find_mbr_bin.sh +USB_DEVICE_PARTED_LABEL=gpt # # The label that is set for the ReaR data partition via the format workflow. # That label must be used for settings like @@ -1079,6 +1085,21 @@ USB_DEVICE_BOOT_LABEL="REARBOOT" # For details see output/USB/Linux-i386/300_create_extlinux.sh USB_BIOS_BOOT_DEFAULT="" # +# Booloader used for the USB medium. +# At the moment only empty/unset and "grub" is supported. +# USB_BOOTLOADER="grub" uses GRUB2 as bootloader for USB with BIOS. GRUB Legacy is not supported. +# Default is using GRUB2 for EFI other then elilo, extlinux for ext, syslinux otherwise: +USB_BOOTLOADER= +# +# USB EFI booting can benefit with a better search string than the default: +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI +# as hardcoded in script output/USB/Linux-i386/100_create_efiboot.sh +# Only to be used by experts. An example of a different setup could be: +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI --hint hd0,msdos1" +# or +# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --file /EFI/BOOT/BOOTX64.efi" +GRUB2_SEARCH_ROOT_COMMAND="" +# # Resulting files that should be copied onto the USB stick: USB_FILES=() # @@ -1106,25 +1127,10 @@ USB_SUFFIX="" # and this setting is ignored when USB_SUFFIX is set (see above). USB_RETAIN_BACKUP_NR=2 # -# Booloader used for the USB medium. -# At the moment only empty/unset and "grub" is supported. -# USB_BOOTLOADER="grub" uses GRUB2 as bootloader for USB with BIOS. GRUB Legacy is not supported. -# Default is using GRUB2 for EFI other then elilo, extlinux for ext, syslinux otherwise: -USB_BOOTLOADER= -# # Variable will probably be filled automatically # if an USB device was manually mounted to avoid recursive backups: AUTOEXCLUDE_USB_PATH=() -# USB EFI booting can benefit with a better search string than the default: -# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI -# as hardcoded in script output/USB/Linux-i386/100_create_efiboot.sh -# Only to be used by experts. An example of a different setup could be: -# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --label REAR-EFI --hint hd0,msdos1" -# or -# GRUB2_SEARCH_ROOT_COMMAND="search --no-floppy --set=root --file /EFI/BOOT/BOOTX64.efi" -GRUB2_SEARCH_ROOT_COMMAND="" - ## # OUTPUT=RAWDISK stuff ## From b906fe5b03c2e08b697370682782a5d53db3a600 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 24 Jun 2022 14:51:16 +0200 Subject: [PATCH 150/215] Update 340_find_mbr_bin.sh In prep/USB/Linux-i386/340_find_mbr_bin.sh try to autodetect what the actual USB device partition table type is and use USB_DEVICE_PARTED_LABEL only as fallback when autodetection fails cf. https://github.com/rear/rear/pull/2829/files#r906006257 --- .../prep/USB/Linux-i386/340_find_mbr_bin.sh | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh b/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh index e113e38fb1..ffcba064ed 100644 --- a/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh +++ b/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh @@ -1,10 +1,46 @@ # The file mbr.bin is only added since syslinux 3.08 # The extlinux -i option is only added since syslinux 3.20 -local mbr_image_file +# Find out what the actual USB device partition table type is +# of the USB disk that is the parent device of the USB data partition +# that is the value of the USB_DEVICE variable. +# For example +# BACKUP_URL=usb:///dev/disk/by-label/REAR-000 +# leads to +# USB_DEVICE=/dev/disk/by-label/REAR-000 +# which is a symbolic link e.g. to /dev/sdb3 (on a hybrid UEFI and BIOS dual boot USB disk) +# so its parent device /dev/sdb is where we need to inspect the partition table type. +# See the code of the write_protection_ids() function in lib/write-protect-functions.sh +# how to get the parent device. +# See the output of +# # find usr/sbin/rear usr/share/rear -type f | xargs grep 'Partition Table' +# for code how to autodetect the partition table type via 'parted ... print'. +# In summary it goes like this example: +# # USB_DEVICE=/dev/disk/by-label/REAR-000 +# # usb_disk="$( lsblk -inpo PKNAME "$USB_DEVICE" 2>/dev/null | awk NF | head -n1 )" +# # echo $usb_disk +# /dev/sdb +# # usb_disk_label=$( parted -s $usb_disk print | grep -E "Partition Table|Disk label" | cut -d ":" -f "2" | tr -d " " ) +# # echo $usb_disk_label +# gpt +# see https://github.com/rear/rear/pull/2829/files#r906006257 -# Choose right MBR image file for right partition table type (issue #1153) -case "$USB_DEVICE_PARTED_LABEL" in +local usb_disk usb_disk_label mbr_image_file + +usb_disk="$( lsblk -inpo PKNAME "$USB_DEVICE" 2>/dev/null | awk NF | head -n1 )" +# Older Linux distributions do not contain lsblk (e.g. SLES10) +# and older lsblk versions do not support the output column PKNAME +# e.g. lsblk in util-linux 2.19.1 in SLES11 supports NAME and KNAME but not PKNAME +# see the code of the write_protection_ids() function in lib/write-protect-functions.sh +# so we use USB_DEVICE_PARTED_LABEL as fallback when the 'lsblk' automatism does not work: +if test -b "$usb_disk" ; then + usb_disk_label=$( parted -s $usb_disk print | grep -E "Partition Table|Disk label" | cut -d ":" -f "2" | tr -d " " ) +else + usb_disk_label=$USB_DEVICE_PARTED_LABEL +fi + +# Choose the right MBR image file for the partition table type (issue #1153) +case "$usb_disk_label" in (msdos) mbr_image_file="mbr.bin" ;; @@ -12,7 +48,7 @@ case "$USB_DEVICE_PARTED_LABEL" in mbr_image_file="gptmbr.bin" ;; (*) - Error "USB_DEVICE_PARTED_LABEL='$USB_DEVICE_PARTED_LABEL' (neither 'msdos' nor 'gpt')" + Error "Unsupported USB disk partition table type '$usb_disk_label' (neither 'msdos' nor 'gpt')" ;; esac From 08ccc37fd5760334af7cb4d73d01d20a2111b1d3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 24 Jun 2022 14:59:49 +0200 Subject: [PATCH 151/215] Update default.conf Adapt the USB_DEVICE_PARTED_LABEL explanation in default.conf regarding USB device partition type autodetection in prep/USB/Linux-i386/340_find_mbr_bin.sh --- usr/share/rear/conf/default.conf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 416d2c9fe5..ccf4eea0fa 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -980,10 +980,12 @@ USB_DEVICE= # so USB_DEVICE_PARTED_LABEL must be set to 'gpt' # see https://github.com/rear/rear/issues/2698 # so we set 'gpt' as default here because USB_DEVICE_PARTED_LABEL -# is needed in prep/USB/Linux-i386/340_find_mbr_bin.sh +# could be needed in prep/USB/Linux-i386/340_find_mbr_bin.sh +# which tries to autodetect what the actual USB device partition type is +# but it uses USB_DEVICE_PARTED_LABEL as fallback when autodetection fails # which means when the format workflow option -b/--bios was used -# the matching USB_DEVICE_PARTED_LABEL=msdos must be set in etc/rear/local.conf -# otherwise things would go wrong in prep/USB/Linux-i386/340_find_mbr_bin.sh +# the matching USB_DEVICE_PARTED_LABEL=msdos should be set in etc/rear/local.conf +# otherwise things could go wrong in prep/USB/Linux-i386/340_find_mbr_bin.sh USB_DEVICE_PARTED_LABEL=gpt # # The label that is set for the ReaR data partition via the format workflow. From 3bc6fc53e68e3db48f311cf631388d4b1edfd308 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 27 Jun 2022 10:00:17 +0200 Subject: [PATCH 152/215] Update 300_format_usb_disk.sh In format/USB/default/300_format_usb_disk.sh tell the user when his specified USB_DEVICE_PARTED_LABEL does not match what format workflow needs and overwrite USB_DEVICE_PARTED_LABEL with what format workflow needs --- .../rear/format/USB/default/300_format_usb_disk.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/format/USB/default/300_format_usb_disk.sh b/usr/share/rear/format/USB/default/300_format_usb_disk.sh index 33c6bf0115..a912119450 100644 --- a/usr/share/rear/format/USB/default/300_format_usb_disk.sh +++ b/usr/share/rear/format/USB/default/300_format_usb_disk.sh @@ -43,9 +43,16 @@ local current_partition_start_byte=$(( USB_PARTITION_ALIGN_BLOCK_SIZE * MiB_byte # to get hybrid boot supporting BIOS and UEFI from the same medium by default # see https://github.com/rear/rear/pull/2705 # so the ordering of the two settings below is crucial -# to ensure a GUID partition table is set up for hybrid boot: -is_true "$FORMAT_BIOS" && USB_DEVICE_PARTED_LABEL="msdos" -is_true "$FORMAT_EFI" && USB_DEVICE_PARTED_LABEL="gpt" +# to ensure a GUID partition table is set up for hybrid boot. +# Set default usb_disk_label="gpt" to be fail-safe if neither FORMAT_BIOS nor FORMAT_EFI is true: +local usb_disk_label="gpt" +is_true "$FORMAT_BIOS" && usb_disk_label="msdos" +is_true "$FORMAT_EFI" && usb_disk_label="gpt" +# Tell the user when his specified USB_DEVICE_PARTED_LABEL does not match what format workflow needs: +if test "$USB_DEVICE_PARTED_LABEL" && test "$usb_disk_label" != "$USB_DEVICE_PARTED_LABEL" ; then + LogPrintError "Overwriting USB_DEVICE_PARTED_LABEL with '$usb_disk_label' to match format workflow settings" +fi +USB_DEVICE_PARTED_LABEL="$usb_disk_label" LogPrint "Creating partition table of type $USB_DEVICE_PARTED_LABEL on $RAW_USB_DEVICE" if ! parted -s $RAW_USB_DEVICE mklabel $USB_DEVICE_PARTED_LABEL ; then Error "Failed to create $USB_DEVICE_PARTED_LABEL partition table on $RAW_USB_DEVICE" From 4e856da2f5e6c3cf5018923938b48010f453642e Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 27 Jun 2022 10:18:56 +0200 Subject: [PATCH 153/215] Update 340_find_mbr_bin.sh In prep/USB/Linux-i386/340_find_mbr_bin.sh tell the user when his specified USB_DEVICE_PARTED_LABEL does not match the actual USB disk partition type and overwrite USB_DEVICE_PARTED_LABEL with the USB disk partition type --- .../prep/USB/Linux-i386/340_find_mbr_bin.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh b/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh index ffcba064ed..74029b68ad 100644 --- a/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh +++ b/usr/share/rear/prep/USB/Linux-i386/340_find_mbr_bin.sh @@ -1,7 +1,7 @@ # The file mbr.bin is only added since syslinux 3.08 # The extlinux -i option is only added since syslinux 3.20 -# Find out what the actual USB device partition table type is +# Find out what the actual USB disk partition table type is # of the USB disk that is the parent device of the USB data partition # that is the value of the USB_DEVICE variable. # For example @@ -32,15 +32,21 @@ usb_disk="$( lsblk -inpo PKNAME "$USB_DEVICE" 2>/dev/null | awk NF | head -n1 )" # and older lsblk versions do not support the output column PKNAME # e.g. lsblk in util-linux 2.19.1 in SLES11 supports NAME and KNAME but not PKNAME # see the code of the write_protection_ids() function in lib/write-protect-functions.sh -# so we use USB_DEVICE_PARTED_LABEL as fallback when the 'lsblk' automatism does not work: +# so we use USB_DEVICE_PARTED_LABEL as fallback when the 'lsblk' automatism does not work +# and also when 'parted' does not show "msdos" or "gpt": if test -b "$usb_disk" ; then usb_disk_label=$( parted -s $usb_disk print | grep -E "Partition Table|Disk label" | cut -d ":" -f "2" | tr -d " " ) -else - usb_disk_label=$USB_DEVICE_PARTED_LABEL + if test "$usb_disk_label" = "msdos" || test "$usb_disk_label" = "gpt" ; then + # Tell the user when his specified USB_DEVICE_PARTED_LABEL does not match the actual USB disk partition type: + if test "$USB_DEVICE_PARTED_LABEL" && test "$usb_disk_label" != "$USB_DEVICE_PARTED_LABEL" ; then + LogPrintError "Overwriting USB_DEVICE_PARTED_LABEL with '$usb_disk_label' to match USB disk partition type" + fi + USB_DEVICE_PARTED_LABEL="$usb_disk_label" + fi fi # Choose the right MBR image file for the partition table type (issue #1153) -case "$usb_disk_label" in +case "$USB_DEVICE_PARTED_LABEL" in (msdos) mbr_image_file="mbr.bin" ;; @@ -48,7 +54,7 @@ case "$usb_disk_label" in mbr_image_file="gptmbr.bin" ;; (*) - Error "Unsupported USB disk partition table type '$usb_disk_label' (neither 'msdos' nor 'gpt')" + Error "Unsupported USB disk partition table type '$USB_DEVICE_PARTED_LABEL' (neither 'msdos' nor 'gpt')" ;; esac From 4d65db25f2eea5d691c749e9aaf8c76968183cbb Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 27 Jun 2022 10:36:29 +0200 Subject: [PATCH 154/215] Update default.conf In default.conf better description of USB_DEVICE_PARTED_LABEL and no longer set a default value because that is now autodetected --- usr/share/rear/conf/default.conf | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index ccf4eea0fa..a417421eca 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -977,16 +977,13 @@ USB_DEVICE= # USB_DEVICE_PARTED_LABEL is set to 'msdos' or 'gpt' accordingly. # When no format workflow option -b/--bios or -e/--efi was specified # it means hybrid boot supporting BIOS and UEFI by default -# so USB_DEVICE_PARTED_LABEL must be set to 'gpt' +# and then USB_DEVICE_PARTED_LABEL is set to 'gpt' # see https://github.com/rear/rear/issues/2698 -# so we set 'gpt' as default here because USB_DEVICE_PARTED_LABEL -# could be needed in prep/USB/Linux-i386/340_find_mbr_bin.sh -# which tries to autodetect what the actual USB device partition type is -# but it uses USB_DEVICE_PARTED_LABEL as fallback when autodetection fails -# which means when the format workflow option -b/--bios was used -# the matching USB_DEVICE_PARTED_LABEL=msdos should be set in etc/rear/local.conf -# otherwise things could go wrong in prep/USB/Linux-i386/340_find_mbr_bin.sh -USB_DEVICE_PARTED_LABEL=gpt +# For "rear mkrescue/mkbackup" USB_DEVICE_PARTED_LABEL +# is set in prep/USB/Linux-i386/340_find_mbr_bin.sh +# which tries to autodetect what the USB disk partition type is +# but uses a specified USB_DEVICE_PARTED_LABEL if autodetection fails. +USB_DEVICE_PARTED_LABEL= # # The label that is set for the ReaR data partition via the format workflow. # That label must be used for settings like From 0ad6be092a033a154287db69829c66727e55ed71 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 27 Jun 2022 14:14:12 +0200 Subject: [PATCH 155/215] Update 100_create_efiboot.sh In output/USB/Linux-i386/100_create_efiboot.sh do "the right thing" and show the right DebugPrint message depending on whether or not there is a ESP partition labeled 'REAR-EFI' and whether or not USING_UEFI_BOOTLOADER is 'true' --- .../USB/Linux-i386/100_create_efiboot.sh | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh b/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh index 7015ff44ad..1f6ca06973 100644 --- a/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh +++ b/usr/share/rear/output/USB/Linux-i386/100_create_efiboot.sh @@ -3,17 +3,30 @@ # because it sets a hardcoded label REAR-EFI in format/USB/default/300_format_usb_disk.sh # for the VFAT EFI filesystem that is needed here. -is_true $USING_UEFI_BOOTLOADER || return 0 - -local uefi_bootloader_basename=$( basename "$UEFI_BOOTLOADER" ) +# Do "the right thing" depending on +# whether or not there is a partition labeled 'REAR-EFI' +# and whether or not USING_UEFI_BOOTLOADER is 'true': local efi_label="REAR-EFI" local efi_part="/dev/disk/by-label/$efi_label" - +if ! test -b "$efi_part" ; then + if ! is_true $USING_UEFI_BOOTLOADER ; then + # There is no partition labeled 'REAR-EFI' and USING_UEFI_BOOTLOADER is not 'true': + DebugPrint "No EFI boot (no EFI partition '$efi_part' and USING_UEFI_BOOTLOADER is not 'true')" + return 0 + fi + # There is no partition labeled 'REAR-EFI' but USING_UEFI_BOOTLOADER is 'true': + Error "USING_UEFI_BOOTLOADER is 'true' but no EFI partition '$efi_part' block device (did you prepare with 'rear format'?)" +fi +# There is a is a partition labeled 'REAR-EFI': +if ! is_true $USING_UEFI_BOOTLOADER ; then + # There is a is a partition labeled 'REAR-EFI' and USING_UEFI_BOOTLOADER is not 'true': + DebugPrint "Skip configuring EFI partition '$efi_part' for EFI boot (USING_UEFI_BOOTLOADER is not 'true')" + return 0 +fi +# There is a is a partition labeled 'REAR-EFI' and USING_UEFI_BOOTLOADER is 'true': +local uefi_bootloader_basename=$( basename "$UEFI_BOOTLOADER" ) DebugPrint "Configuring EFI partition '$efi_part' for EFI boot with '$uefi_bootloader_basename'" -# Fail if EFI partition is not present -test -b "$efi_part" || Error "EFI partition '$efi_part' is no block device (did you use 'rear format -- --efi ...' for correct format?)" - # $BUILD_DIR is not present at this stage so TMPDIR (by default /var/tmp see default.conf) will be used instead. # Slackware version of mktemp requires 6 Xs in template and # plain 'mktemp' uses XXXXXXXXXX by default (at least on SLES11 and openSUSE Leap 15.0) From f2dfda140373625fc230d81db2459bc4e101baf9 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 28 Jun 2022 09:49:01 +0200 Subject: [PATCH 156/215] Update README.wipe_disks Typo fix in layout/recreate/default/README.wipe_disks 'To wipe to wipe LVM metadata' -> 'To wipe LVM metadata' --- usr/share/rear/layout/recreate/default/README.wipe_disks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/recreate/default/README.wipe_disks b/usr/share/rear/layout/recreate/default/README.wipe_disks index 4aa1594ee9..05b1ff88d1 100644 --- a/usr/share/rear/layout/recreate/default/README.wipe_disks +++ b/usr/share/rear/layout/recreate/default/README.wipe_disks @@ -627,7 +627,7 @@ RAID plus LVM plus LUKS To wipe RAID Superblocks it is sufficient to wipe 133 KiB at the beginning and at the end of the device. -To wipe to wipe LVM metadata is should be sufficient to wipe 4 MiB at the beginning and at the end of the device. +To wipe LVM metadata is should be sufficient to wipe 4 MiB at the beginning and at the end of the device. To wipe LUKS headers is should be sufficient to wipe 8 MiB at the beginning of the device. From 121bfc1511efc9a842c787369f4fbfae482befe8 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 28 Jun 2022 14:41:46 +0200 Subject: [PATCH 157/215] Updated doc/rear-release-notes.txt --- doc/rear-release-notes.txt | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index be9cdeec3b..5f742c758c 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -186,7 +186,7 @@ must re-validate that your disaster recovery procedure still works for you. The references pointing to fix #nr or issue #nr refer to our GitHub issues tracker https://github.com/rear/rear/issues -Version 2.7 (June 2022) +Version 2.7 (July 2022) Abstract @@ -285,9 +285,13 @@ New features, bigger enhancements, and possibly backward incompatible changes: - Now "rear format" has in addition to the '-efi' switch a '-bios' switch. If none is given (i.e. by default) it will now do hybrid formatting with a - BIOS boot partition (on GPT) and an EFI system partition. This is a - starting point for implementing OUTPUT=USB support for UEFI and BIOS dual - boot from the same medium. See https://github.com/rear/rear/issues/2698 + BIOS boot partition (on GPT) and an EFI system partition. Accordingly the + USB_DEVICE_PARTED_LABEL default is no longer "msdos" (see the updated + description in /usr/share/rear/conf/default.conf). This is a starting point + for implementing OUTPUT=USB support for UEFI and BIOS dual boot from the + same medium. See https://github.com/rear/rear/issues/2698 + and https://github.com/rear/rear/issues/2818 + and https://github.com/rear/rear/pull/2829 - Overhauled serial console support code. A serial console of the ReaR recovery system can now be specified separately for the kernel and the @@ -313,6 +317,25 @@ New features, bigger enhancements, and possibly backward incompatible changes: Details (mostly in chronological order - newest topmost): +- In usr/share/rear/lib/format-workflow.sh do actually recognise -b/--bios + options, see https://github.com/rear/rear/pull/2828 + +- In layout/prepare/GNU/Linux/110_include_lvm_code.sh use a fail-safe + 'yes' pipe for "lvm lvcreate" to pipe as many 'y' as asked for into + "lvm lvcreate" see https://github.com/rear/rear/issues/513 + and https://github.com/rear/rear/issues/2820 + +- Avoid creating a /bin/vim symlink to vi: The symlink is more confusing + than helpful: If one has both vi and vim, and they are different + (vi usually has less features), one expects to get the more featureful + version by executing "vim", but one gets "vi". More importantly, in + recent Fedora and RHEL, "vi" is a shell script that executes "vim" + if found, so linking "vim" to "vi" leads to an infinite loop. + See https://github.com/rear/rear/pull/2822 + +- RPM spec: update build requirement for Fedora to unblock Packit build and + run make validate after build, see https://github.com/rear/rear/pull/2816 + - In finalize/Linux-i386/660_install_grub2.sh explain that it is also used as fallback to install the nowadays most often used bootloader GRUB2 unless the BOOTLOADER variable tells to install another bootloader (other From 46b2c82c56df06a2ee56de4e7722ed057d415021 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Fri, 17 Jun 2022 14:10:18 +0200 Subject: [PATCH 158/215] Refactor rsync:// URL support The code to parse rsync:// URLs was BACKUP_URL specific. If one had specified BACKUP=RSYNC and an OUTPUT_URL different from BACKUP_URL, the OUTPUT_URL was ignored and the output files went to BACKUP_URL. Fix by introducing generic functions for rsync URL parsing and use them for both BACKUP_URL and OUTPUT_URL, as appropriate. Replace all uses of global RSYNC_* variables derived from BACKUP_URL by those functions. --- .../RSYNC/GNU/Linux/610_start_selinux.sh | 16 +- .../RSYNC/GNU/Linux/620_force_autorelabel.sh | 16 +- .../RSYNC/default/450_calculate_req_space.sh | 16 +- .../RSYNC/default/500_make_rsync_backup.sh | 21 ++- .../RSYNC/default/700_copy_backup_log.sh | 13 +- usr/share/rear/lib/rsync-functions.sh | 163 ++++++++++++++++++ .../RSYNC/default/200_make_prefix_dir.sh | 15 +- .../RSYNC/default/900_copy_result_files.sh | 13 +- .../RSYNC/GNU/Linux/200_selinux_in_use.sh | 4 +- .../prep/RSYNC/default/100_check_rsync.sh | 79 ++------- .../150_check_rsync_protocol_version.sh | 25 +-- .../RSYNC/default/400_restore_rsync_backup.sh | 14 +- .../550_check_remote_backup_archive.sh | 10 +- 13 files changed, 270 insertions(+), 135 deletions(-) create mode 100644 usr/share/rear/lib/rsync-functions.sh diff --git a/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh b/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh index 1692ba4cb5..dd198ede48 100644 --- a/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh +++ b/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh @@ -6,29 +6,29 @@ local backup_prog_rc touch "${TMP_DIR}/selinux.autorelabel" cat $TMP_DIR/selinux.mode > $SELINUX_ENFORCE Log "Restored original SELinux mode" - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) # for some reason rsync changes the mode of backup after each run to 666 # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - ssh $RSYNC_USER@$RSYNC_HOST "chmod $v 755 ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" 2>/dev/null + ssh $(rsync_remote_ssh "$BACKUP_URL") "chmod $v 755 $(rsync_path_full "$BACKUP_URL")/backup" 2>/dev/null $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" \ - "$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" 2>/dev/null + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" 2>/dev/null backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; (rsync) $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; diff --git a/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh b/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh index 9a17d6bbfe..de57d57175 100644 --- a/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh +++ b/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh @@ -4,29 +4,29 @@ local backup_prog_rc > "${TMP_DIR}/selinux.autorelabel" - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) # for some reason rsync changes the mode of backup after each run to 666 # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - ssh $RSYNC_USER@$RSYNC_HOST "chmod $v 755 ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" 2>/dev/null + ssh $(rsync_remote_ssh "$BACKUP_URL") "chmod $v 755 $(rsync_path_full "$BACKUP_URL")/backup" 2>/dev/null $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" \ - "$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" 2>/dev/null + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" 2>/dev/null backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; (rsync) $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; diff --git a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh index 4eccf25313..dea270fd1e 100644 --- a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh +++ b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh @@ -1,6 +1,12 @@ # here we will calculate the space required to hold the backup archive on the remote rsync system # This file is part of Relax-and-Recover, licensed under the GNU General # Public License. Refer to the included COPYING for full text of license. +local proto host path + +proto="$(rsync_proto "$BACKUP_URL")" +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" + _local_size=0 _remote_size=0 while read -r ; do @@ -14,17 +20,17 @@ done < $TMP_DIR/backup-include.txt LogPrint "Estimated size of local file systems is $(( _local_size / 1024 )) MB" # Commenting out next block according decision of issue #2760 -#case $RSYNC_PROTO in +#case $proto in # (ssh) -# LogPrint "Calculating size of $RSYNC_HOST:$RSYNC_PATH" -# ssh -l $RSYNC_USER $RSYNC_HOST "df -P $RSYNC_PATH" >$TMP_DIR/rs_size -# StopIfError "Failed to determine size of $RSYNC_PATH" +# LogPrint "Calculating size of ${host}:${path}" +# ssh $(rsync_remote_ssh "$BACKUP_URL") "df -P ${path}" >$TMP_DIR/rs_size +# StopIfError "Failed to determine size of ${path}" # _div=1 # 1024-blocks # grep -q "512-blocks" $TMP_DIR/rs_size && _div=2 # HPUX: divide with 2 to get kB size # _remote_size=$( tail -n 1 $TMP_DIR/rs_size | awk '{print $2}' ) # _remote_size=$(( _remote_size / _div )) # [[ $_remote_size -gt $_local_size ]] -# StopIfError "Not enough disk space available on $RSYNC_HOST:$RSYNC_PATH ($_remote_size < $_local_size)" +# StopIfError "Not enough disk space available on ${host}:${path} ($_remote_size < $_local_size)" # ;; # (rsync) # # TODO: how can we calculate the free size on remote system via rsync protocol?? diff --git a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh index 750a04cae7..e8a2b87c2b 100644 --- a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh +++ b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh @@ -5,6 +5,11 @@ local backup_prog_rc local backup_log_message +local host path + +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" + Log "Include list:" while read -r ; do Log " $REPLY" @@ -14,7 +19,7 @@ while read -r ; do Log " $REPLY" done < $TMP_DIR/backup-exclude.txt -LogPrint "Creating $BACKUP_PROG backup on '${RSYNC_HOST}:${RSYNC_PATH}'" +LogPrint "Creating $BACKUP_PROG backup on '${host}:${path}'" ProgressStart "Running backup operation" ( @@ -23,17 +28,17 @@ ProgressStart "Running backup operation" (rsync) BACKUP_RSYNC_OPTIONS+=( --one-file-system --delete --exclude-from=$TMP_DIR/backup-exclude.txt --delete-excluded ) - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) - Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) "$(rsync_remote_full "$BACKUP_URL")/backup" $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) \ - "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + "$(rsync_remote_full "$BACKUP_URL")/backup" ;; (rsync) $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + "$(rsync_remote_full "$BACKUP_URL")/backup" ;; esac @@ -57,11 +62,11 @@ get_size() { } check_remote_df() { - echo $(ssh ${RSYNC_USER}@${RSYNC_HOST} df -P ${RSYNC_PATH} 2>/dev/null | tail -1 | awk '{print $5}' | sed -e 's/%//') + echo $(ssh $(rsync_remote_ssh "$BACKUP_URL") df -P ${path} 2>/dev/null | tail -1 | awk '{print $5}' | sed -e 's/%//') } check_remote_du() { - x=$(ssh ${RSYNC_USER}@${RSYNC_HOST} du -sb ${RSYNC_PATH}/${RSYNC_PREFIX}/backup 2>/dev/null | awk '{print $1}') + x=$(ssh $(rsync_remote_ssh "$BACKUP_URL") du -sb $(rsync_path_full "$BACKUP_URL")/backup 2>/dev/null | awk '{print $1}') [[ -z "${x}" ]] && x=0 echo $x } @@ -81,7 +86,7 @@ case "$(basename $BACKUP_PROG)" in case $i in 300) - [[ $(check_remote_df) -eq 100 ]] && Error "Disk is full on system ${RSYNC_HOST}" + [[ $(check_remote_df) -eq 100 ]] && Error "Disk is full on system ${host}" ;; 15|30|45|60|75|90|105|120|135|150|165|180|195|210|225|240|255|270|285) diff --git a/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh b/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh index b90d459b67..76b9f971d6 100644 --- a/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh +++ b/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh @@ -1,26 +1,27 @@ # copy the backup.log & rear.log file to remote destination with timestamp added -local timestamp +local timestamp proto timestamp=$( date +%Y%m%d.%H%M ) +proto="$(rsync_proto "$BACKUP_URL")" # compress the log file first gzip "$TMP_DIR/$BACKUP_PROG_ARCHIVE.log" || Error "Failed to 'gzip $TMP_DIR/$BACKUP_PROG_ARCHIVE.log'" -case $RSYNC_PROTO in +case $proto in (ssh) # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 $BACKUP_PROG -a "${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log.gz" \ - "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" 2>/dev/null + "$(rsync_remote_full "$BACKUP_URL")/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" 2>/dev/null - $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/rear-${timestamp}.log" 2>/dev/null + $BACKUP_PROG -a "$RUNTIME_LOGFILE" "$(rsync_remote_full "$BACKUP_URL")/rear-${timestamp}.log" 2>/dev/null ;; (rsync) $BACKUP_PROG -a "${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log.gz" "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" + "$(rsync_remote_full "$BACKUP_URL")/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" - $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}//rear-${timestamp}.log" + $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$BACKUP_URL")//rear-${timestamp}.log" ;; esac diff --git a/usr/share/rear/lib/rsync-functions.sh b/usr/share/rear/lib/rsync-functions.sh new file mode 100644 index 0000000000..cf98b90dc4 --- /dev/null +++ b/usr/share/rear/lib/rsync-functions.sh @@ -0,0 +1,163 @@ +# Functions for manipulation of rsync URLs (both OUTPUT_URL and BACKUP_URL) + +#### OLD STYLE: +# BACKUP_URL=[USER@]HOST:PATH # using ssh (no rsh) +# +# with rsync protocol PATH is a MODULE name defined in remote /etc/rsyncd.conf file +# BACKUP_URL=[USER@]HOST::PATH # using rsync +# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using rsync (is not compatible with new style!!!) + +#### NEW STYLE: +# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using ssh +# BACKUP_URL=rsync://[USER@]HOST[:PORT]::/PATH # using rsync + +function rsync_validate () { + local url="$1" + + if [[ "$(url_scheme "$url")" != "rsync" ]]; then # url_scheme still recognizes old style + Error "Non-rsync URL $url !" + fi +} + +# Determine whether the URL specifies the use of the rsync protocol (rsyncd) or ssh +function rsync_proto () { + local url="$1" + + rsync_validate "$url" + if egrep -q '(::)' <<< $url ; then # new style '::' means rsync protocol + echo rsync + else + echo ssh + fi +} + +# Functions to parse the URL into its components: +# USER, HOST, PORT, PATH + +function rsync_user () { + local url="$1" + local host + + host=$(url_host "$url") + + if grep -q '@' <<< $host ; then + echo "${host%%@*}" # grab user name + else + echo root + fi +} + +function rsync_host () { + local url="$1" + local host + local path + + host=$(url_host "$url") + path=$(url_path "$url") + # remove USER@ if present + local tmp2="${host#*@}" + + case "$(rsync_proto "$url")" in + (rsync) + # tmp2=witsbebelnx02::backup or tmp2=witsbebelnx02:: + echo "${tmp2%%::*}" + ;; + (ssh) + # tmp2=host or tmp2=host: + echo "${tmp2%%:*}" + ;; + esac +} + +function rsync_path () { + local url="$1" + local host + local path + + host=$(url_host "$url") + path=$(url_path "$url") + local tmp2="${host#*@}" + + case "$(rsync_proto "$url")" in + + (rsync) + # path=/gdhaese1@witsbebelnx02::backup or path=/backup + if grep -q '::' <<< $path ; then + echo "${path##*::}" + else + # XXX what if path=/backup/sub/directory ? Should we remove + # longest prefix? + echo "${path##*/}" + fi + ;; + (ssh) + echo "$path" + ;; + + esac +} + +function rsync_port () { + # XXX changing port not implemented yet + echo 873 +} + +# Full path to the destination directory on the remote server, +# includes RSYNC_PREFIX. RSYNC_PREFIX is not given by the URL, +# it is a global parameter (by default derived from hostname). +function rsync_path_full () { + local url="$1" + + echo "$(rsync_path "$url")/${RSYNC_PREFIX}" +} + +# Argument for the ssh command to log in to the remote host ("user@host") +function rsync_remote_ssh () { + local url="$1" + + local user host + + user="$(rsync_user "$url")" + host="$(rsync_host "$url")" + + echo "${user}@${host}" +} + +# Argument for the rsync command to reach the remote host, without path. +function rsync_remote_base () { + local url="$1" + + local user host port + + user="$(rsync_user "$url")" + host="$(rsync_host "$url")" + port="$(rsync_port "$url")" + + case "$(rsync_proto "$url")" in + + (rsync) + echo "rsync://${user}@${host}:${port}/" + ;; + (ssh) + echo "$(rsync_remote_ssh "$url"):" + ;; + + esac +} + +# Complete argument to rsync to reach the remote location identified by URL, +# but without the added RSYNC_PREFIX. +# This essentially converts our rsync:// URLs into a form accepted by the rsync command. +function rsync_remote () { + local url="$1" + + echo "$(rsync_remote_base "$url")$(rsync_path "$url")" +} + +# Complete argument to rsync including even RSYNC_PREFIX. +# Determined from the URL and RSYNC_PREFIX. +function rsync_remote_full () { + local url="$1" + + echo "$(rsync_remote_base "$url")$(rsync_path_full "$url")" +} diff --git a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh index 519febf583..82cb1c55e2 100644 --- a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh +++ b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh @@ -1,20 +1,25 @@ # Create RSYNC_PREFIX under the local TMP_DIR and also on remote rsync server # RSYNC_PREFIX=$HOSTNAME as set in default.conf +local proto host + +proto="$(rsync_proto "$OUTPUT_URL")" +host="$(rsync_host "$OUTPUT_URL")" + # create temporary local work-spaces to collect files (we already make the remote backup dir with the correct mode!!) mkdir -p $v -m0750 "${TMP_DIR}/rsync/${RSYNC_PREFIX}" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}'" mkdir -p $v -m0755 "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup'" -case $RSYNC_PROTO in +case $proto in (ssh) - $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}" >/dev/null 2>&1 \ - || Error "Could not create '${RSYNC_PATH}/${RSYNC_PREFIX}' on remote ${RSYNC_HOST}" + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "$(rsync_remote "$OUTPUT_URL")" >/dev/null 2>&1 \ + || Error "Could not create '$(rsync_path_full "$OUTPUT_URL")' on remote ${host}" ;; (rsync) - $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/" >/dev/null \ - || Error "Could not create '${RSYNC_PATH}/${RSYNC_PREFIX}' on remote ${RSYNC_HOST}" + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote "$OUTPUT_URL")/" >/dev/null \ + || Error "Could not create '$(rsync_path_full "$OUTPUT_URL")' on remote ${host}" ;; esac diff --git a/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh b/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh index 96b62da116..517587eea2 100644 --- a/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh +++ b/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh @@ -1,6 +1,9 @@ # # copy resulting files to remote network (backup) location +local proto +proto="$(rsync_proto "$OUTPUT_URL")" + LogPrint "Copying resulting files to $OUTPUT_URL location" # if called as mkbackuponly then we just don't have any result files. @@ -19,21 +22,21 @@ cp $v $(get_template "RESULT_usage_$OUTPUT.txt") "${TMP_DIR}/rsync/${RSYNC_PREFI cat "$RUNTIME_LOGFILE" >"${TMP_DIR}/rsync/${RSYNC_PREFIX}/rear.log" \ || Error "Could not copy $RUNTIME_LOGFILE to local rsync location" -case $RSYNC_PROTO in +case $proto in (ssh) - Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ ${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/" + Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ $(rsync_remote_full "$OUTPUT_URL")/" # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/" 2>/dev/null \ + $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "$(rsync_remote_full "$OUTPUT_URL")/" 2>/dev/null \ || Error "Could not copy '${RESULT_FILES[*]}' to $OUTPUT_URL location" ;; (rsync) - Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ ${BACKUP_RSYNC_OPTIONS[*]} ${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/" + Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ ${BACKUP_RSYNC_OPTIONS[*]} $(rsync_remote_full "$OUTPUT_URL")/" # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/" 2>/dev/null \ + $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$OUTPUT_URL")/" 2>/dev/null \ || Error "Could not copy '${RESULT_FILES[*]}' to $OUTPUT_URL location" ;; diff --git a/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh b/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh index eb7df29e41..845000399f 100644 --- a/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh +++ b/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh @@ -25,8 +25,10 @@ case $(basename $BACKUP_PROG) in (rsync) if grep -q "no xattrs" "$TMP_DIR/rsync_protocol"; then + local host + host="$(rsync_host "$BACKUP_URL")" # no xattrs compiled in remote rsync, so saving SELinux attributes are not possible - Log "WARNING: --xattrs not possible on system ($RSYNC_HOST) (no xattrs compiled in rsync)" + Log "WARNING: --xattrs not possible on system ($host) (no xattrs compiled in rsync)" # $TMP_DIR/selinux.mode is a trigger during backup to disable SELinux cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode RSYNC_SELINUX= # internal variable used in recover mode (empty means disable SELinux) diff --git a/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh b/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh index c964a14813..448a1b1a40 100644 --- a/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh +++ b/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh @@ -3,97 +3,40 @@ # This file is part of Relax-and-Recover, licensed under the GNU General # Public License. Refer to the included COPYING for full text of license. -#### OLD STYLE: -# BACKUP_URL=[USER@]HOST:PATH # using ssh (no rsh) -# -# with rsync protocol PATH is a MODULE name defined in remote /etc/rsyncd.conf file -# BACKUP_URL=[USER@]HOST::PATH # using rsync -# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using rsync (is not compatible with new style!!!) - -#### NEW STYLE: -# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using ssh -# BACKUP_URL=rsync://[USER@]HOST[:PORT]::/PATH # using rsync - if test -z "$BACKUP_URL" ; then Error "Missing BACKUP_URL=rsync://[USER@]HOST[:PORT][::]/PATH !" fi -local host=$(url_host $BACKUP_URL) local scheme=$(url_scheme $BACKUP_URL) # url_scheme still recognizes old style -local path=$(url_path $BACKUP_URL) if [[ "$scheme" != "rsync" ]]; then Error "Missing BACKUP_URL=rsync://[USER@]HOST[:PORT][::]/PATH !" fi -RSYNC_PROTO= # ssh or rsync -RSYNC_USER= -RSYNC_HOST= -RSYNC_PORT=873 # default port (of rsync server) -RSYNC_PATH= - - -if egrep -q '(::)' <<< $BACKUP_URL ; then # new style '::' means rsync protocol - RSYNC_PROTO=rsync -else - RSYNC_PROTO=ssh -fi - -if grep -q '@' <<< $host ; then - RSYNC_USER="${host%%@*}" # grab user name -else - RSYNC_USER=root -fi - -# remove USER@ if present (we don't need it anymore) -local tmp2="${host#*@}" - -case "$RSYNC_PROTO" in - - (rsync) - # tmp2=witsbebelnx02::backup or tmp2=witsbebelnx02:: - RSYNC_HOST="${tmp2%%::*}" - # path=/gdhaese1@witsbebelnx02::backup or path=/backup - if grep -q '::' <<< $path ; then - RSYNC_PATH="${path##*::}" - else - RSYNC_PATH="${path##*/}" - fi - ;; - (ssh) - # tmp2=host or tmp2=host: - RSYNC_HOST="${tmp2%%:*}" - RSYNC_PATH=$path - ;; - -esac - -#echo RSYNC_PROTO=$RSYNC_PROTO -#echo RSYNC_USER=$RSYNC_USER -#echo RSYNC_HOST=$RSYNC_HOST -#echo RSYNC_PORT=$RSYNC_PORT -#echo RSYNC_PATH=$RSYNC_PATH +local host proto +host="$(rsync_host "$BACKUP_URL")" +proto="$(rsync_proto "$BACKUP_URL")" # check if host is reachable if test "$PING" ; then - ping -c 2 "$RSYNC_HOST" >/dev/null || Error "Backup host [$RSYNC_HOST] not reachable." + ping -c 2 "$host" >/dev/null || Error "Backup host [$host] not reachable." else Log "Skipping ping test" fi # check protocol connectivity -case "$RSYNC_PROTO" in +case "$proto" in (rsync) - Log "Test: $BACKUP_PROG ${BACKUP_RSYNC_OPTIONS[*]} ${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/" - $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" ${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/ >/dev/null \ - || Error "Rsync daemon not running on $RSYNC_HOST" + Log "Test: $BACKUP_PROG ${BACKUP_RSYNC_OPTIONS[*]} $(rsync_remote_base "$BACKUP_URL")" + $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(rsync_remote_base "$BACKUP_URL") >/dev/null \ + || Error "Rsync daemon not running on $host" ;; (ssh) - Log "Test: ssh ${RSYNC_USER}@${RSYNC_HOST} /bin/true" - ssh ${RSYNC_USER}@${RSYNC_HOST} /bin/true >/dev/null 2>&1 \ - || Error "Secure shell connection not setup properly [$RSYNC_USER@$RSYNC_HOST]" + Log "Test: ssh $(rsync_remote_ssh "$BACKUP_URL") /bin/true" + ssh $(rsync_remote_ssh "$BACKUP_URL") /bin/true >/dev/null 2>&1 \ + || Error "Secure shell connection not setup properly [$(rsync_remote_ssh "$BACKUP_URL")]" ;; esac diff --git a/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh b/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh index e9103531f9..becf35a0d8 100644 --- a/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh +++ b/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh @@ -3,15 +3,18 @@ # Public License. Refer to the included COPYING for full text of license. # try to grab the rsync protocol version of rsync on the remote server -local remote_mountpoint +local remote_mountpoint host path proto +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" +proto="$(rsync_proto "$BACKUP_URL")" if [ -z "$RSYNC_PROTOCOL_VERSION" ]; then - case $RSYNC_PROTO in + case $proto in (ssh) - ssh ${RSYNC_USER}@${RSYNC_HOST} rsync --version >"$TMP_DIR/rsync_protocol" 2>&1 \ - || Error "Secure shell connection not setup properly [$RSYNC_USER@$RSYNC_HOST]" + ssh $(rsync_remote_ssh "$BACKUP_URL") rsync --version >"$TMP_DIR/rsync_protocol" 2>&1 \ + || Error "Secure shell connection not setup properly [$(rsync_remote_ssh "$BACKUP_URL")]" if grep -q "protocol version" "$TMP_DIR/rsync_protocol" ; then RSYNC_PROTOCOL_VERSION=$(grep 'protocol version' "$TMP_DIR/rsync_protocol" | awk '{print $6}') else @@ -24,29 +27,29 @@ if [ -z "$RSYNC_PROTOCOL_VERSION" ]; then RSYNC_PROTOCOL_VERSION=29 # being conservative (old rsync) ;; esac - Log "Remote rsync system ($RSYNC_HOST) uses rsync protocol version $RSYNC_PROTOCOL_VERSION" + Log "Remote rsync system ($host) uses rsync protocol version $RSYNC_PROTOCOL_VERSION" else - Log "Remote rsync system ($RSYNC_HOST) uses rsync protocol version $RSYNC_PROTOCOL_VERSION (overruled by user)" + Log "Remote rsync system ($host) uses rsync protocol version $RSYNC_PROTOCOL_VERSION (overruled by user)" fi -if [ "${RSYNC_USER}" != "root" -a $RSYNC_PROTO = "ssh" ]; then +if [ "$(rsync_user "$BACKUP_URL")" != "root" -a $proto = "ssh" ]; then if [ $RSYNC_PROTOCOL_VERSION -gt 29 ]; then if grep -q "no xattrs" "$TMP_DIR/rsync_protocol"; then # no xattrs available in remote rsync, so --fake-super is not possible - Error "rsync --fake-super not possible on system ($RSYNC_HOST) (no xattrs compiled in rsync)" + Error "rsync --fake-super not possible on system ($host) (no xattrs compiled in rsync)" else # when using --fake-super we must have user_xattr mount options on the remote mntpt - remote_mountpoint=$(ssh ${RSYNC_USER}@${RSYNC_HOST} 'cd ${RSYNC_PATH}; df -P .' 2>/dev/null | tail -1 | awk '{print $6}') - ssh ${RSYNC_USER}@${RSYNC_HOST} "cd ${RSYNC_PATH} && touch .is_xattr_supported && setfattr -n user.comment -v 'File created by ReaR to test if this filesystems supports extended attributes.' .is_xattr_supported && getfattr -n user.comment .is_xattr_supported 1>/dev/null; find .is_xattr_supported -empty -delete" \ + remote_mountpoint=$(ssh $(rsync_remote_ssh "$BACKUP_URL") 'cd ${path}; df -P .' 2>/dev/null | tail -1 | awk '{print $6}') + ssh $(rsync_remote_ssh "$BACKUP_URL") "cd ${path} && touch .is_xattr_supported && setfattr -n user.comment -v 'File created by ReaR to test if this filesystems supports extended attributes.' .is_xattr_supported && getfattr -n user.comment .is_xattr_supported 1>/dev/null; find .is_xattr_supported -empty -delete" \ || Error "Remote file system $remote_mountpoint does not have user_xattr mount option set!" #BACKUP_RSYNC_OPTIONS+=( --xattrs --rsync-path="rsync --fake-super" ) # see issue #366 for explanation of removing --xattrs BACKUP_RSYNC_OPTIONS+=( --rsync-path="rsync --fake-super" ) fi else - Error "rsync --fake-super not possible on system ($RSYNC_HOST) (please upgrade rsync to 3.x)" + Error "rsync --fake-super not possible on system ($host) (please upgrade rsync to 3.x)" fi fi diff --git a/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh b/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh index 993088becc..0fa085879b 100644 --- a/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh +++ b/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh @@ -7,7 +7,11 @@ get_size() { local backup_prog_rc local restore_log_message -LogPrint "Restoring $BACKUP_PROG backup from '${RSYNC_HOST}:${RSYNC_PATH}'" +local host path +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" + +LogPrint "Restoring $BACKUP_PROG backup from '${host}:${path}'" ProgressStart "Restore operation" ( @@ -15,18 +19,18 @@ ProgressStart "Restore operation" (rsync) - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) - Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup"/ $TARGET_FS_ROOT/ + Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$BACKUP_URL")/backup"/ $TARGET_FS_ROOT/ $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup"/ \ + "$(rsync_remote_full "$BACKUP_URL")/backup"/ \ $TARGET_FS_ROOT/ ;; (rsync) $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup"/ $TARGET_FS_ROOT/ + "$(rsync_remote_full "$BACKUP_URL")/backup"/ $TARGET_FS_ROOT/ ;; esac diff --git a/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh b/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh index b2fb72f509..761327940e 100644 --- a/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh +++ b/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh @@ -1,14 +1,14 @@ # check the backup archive on remote rsync server -case $RSYNC_PROTO in +case $(rsync_proto "$BACKUP_URL") in (ssh) - ssh ${RSYNC_USER}@${RSYNC_HOST} "ls -ld ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" >/dev/null 2>&1 \ - || Error "Archive not found on [$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}]" + ssh $(rsync_remote_ssh "$BACKUP_URL") "ls -ld $(rsync_path_full "$BACKUP_URL")/backup" >/dev/null 2>&1 \ + || Error "Archive not found on [$(rsync_remote_full "$BACKUP_URL")]" ;; (rsync) - $BACKUP_PROG "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup" >/dev/null 2>&1 \ - || Error "Archive not found on [$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}]" + $BACKUP_PROG "$(rsync_remote_full "$BACKUP_URL")/backup" >/dev/null 2>&1 \ + || Error "Archive not found on [$(rsync_remote_full "$BACKUP_URL")]" ;; esac From 0df5eb794d3ce2e1b5c29552efec39bd774fa096 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Fri, 17 Jun 2022 17:47:21 +0200 Subject: [PATCH 159/215] Fix creation of rsync backup directory move it from output stage to backup stage, where it belongs. --- .../200_check_rsync_relative_option.sh | 1 - .../210_check_rsync_relative_option.sh | 1 + .../RSYNC/default/200_make_prefix_dir.sh | 28 +++++++++++++++++++ ....sh => 210_check_rsync_relative_option.sh} | 2 +- .../RSYNC/default/200_make_prefix_dir.sh | 4 +-- 5 files changed, 32 insertions(+), 4 deletions(-) delete mode 120000 usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh create mode 120000 usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh create mode 100644 usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh rename usr/share/rear/backup/RSYNC/default/{200_check_rsync_relative_option.sh => 210_check_rsync_relative_option.sh} (91%) diff --git a/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh b/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh deleted file mode 120000 index 336b83f59a..0000000000 --- a/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh +++ /dev/null @@ -1 +0,0 @@ -../../RSYNC/default/200_check_rsync_relative_option.sh \ No newline at end of file diff --git a/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh b/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh new file mode 120000 index 0000000000..0570eb4496 --- /dev/null +++ b/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh @@ -0,0 +1 @@ +../../RSYNC/default/210_check_rsync_relative_option.sh \ No newline at end of file diff --git a/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh new file mode 100644 index 0000000000..81aa68790c --- /dev/null +++ b/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh @@ -0,0 +1,28 @@ +# Create RSYNC_PREFIX/backup on remote rsync server +# RSYNC_PREFIX=$HOSTNAME as set in default.conf + +local proto host + +proto="$(rsync_proto "$BACKUP_URL")" +host="$(rsync_host "$BACKUP_URL")" + +mkdir -p $v -m0750 "${TMP_DIR}/rsync/${RSYNC_PREFIX}" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}'" +mkdir -p $v -m0755 "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup'" + +case $proto in + + (ssh) + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "$(rsync_remote "$BACKUP_URL")" >/dev/null 2>&1 \ + || Error "Could not create '$(rsync_path_full "$BACKUP_URL")' on remote ${host}" + ;; + + (rsync) + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote "$BACKUP_URL")/" >/dev/null \ + || Error "Could not create '$(rsync_path_full "$BACKUP_URL")' on remote ${host}" + ;; + +esac + +# We don't need it anymore, from now we operate on the remote copy +rmdir $v "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" +rmdir $v "${TMP_DIR}/rsync/${RSYNC_PREFIX}" diff --git a/usr/share/rear/backup/RSYNC/default/200_check_rsync_relative_option.sh b/usr/share/rear/backup/RSYNC/default/210_check_rsync_relative_option.sh similarity index 91% rename from usr/share/rear/backup/RSYNC/default/200_check_rsync_relative_option.sh rename to usr/share/rear/backup/RSYNC/default/210_check_rsync_relative_option.sh index cedee9cec2..692616b7c9 100644 --- a/usr/share/rear/backup/RSYNC/default/200_check_rsync_relative_option.sh +++ b/usr/share/rear/backup/RSYNC/default/210_check_rsync_relative_option.sh @@ -1,4 +1,4 @@ -# 200_check_rsync_relative_option.sh +# 210_check_rsync_relative_option.sh # See issue #871 for details # check for the --relative option in BACKUP_RSYNC_OPTIONS array diff --git a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh index 82cb1c55e2..87b4a9198a 100644 --- a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh +++ b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh @@ -6,9 +6,8 @@ local proto host proto="$(rsync_proto "$OUTPUT_URL")" host="$(rsync_host "$OUTPUT_URL")" -# create temporary local work-spaces to collect files (we already make the remote backup dir with the correct mode!!) +# create temporary local work-spaces to collect files mkdir -p $v -m0750 "${TMP_DIR}/rsync/${RSYNC_PREFIX}" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}'" -mkdir -p $v -m0755 "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup'" case $proto in @@ -18,6 +17,7 @@ case $proto in ;; (rsync) + # This must run before the backup stage. Otherwise --relative gets added to BACKUP_RSYNC_OPTIONS $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote "$OUTPUT_URL")/" >/dev/null \ || Error "Could not create '$(rsync_path_full "$OUTPUT_URL")' on remote ${host}" ;; From c526fd0999bd87635f556c8ea1796c794206f09c Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Mon, 20 Jun 2022 11:49:39 +0200 Subject: [PATCH 160/215] Explain why altering BACKUP_RSYNC_OPTIONS is ok --- usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh index e8a2b87c2b..aa8192c031 100644 --- a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh +++ b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh @@ -26,6 +26,7 @@ ProgressStart "Running backup operation" case "$(basename $BACKUP_PROG)" in (rsync) + # We are in a subshell, so this change will not propagate to later scripts BACKUP_RSYNC_OPTIONS+=( --one-file-system --delete --exclude-from=$TMP_DIR/backup-exclude.txt --delete-excluded ) case $(rsync_proto "$BACKUP_URL") in From 41c519832c8064c57c742ed2099d845388fc40c9 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Tue, 21 Jun 2022 13:57:06 +0200 Subject: [PATCH 161/215] Handle only rsync:// schemes in rsync output code --- .../rear/output/RSYNC/default/200_make_prefix_dir.sh | 9 ++++++++- .../rear/output/RSYNC/default/900_copy_result_files.sh | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh index 87b4a9198a..d00d15e44a 100644 --- a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh +++ b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh @@ -1,7 +1,14 @@ # Create RSYNC_PREFIX under the local TMP_DIR and also on remote rsync server # RSYNC_PREFIX=$HOSTNAME as set in default.conf -local proto host +local proto host scheme + +scheme="$(url_scheme "$OUTPUT_URL")" + +# we handle only rsync:// output schemes. +# ToDo: why does handling of the output URL scheme belong under RSYNC (which is a backup method)? +# OUTPUT_URL is independent on the chosen backup method, so this code should be moved to be backup-independent. +test "rsync" = "$scheme" || return 0 proto="$(rsync_proto "$OUTPUT_URL")" host="$(rsync_host "$OUTPUT_URL")" diff --git a/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh b/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh index 517587eea2..4ddf3cb4b7 100644 --- a/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh +++ b/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh @@ -1,7 +1,15 @@ # # copy resulting files to remote network (backup) location -local proto +local proto scheme + +scheme="$(url_scheme "$OUTPUT_URL")" + +# we handle only rsync:// output schemes. +# ToDo: why does handling of the output URL scheme belong under RSYNC (which is a backup method)? +# OUTPUT_URL is independent on the chosen backup method, so this code should be moved to be backup-independent. +test "rsync" = "$scheme" || return 0 + proto="$(rsync_proto "$OUTPUT_URL")" LogPrint "Copying resulting files to $OUTPUT_URL location" From 8b8ad86c71eaea0ad03d64f5892e3f37d797a5e0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 29 Jun 2022 13:15:30 +0200 Subject: [PATCH 162/215] Update output-functions.sh Typo fix for the assert_ISO_FILE_SIZE_LIMIT() comment: "the callers environment" -> "the caller's environment" --- usr/share/rear/lib/output-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/output-functions.sh b/usr/share/rear/lib/output-functions.sh index d71cb23b37..637f34bdb2 100644 --- a/usr/share/rear/lib/output-functions.sh +++ b/usr/share/rear/lib/output-functions.sh @@ -28,7 +28,7 @@ function FindUsbDevices () { # Error out when files greater or equal ISO_FILE_SIZE_LIMIT should be included in the ISO (cf. default.conf) # for files passed as arguments e.g: assert_ISO_FILE_SIZE_LIMIT file1 relative/path/file2 /absolute/path/file3 ... # Normally there should be no error exit inside a function but a function should return non-zero exit code -# and leave it to its caller what to do depending on the callers environment. But this function is an exception. +# and leave it to its caller what to do depending on the caller's environment. But this function is an exception. # It is meant like the "assert" macro in C that outputs a message on stderr and then exits with abort(). # Furthermore it is less duplicated code to implement the error exit inside this function # than to let this function return non-zero exit code and implement the error exit in each caller From 0f584adc6d0a1db98ff50cdd53ac00695d1d4437 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 29 Jun 2022 15:00:10 +0200 Subject: [PATCH 163/215] In doc/rear-release-notes.txt tell that recreating LUKS encrypted volumes does no longer work for SLES11 SP4 (too old cryptsetup). --- doc/rear-release-notes.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index 5f742c758c..2694b7e931 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -3823,7 +3823,9 @@ software are possible at any time that could arbitrarily break how ReaR works. ReaR 2.7 may still work for SLES 11 and openSUSE Leap 42.x or even earlier openSUSE versions but it is no longer sufficiently well tested there so -arbitrary regressions could appear. +arbitrary regressions could appear. For example recreating LUKS encrypted volumes +does no longer work with SLES11 SP4 because cryptsetup 1.1.3 in SLES11 SP4 does +not support UUID and some other cryptsetup options that are used by default in ReaR 2.7. ReaR 2.7 and earlier versions are known to no longer work reasonably well for the following Linux based operating systems: @@ -3978,3 +3980,4 @@ actual fresh kernel available. Reboot your server before using ReaR again, which is a good practice anyway after upgrading the GNU/Linux kernel. + From 2008a537ec5c152438488b849c72de4b2e36f141 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Wed, 29 Jun 2022 19:26:03 +0200 Subject: [PATCH 164/215] Update a comment to point to a new file --- usr/share/rear/lib/global-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index 2e883fed7d..5f58f7a001 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -259,7 +259,7 @@ function url_scheme() { # the scheme is the leading part up to '://' local scheme=${url%%://*} # rsync scheme does not have to start with rsync:// it can also be scp style - # see the comments in usr/share/rear/prep/RSYNC/default/100_check_rsync.sh + # see the comments in usr/share/rear/lib/rsync-functions.sh echo $scheme | grep -q ":" && echo rsync || echo $scheme } From 5cb10407d08e0d923219c9b013d420a4b632843f Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Wed, 29 Jun 2022 19:27:58 +0200 Subject: [PATCH 165/215] Calling rsync_*() with a non-rsync URL is a bug --- usr/share/rear/lib/rsync-functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/lib/rsync-functions.sh b/usr/share/rear/lib/rsync-functions.sh index cf98b90dc4..0e64781bcc 100644 --- a/usr/share/rear/lib/rsync-functions.sh +++ b/usr/share/rear/lib/rsync-functions.sh @@ -15,11 +15,12 @@ function rsync_validate () { local url="$1" if [[ "$(url_scheme "$url")" != "rsync" ]]; then # url_scheme still recognizes old style - Error "Non-rsync URL $url !" + BugError "Non-rsync URL $url !" fi } # Determine whether the URL specifies the use of the rsync protocol (rsyncd) or ssh +# Do not call on non-rsync URLs (use url_scheme first) function rsync_proto () { local url="$1" From 8f40d02fa8717659426b2e3ce02ecea71202dc4b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 30 Jun 2022 09:02:25 +0200 Subject: [PATCH 166/215] In rear.8.adoc and rear.8 show Pavel Cahyna real name --- doc/rear.8 | 4 ++-- doc/rear.8.adoc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/rear.8 b/doc/rear.8 index 0ad3ad8d04..9fa07c2794 100644 --- a/doc/rear.8 +++ b/doc/rear.8 @@ -888,7 +888,7 @@ Sebastien Chabrolles (\c Renaud Metrich (\c .URL "https://github.com/rmetrich" "" ")" and -"pcahyna" (\c +Pavel Cahyna (\c .URL "https://github.com/pcahyna" "" ")." We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. Refer to the MAINTAINERS (\c @@ -915,4 +915,4 @@ The copyright is held by the original authors of the respective code pieces as c .sp Relax\-and\-Recover comes with ABSOLUTELY NO WARRANTY; for details see the GNU General Public License at \c -.URL "http://www.gnu.org/licenses/gpl.html" "" "" \ No newline at end of file +.URL "http://www.gnu.org/licenses/gpl.html" "" "" diff --git a/doc/rear.8.adoc b/doc/rear.8.adoc index 4b5bd9dc71..b5392eec30 100644 --- a/doc/rear.8.adoc +++ b/doc/rear.8.adoc @@ -591,7 +591,7 @@ Johannes Meixner (https://github.com/jsmeix), Vladimir Gozora (https://github.com/gozora), Sebastien Chabrolles (https://github.com/schabrolles), Renaud Metrich (https://github.com/rmetrich) and -"pcahyna" (https://github.com/pcahyna). +Pavel Cahyna (https://github.com/pcahyna). We hope that ReaR continues to prove useful and to attract more developers who agree to be maintainers. Refer to the MAINTAINERS (https://github.com/rear/rear/blob/master/MAINTAINERS) file for the list of active and past maintainers. From a603559f7a9f263f0a02632523cdd68d50172f7e Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 30 Jun 2022 09:53:25 +0200 Subject: [PATCH 167/215] In rear-release-notes.txt new entry about 'Rsync OUTPUT_URLs are now properly supported with BACKUP=RSYNC' see https://github.com/rear/rear/pull/2831#issuecomment-1170428318 --- doc/rear-release-notes.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index 2694b7e931..c6e0dee559 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -241,6 +241,17 @@ New features, bigger enhancements, and possibly backward incompatible changes: 2GiB and why we error out when files >= ISO_FILE_SIZE_LIMIT should be included in the ISO. +- Rsync OUTPUT_URLs are now properly supported with BACKUP=RSYNC. + Previously the output went to the location specified by BACKUP_URL + and OUTPUT_URL was ignored. One exception was OUTPUT=PXE, where the output + was uploaded to OUTPUT_URL in addition to BACKUP_URL, but RSYNC_PREFIX was + not respected and the interpretation of the URL was different: A URL of + the form rsync://[USER@]HOST[:PORT]/PATH was interpreted as using the + rsync protocol, while in all other cases such URL would be interpreted + as using rsync over ssh. This special handling is now removed: + An rsync OUTPUT_URL with OUTPUT=PXE now creates the RSYNC_PREFIX directory + at the destination and the URL is interpreted as in all other cases. + - RAWDISK: Add local rescue partition installation capability: Introduce a configuration variable 'RAWDISK_INSTALL_GPT_PARTITION_NAME', which is unset by default. When set, the rescue system will be installed to local From 479f158c0a590373207347b7f602ea40b99a9d7b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 30 Jun 2022 13:19:48 +0200 Subject: [PATCH 168/215] Update _input-output-functions.sh In lib/_input-output-functions.sh add comments that describe the difference between the Error function (the cause is not in ReaR's code) versus the BugError function (the cause is a bug in ReaR's code), cf. https://github.com/rear/rear/pull/2831#discussion_r909619554 --- usr/share/rear/lib/_input-output-functions.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/lib/_input-output-functions.sh b/usr/share/rear/lib/_input-output-functions.sh index fae162798b..7b3c29508f 100644 --- a/usr/share/rear/lib/_input-output-functions.sh +++ b/usr/share/rear/lib/_input-output-functions.sh @@ -578,6 +578,9 @@ function CallerSource () { } # Error exit: +# It is an Error when the cause is not in ReaR's code +# for example when the user specified something wrong +# or when a called program exits with a fatal error. function Error () { # Get the last sourced script out of the log file: # Using the CallerSource function is not sufficient here because CallerSource results @@ -771,7 +774,10 @@ function Error () { fi } -# Exit if there is a bug in ReaR: +# Exit if there is a bug inside ReaR: +# It is a BugError when the cause is in ReaR's code, +# for example when a ReaR function is called with wrong +# or missing required parameters and things like that. function BugError () { { local caller_source="$( CallerSource )" ; } 2>>/dev/$DISPENSABLE_OUTPUT_DEV Error " From e04125b03d1ac2f10e965852e9f33d40fe913447 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 1 Jul 2022 10:20:09 +0200 Subject: [PATCH 169/215] Enhance check for other simultaneously running 'rear' In usr/sbin/rear enhance check for other simultaneously running 'rear' to let it find running 'rear' instances independent of how it was called by the user in particular also when 'rear' is run from a GitHub checkout/clone cf. https://github.com/rear/rear/issues/2826#issuecomment-1172074862 --- usr/sbin/rear | 68 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/usr/sbin/rear b/usr/sbin/rear index 04979257f9..9ae9e84279 100755 --- a/usr/sbin/rear +++ b/usr/sbin/rear @@ -384,16 +384,74 @@ readonly RUNTIME_LOGFILE # When this currently running instance cannot run simultaneously with another instance # test that this currently running instance does not run simultaneously with another instance: if ! test "$can_run_simultaneously" ; then - # In this case pidof is needed to test what running instances there are: + # In this case pidof is needed to test what running instances are there: if ! type pidof 1>/dev/null ; then - echo "ERROR: Required program 'pidof' missing, please check your PATH" >&2 + echo "ERROR: Required program 'pidof' missing" >&2 exit 1 fi # For unknown reasons '-o %PPID' does not work for pidof at least in SLES11 - # so that a manual test is done to find out if another pid != $$ is running: - for pid in $( pidof -x "$SCRIPT_FILE" ) ; do + # so that a manual test is done to find out if another pid != $$ is running. + # This test is only some best effort attempt to find what running instances are there + # because what pidof finds depends how 'rear' is called and what pidof version is used. + # For example pidof in SLES10 SP4 seems to only consider the basename: + # # ps auxw | grep ssh + # ... 3132 ... /usr/sbin/sshd -o PidFile=/var/run/sshd.init.pid + # ... 3622 ... sshd: root@pts/1 + # # pidof -x sshd + # 3622 3132 + # # pidof -x /usr/sbin/sshd + # 3622 3132 + # # pidof -x in/sshd + # 3622 3132 + # In contrast pidof in openSUSE Leap 15.3 checks exactly what was specified when it contains a path: + # # ps auxw | grep ssh + # ... 2991 ... /usr/bin/ssh-agent /usr/bin/gpg-agent --sh --daemon --keep-display /etc/X11/xinit/xinitrc + # ... 18219 ... /usr/bin/ssh-agent -D -a /run/user/1000/keyring/.ssh + # # pidof -x "ssh-agent" + # 18219 2991 + # # pidof -x "/usr/bin/ssh-agent" + # 18219 2991 + # # pidof -x "usr/bin/ssh-agent" + # [no output] + # We do not want to use only the basename 'rear' to avoid that pidof accidentally also shows + # other 'rear' programs which are not Relax-and-Recover but some different software. + # Examples how 'rear' looks in the 'ps' output: + # When Relax-and-Recover is normally installed (e.g. as RPM package) so the 'rear' program is /usr/sbin/rear: + # # type -a rear + # rear is /usr/sbin/rear + # # rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash /usr/sbin/rear mkrescue + # # cd / + # # ./usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./usr/sbin/rear mkrescue + # # cd /usr/sbin + # # ./rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./rear mkrescue + # When a Relax-and-Recover GitHub code checkout/clone is used so the 'rear' program is /path/to/checkout/usr/sbin/rear: + # # /path/to/checkout/usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash /path/to/checkout/usr/sbin/rear mkrescue + # # cd /path/to/checkout + # # usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash usr/sbin/rear mkrescue + # # ./usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./usr/sbin/rear mkrescue + # # cd /path/to/checkout/usr/sbin + # # ./rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./rear mkrescue + # So we check $SCRIPT_FILE which is /usr/sbin/rear or /path/to/checkout/usr/sbin/rear + # and /usr/sbin/rear usr/sbin/rear ./usr/sbin/rear ./sbin/rear ./rear ($PROGRAM is 'rear'). + # We need to explicitly check /usr/sbin/rear because $SCRIPT_FILE could be /path/to/checkout/usr/sbin/rear + # and then another simultaneously running instance could be /usr/sbin/rear (e.g. from an RPM package): + for pid in $( pidof -x "$SCRIPT_FILE" "/usr/sbin/$PROGRAM" "usr/sbin/$PROGRAM" "./usr/sbin/$PROGRAM" "./sbin/$PROGRAM" "./$PROGRAM" ) ; do if test "$pid" != $$ ; then - echo "ERROR: $PROGRAM is already running, not starting again" >&2 + echo "ERROR: $PROGRAM is already running with PID $pid, not starting again" >&2 exit 1 fi done From 63e0540e3617d1b7a024f7c188d5135cce733095 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 1 Jul 2022 12:28:10 +0200 Subject: [PATCH 170/215] Update rear Added a comment in usr/sbin/rear that describes how the check for other simultaneously running 'rear' works from inside a booted ReaR recovery system cf. https://github.com/rear/rear/issues/2826#issuecomment-1172111440 and describe the limitations of that check, cf. https://github.com/rear/rear/issues/2826#issuecomment-1172138188 --- usr/sbin/rear | 75 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/usr/sbin/rear b/usr/sbin/rear index 9ae9e84279..d3d75cf78f 100755 --- a/usr/sbin/rear +++ b/usr/sbin/rear @@ -417,38 +417,57 @@ if ! test "$can_run_simultaneously" ; then # other 'rear' programs which are not Relax-and-Recover but some different software. # Examples how 'rear' looks in the 'ps' output: # When Relax-and-Recover is normally installed (e.g. as RPM package) so the 'rear' program is /usr/sbin/rear: - # # type -a rear - # rear is /usr/sbin/rear - # # rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash /usr/sbin/rear mkrescue - # # cd / - # # ./usr/sbin/rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash ./usr/sbin/rear mkrescue - # # cd /usr/sbin - # # ./rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash ./rear mkrescue + # # type -a rear + # rear is /usr/sbin/rear + # # rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash /usr/sbin/rear mkrescue + # # cd / + # # ./usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./usr/sbin/rear mkrescue + # # cd /usr/sbin + # # ./rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./rear mkrescue # When a Relax-and-Recover GitHub code checkout/clone is used so the 'rear' program is /path/to/checkout/usr/sbin/rear: - # # /path/to/checkout/usr/sbin/rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash /path/to/checkout/usr/sbin/rear mkrescue - # # cd /path/to/checkout - # # usr/sbin/rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash usr/sbin/rear mkrescue - # # ./usr/sbin/rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash ./usr/sbin/rear mkrescue - # # cd /path/to/checkout/usr/sbin - # # ./rear mkrescue - # # ps auxw | grep rear - # ... /bin/bash ./rear mkrescue + # # /path/to/checkout/usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash /path/to/checkout/usr/sbin/rear mkrescue + # # cd /path/to/checkout + # # usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash usr/sbin/rear mkrescue + # # ./usr/sbin/rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./usr/sbin/rear mkrescue + # # cd /path/to/checkout/usr/sbin + # # ./rear mkrescue + # # ps auxw | grep rear + # ... /bin/bash ./rear mkrescue # So we check $SCRIPT_FILE which is /usr/sbin/rear or /path/to/checkout/usr/sbin/rear # and /usr/sbin/rear usr/sbin/rear ./usr/sbin/rear ./sbin/rear ./rear ($PROGRAM is 'rear'). # We need to explicitly check /usr/sbin/rear because $SCRIPT_FILE could be /path/to/checkout/usr/sbin/rear - # and then another simultaneously running instance could be /usr/sbin/rear (e.g. from an RPM package): + # and then another simultaneously running instance could be /usr/sbin/rear (e.g. from an RPM package). + # Things are different when 'rear' is run inside a booted ReaR recovery system: + # Inside a booted ReaR recovery system 'rear' is '/bin/rear' + # and /sbin /usr/sbin /usr/bin are symbolic links to /bin + # and $PATH is only 'bin' so inside a booted ReaR recovery system we have + # RESCUE localhost:~ # type -a rear + # rear is /bin/rear + # cf. https://github.com/rear/rear/issues/2826#issuecomment-1172111440 + # When in the ReaR recovery system 'rear' is called normally like + # RESCUE localhost:~ # rear recover + # we get + # RESCUE localhost:~ # ps auwx | grep rear + # ... /bin/bash /bin/rear recover + # and $SCRIPT_FILE is /bin/rear so the check below works. + # The check below fails when in the ReaR recovery system 'rear' is not called normally like + # RESCUE localhost:~ # pwd + # /root + # RESCUE localhost:~ # ../bin/rear recover + # cf. https://github.com/rear/rear/issues/2826#issuecomment-1172138188 + # This check is not meant to detect any possible way how another rear instance might have been called. for pid in $( pidof -x "$SCRIPT_FILE" "/usr/sbin/$PROGRAM" "usr/sbin/$PROGRAM" "./usr/sbin/$PROGRAM" "./sbin/$PROGRAM" "./$PROGRAM" ) ; do if test "$pid" != $$ ; then echo "ERROR: $PROGRAM is already running with PID $pid, not starting again" >&2 From 7fd1839c8a109db2bafd4dcc300487626b0e6120 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 1 Jul 2022 14:39:41 +0200 Subject: [PATCH 171/215] Updated release-notes.txt --- doc/rear-release-notes.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index c6e0dee559..d91959f607 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -328,6 +328,19 @@ New features, bigger enhancements, and possibly backward incompatible changes: Details (mostly in chronological order - newest topmost): +- Set USB_DEVICE_PARTED_LABEL to match format-workflow.sh: + In format/USB/default/300_format_usb_disk.sh it is set + depending on the format workflow option -b/--bios or -e/--efi + and in in prep/USB/Linux-i386/340_find_mbr_bin.sh try to + autodetect what the USB disk partition type is and + use a specified USB_DEVICE_PARTED_LABEL if autodetection fails + see https://github.com/rear/rear/pull/2829 + +- In usr/sbin/rear enhanced the check for other simultaneously running 'rear' + to let it find running 'rear' instances independent of how it was called by the user + in particular also when 'rear' is run from a GitHub checkout/clone + cf. https://github.com/rear/rear/issues/2826 + - In usr/share/rear/lib/format-workflow.sh do actually recognise -b/--bios options, see https://github.com/rear/rear/pull/2828 From e56402fd204b89ccda0a7196c9bf5637e6bba6c1 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Fri, 1 Jul 2022 22:27:35 +0200 Subject: [PATCH 172/215] Add yes to REQUIRED_PROGS Since PR #2827 we have been piping the output of yes to lvcreate, but yes has not been added to the rescue system. Fix that. (We could have added it to REQUIRED_PROGS only if lvm is present, but let's not complicate it and add it always.) --- usr/share/rear/conf/GNU/Linux.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usr/share/rear/conf/GNU/Linux.conf b/usr/share/rear/conf/GNU/Linux.conf index 318aa45174..820077195a 100644 --- a/usr/share/rear/conf/GNU/Linux.conf +++ b/usr/share/rear/conf/GNU/Linux.conf @@ -5,6 +5,8 @@ ip less parted readlink +# For noninteractive confirmation in lvm commands during layout recreation +yes ) PROGS+=( From 9fc626918dc61b855e796d38b7fcee579136e88c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 5 Jul 2022 14:04:19 +0200 Subject: [PATCH 173/215] Update default.conf In default.conf use a more generic description about quoting to avoid bash pathname expansion because in case of BACKUP=BORG a quoted BACKUP_PROG_INCLUDE=( '/dir/*' ) seems to let 'borg' fail with "/dir/*: [Errno 2] No such file or directory: '*'" cf. https://github.com/rear/rear/issues/2833#issuecomment-1174966416 --- usr/share/rear/conf/default.conf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index a417421eca..9c549c61e3 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -1346,16 +1346,18 @@ BACKUP_PROG_DECRYPT_OPTIONS="/usr/bin/openssl des3 -d -k " test "$BACKUP_PROG_ARCHIVE" || BACKUP_PROG_ARCHIVE="backup" # BACKUP_PROG_EXCLUDE is an array of strings that get written into a backup-exclude.txt file # that is used e.g. in 'tar -X backup-exclude.txt' to get things excluded from the backup. -# Proper quoting of the BACKUP_PROG_EXCLUDE array members is crucial to avoid bash expansions. +# Quoting of the BACKUP_PROG_EXCLUDE array members avoids bash pathname expansion +# when bash pathname expansion is not wanted for the BACKUP_PROG_EXCLUDE array members. # In /etc/rear/local.conf use BACKUP_PROG_EXCLUDE+=( '/this/*' '/that/*' ) # to specify your particular items that should be excluded from the backup in addition to what # gets excluded from the backup by default here (see also BACKUP_ONLY_EXCLUDE below): BACKUP_PROG_EXCLUDE=( '/tmp/*' '/dev/shm/*' "$VAR_DIR/output/*" ) # BACKUP_PROG_INCLUDE is an array of strings that get written into a backup-include.txt file # that is used e.g. in 'tar -c $(cat backup-include.txt)' to get things included in the backup. -# Proper quoting of the BACKUP_PROG_INCLUDE array members is crucial to avoid bash expansions. -# In /etc/rear/local.conf use BACKUP_PROG_INCLUDE=( '/this/*' '/that/*' ) -# to specify your particular items that should be included in the backup in addition to what +# Quoting of the BACKUP_PROG_INCLUDE array members avoids bash pathname expansion +# when bash pathname expansion is not wanted for the BACKUP_PROG_INCLUDE array members. +# In /etc/rear/local.conf use BACKUP_PROG_INCLUDE=( /some/directory /path/to/some/file ) +# to specify particular items that should be included in the backup in addition to what # gets included in the backup by default (see BACKUP_ONLY_INCLUDE below): BACKUP_PROG_INCLUDE=( ) # When BACKUP_ONLY_INCLUDE is set to a 'true' value From 29a78a4ca9bd007db0013a44ce1741f7e514b3a5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 7 Jul 2022 11:29:30 +0200 Subject: [PATCH 174/215] Update 100_create_layout_file.sh In layout/save/GNU/Linux/100_create_layout_file.sh add the 'lsblk' output column 'WWN' (unique storage identifier / WWID) that is shown as comment in disklayout.conf because the WWN/WWID helps to see in particular in case of multipath what real disks are there and which device nodes correspond to the same physical disk, cf. https://github.com/rear/rear/issues/2812#issuecomment-1136894247 --- .../rear/layout/save/GNU/Linux/100_create_layout_file.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/100_create_layout_file.sh b/usr/share/rear/layout/save/GNU/Linux/100_create_layout_file.sh index 91789134bc..e4e5390a26 100644 --- a/usr/share/rear/layout/save/GNU/Linux/100_create_layout_file.sh +++ b/usr/share/rear/layout/save/GNU/Linux/100_create_layout_file.sh @@ -24,15 +24,15 @@ test -e "$DISKLAYOUT_FILE" && LogPrint "Overwriting existing disk layout file $D echo "Disk layout dated $START_DATE_TIME_NUMBER (YYYYmmddHHMMSS)" >$DISKLAYOUT_FILE # Have the actual storage layout as header comment in disklayout.conf # so that it is easier to make sense of the values in the subsequent entries. -# First try the command -# lsblk -ipo NAME,KNAME,PKNAME,TRAN,TYPE,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID +# First try the command (which works on SLES12 and SLES15) +# lsblk -ipo NAME,KNAME,PKNAME,TRAN,TYPE,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID,WWN # but on older systems (like SLES11) that do not support all that lsblk things # cf. https://github.com/rear/rear/pull/2626#issuecomment-856700823 # try the simpler command # lsblk -io NAME,KNAME,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID # and as fallback try 'lsblk -i' and finally try plain 'lsblk'. # When there is no 'lsblk' command there is no output (bad luck, no harm): -{ lsblk -ipo NAME,KNAME,PKNAME,TRAN,TYPE,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID || lsblk -io NAME,KNAME,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID || lsblk -i || lsblk ; } >>$DISKLAYOUT_FILE +{ lsblk -ipo NAME,KNAME,PKNAME,TRAN,TYPE,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID,WWN || lsblk -io NAME,KNAME,FSTYPE,LABEL,SIZE,MOUNTPOINT,UUID || lsblk -i || lsblk ; } >>$DISKLAYOUT_FILE # Make all lines in disklayout.conf up to now as header comments: sed -i -e 's/^/# /' $DISKLAYOUT_FILE From 4b3895f6963e7d5142bc6cc696f04304452b7a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zaoral?= Date: Fri, 24 Jun 2022 15:46:33 +0200 Subject: [PATCH 175/215] Support creating ESP partitions also on older GNU parted Parted 3.2 added the support for 'esp' keyword. Thus, parted 3.1 on RHEL 7 does not support it yet so try a fallback that sets the corresponding partition type using sgdisk. --- usr/share/rear/format/USB/default/300_format_usb_disk.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/format/USB/default/300_format_usb_disk.sh b/usr/share/rear/format/USB/default/300_format_usb_disk.sh index a912119450..4b5333fc3f 100644 --- a/usr/share/rear/format/USB/default/300_format_usb_disk.sh +++ b/usr/share/rear/format/USB/default/300_format_usb_disk.sh @@ -147,7 +147,13 @@ if is_true "$FORMAT_EFI" ; then # Set the right flag for the EFI partition: LogPrint "Setting 'esp' flag on EFI partition $RAW_USB_DEVICE$current_partition_number" if ! parted -s $RAW_USB_DEVICE set $current_partition_number esp on ; then - Error "Failed to set 'esp' flag on EFI partition $RAW_USB_DEVICE$current_partition_number" + LogPrintError "Failed to set 'esp' flag on EFI partition $RAW_USB_DEVICE$current_partition_number" + # parted 3.2 added the support for 'esp' keyword. Thus, parted 3.1 in RHEL 7 does not support it yet so try a fallback + # that sets the corresponding partition type using sgdisk. + if ! sgdisk $RAW_USB_DEVICE --typecode="$current_partition_number:EF00" ; then + Error "Failed to set 'esp' flag using sgdisk as fallback on EFI partition $RAW_USB_DEVICE$current_partition_number" + fi + LogPrintError "Set 'esp' flag using sgdisk as fallback on EFI partition $RAW_USB_DEVICE$current_partition_number" fi # Partition 1 is the EFI system partition (vfat partition) on which EFI/BOOT/BOOTX86.EFI resides # so the number of the partition that can be set up next has to be one more (i.e. now 2): From e7556a13eb95f21b5fb749c089facaf8678f3cd4 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Fri, 8 Jul 2022 18:43:03 +0200 Subject: [PATCH 176/215] Fix parsing of rsync URLs The original code did not cope well with rsync protocol and multi-compontent paths (like module/path instead of just module). It also did not cope well with trailing slashes at the end of paths when using the rsync protocol and old-style ssh URLs with paths that do not have a leading slash ([USER@]HOST:PATH instead of [USER@]HOST:/PATH). --- usr/share/rear/lib/rsync-functions.sh | 28 ++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/usr/share/rear/lib/rsync-functions.sh b/usr/share/rear/lib/rsync-functions.sh index 0e64781bcc..443a9625ab 100644 --- a/usr/share/rear/lib/rsync-functions.sh +++ b/usr/share/rear/lib/rsync-functions.sh @@ -74,25 +74,39 @@ function rsync_path () { local url="$1" local host local path + local url_without_scheme + local url_without_scheme_user host=$(url_host "$url") path=$(url_path "$url") local tmp2="${host#*@}" + url_without_scheme="${url#*//}" + url_without_scheme_user="${url_without_scheme#$(rsync_user "$url")@}" + case "$(rsync_proto "$url")" in (rsync) - # path=/gdhaese1@witsbebelnx02::backup or path=/backup - if grep -q '::' <<< $path ; then - echo "${path##*::}" + if grep -q '::' <<< $url_without_scheme_user ; then + # we can not use url_path here, it uses / as separator, not :: + local url_after_separator="${url_without_scheme_user##*::}" + # remove leading / - this is a module name + echo "${url_after_separator#/}" else - # XXX what if path=/backup/sub/directory ? Should we remove - # longest prefix? - echo "${path##*/}" + echo "${path#*/}" fi ;; (ssh) - echo "$path" + if [ "$url_without_scheme" == "$url" ]; then + # no scheme - old-style URL + if grep -q ':' <<< $url_without_scheme_user ; then + echo "${url_without_scheme_user##*:}" + else + BugError "Old-style rsync URL $url without : !" + fi + else + echo "$path" + fi ;; esac From 93155eb60131b3fa8a42d8ae0957f14b6c89b170 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 12 Jul 2022 14:06:03 +0200 Subject: [PATCH 177/215] Updated release-notes.txt (PR#2830 PR#2831) --- doc/rear-release-notes.txt | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/rear-release-notes.txt b/doc/rear-release-notes.txt index d91959f607..7d83962996 100644 --- a/doc/rear-release-notes.txt +++ b/doc/rear-release-notes.txt @@ -328,6 +328,27 @@ New features, bigger enhancements, and possibly backward incompatible changes: Details (mostly in chronological order - newest topmost): +- Refactor rsync URL support, fixes rsync OUTPUT_URL: + The code to parse rsync:// URLs was BACKUP_URL specific. + If one specified BACKUP=RSYNC and an OUTPUT_URL different from BACKUP_URL, + the OUTPUT_URL was ignored and the output files went to BACKUP_URL. + Fix by introducing generic functions for rsync URL parsing and + use them for both BACKUP_URL and OUTPUT_URL, as appropriate. + Replace all uses of global RSYNC_* variables derived + from BACKUP_URL by those functions. + There also was inconsistent special handling for OUTPUT=PXE which is now removed: + An rsync OUTPUT_URL with OUTPUT=PXE now creates the RSYNC_PREFIX directory + at the destination and the URL is interpreted as in all other cases. + See https://github.com/rear/rear/pull/2831 + and https://github.com/rear/rear/issues/2781 + +- Support creation of EFI system partition on systems + with older 'parted' that do not support the 'esp' keyword: + In format/USB/default/300_format_usb_disk.sh try + 'sgdisk ... --typecode="partition_number:EF00"' as fallback + when 'parted ... set partition_number esp on' failed + see https://github.com/rear/rear/pull/2830 + - Set USB_DEVICE_PARTED_LABEL to match format-workflow.sh: In format/USB/default/300_format_usb_disk.sh it is set depending on the format workflow option -b/--bios or -e/--efi @@ -3808,8 +3829,8 @@ up-to-date with feedback we receive from the community. ReaR 2.7 is supported on the following Linux based operating systems: -- Fedora 29, 30, 31, and 32 -- RHEL 6, 7, and 8 +- Fedora 29, 30, 31, 32, 33, and 34 +- RHEL 6, 7, 8, and 9 - CentOS 6, 7, and 8 - Scientific Linux 6 and 7 - SLES 12 and 15 From 606cb40607b1a90b91e44c8744c49c3e53adb203 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 12 Jul 2022 14:25:39 +0200 Subject: [PATCH 178/215] Update global-functions.sh In lib/global-functions.sh fixed the comment that shows a generic URI/URL syntax cf. https://github.com/rear/rear/pull/2831#discussion_r918814803 --- usr/share/rear/lib/global-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index 5f58f7a001..b4a351b894 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -238,7 +238,7 @@ function percent_decode() { # URL is the most common form of URI # see https://en.wikipedia.org/wiki/Uniform_Resource_Identifier # where a generic URI is of the form -# scheme:[//[user:password@]host[:port]][/]path[?query][#fragment] +# scheme:[//[[user:password@]host[:port]]/path[?query][#fragment] # e.g. for BACKUP_URL=sshfs://user@host/G/rear/ # url_scheme = 'sshfs' , url_host = 'user@host' , url_hostname = 'host' , url_username = 'user' , url_path = '/G/rear/' # e.g. for BACKUP_URL=usb:///dev/sdb1 From bdc467363e5189cecd4a649de7947eb59b064fa3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 12 Jul 2022 14:46:57 +0200 Subject: [PATCH 179/215] Update global-functions.sh In lib/global-functions.sh fixed the falsely fixed comment that shows now only a usual generic URI/URL syntax so it is now scheme://[[user:password@]host[:port]]/path[?query][#fragment] so the case scheme:path (like mailto:John.Doe@example.com) is left out for simplicity in that comment cf. https://github.com/rear/rear/pull/2831#discussion_r918916010 --- usr/share/rear/lib/global-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index b4a351b894..85408fccb2 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -237,8 +237,8 @@ function percent_decode() { ###### # URL is the most common form of URI # see https://en.wikipedia.org/wiki/Uniform_Resource_Identifier -# where a generic URI is of the form -# scheme:[//[[user:password@]host[:port]]/path[?query][#fragment] +# where a generic URI is usually of the form +# scheme://[[user:password@]host[:port]]/path[?query][#fragment] # e.g. for BACKUP_URL=sshfs://user@host/G/rear/ # url_scheme = 'sshfs' , url_host = 'user@host' , url_hostname = 'host' , url_username = 'user' , url_path = '/G/rear/' # e.g. for BACKUP_URL=usb:///dev/sdb1 From 4e21f08a39da239f6dbd0ca9bd7ba0d65b52dca6 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 13 Jul 2022 12:36:22 +0200 Subject: [PATCH 180/215] Update global-functions.sh In lib/global-functions.sh add TODO comment to describe that the current url_... functions do not support the minimal 'scheme:path' case of an URL cf. https://github.com/rear/rear/pull/2831#discussion_r919908570 --- usr/share/rear/lib/global-functions.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index 85408fccb2..ad8cba15ae 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -243,6 +243,21 @@ function percent_decode() { # url_scheme = 'sshfs' , url_host = 'user@host' , url_hostname = 'host' , url_username = 'user' , url_path = '/G/rear/' # e.g. for BACKUP_URL=usb:///dev/sdb1 # url_scheme = 'usb' , url_host = '' , url_hostname = '' , url_username = '' , url_path = '/dev/sdb1' +# TODO: the url_* functions do not support the minimal scheme:path case of an URL +# for example +# # url='mailto:John.Doe@example.com' +# # url_scheme "$url" +# rsync +# # url_host "$url" +# mailto:John.Doe@example.com +# # url_hostname "$url" +# example.com +# # url_username "$url" +# mailto +# # url_password "$url" +# John.Doe +# # url_path "$url" +# /mailto:John.Doe@example.com # FIXME: the ulr_* functions are not safe against special characters # for example they break when the password contains spaces # but on the other hand permitted characters for values in a URI From 91e671557b4edd0b750582fc4b0b14f0588dc0e9 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 14 Jul 2022 10:31:11 +0200 Subject: [PATCH 181/215] Update changelog In packaging/debian/changelog use timestamp '12:34:56 +0100' because otherwise with a timestamp like e.g. '12:34:56 UTC' "make deb" shows warnings like dpkg-genbuildinfo: warning: debian/changelog: badly formatted trailer line dpkg-genbuildinfo: warning: debian/changelog: found start of entry where expected more change data or trailer dpkg-genchanges: warning: debian/changelog: found end of file where expected more change data or trailer --- packaging/debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/debian/changelog b/packaging/debian/changelog index 3af1e40f33..ddc3392498 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -2,7 +2,7 @@ rear (2.7-1) stable; urgency=low * For details see the rear-release-notes.txt file - -- Johannes Meixner Thu, 9 Jun 2022 09:04:55 UTC + -- Johannes Meixner Thu, 9 Jun 2022 12:34:56 +0100 rear (2.6-1) stable; urgency=low * Mainly bug fixes From a5f78a60089fae811f867a37a409e5d6e2d2aa68 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 14 Jul 2022 10:47:57 +0200 Subject: [PATCH 182/215] ReaR 2.7 release From b3fd58fc871e00bd713a0cb081de54d746ffffb3 Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Thu, 14 Jul 2022 15:43:58 +0200 Subject: [PATCH 183/215] Pass -y to lvcreate to stop asking for input Revert "Use fail-safe 'yes' pipe for "lvm lvcreate"" This reverts commit 9d6ce289938092834425cb1c27658500f68bc69f. The -y argument is much simpler and less error-prone than a 'yes' pipe (no error termination because of SIGPIPE, for instance). But we lose support for SLES11, where lvcreate does not know the -y option. 'yes' remains among REQUIRED_PROGS, it can be useful in the rescue system. (It could be theoretically moved to PROGS, because it is not required anymore. This could be done separately if really desired, but for now we prefer to keep it in REQUIRED_PROGS, sooner or later something will start requiring it.) See https://github.com/rear/rear/pull/2834 --- .../prepare/GNU/Linux/110_include_lvm_code.sh | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh b/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh index 0bd863acd7..45b1a344a1 100644 --- a/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh +++ b/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh @@ -287,25 +287,13 @@ create_lvmvol() { # so e.g. 'lvcreate -L 123456b -n LV VG' becomes 'lvcreate -l 100%FREE -n LV VG' fallbacklvopts="$( sed -e 's/-L [0-9b]*/-l 100%FREE/' <<< "$lvopts" )" - # In SLES11 "man lvcreate" does not show '-y' or '--yes' - # so we cannot use "lvm lvcreate -y ..." - # see https://github.com/rear/rear/issues/2820#issuecomment-1161934013 - # instead we input as many 'y' as asked for by "lvm lvcreate" - # see https://github.com/rear/rear/issues/513 - # and https://github.com/rear/rear/issues/2820 - # plus be safe against possible 'set -o pipefail' non-zero exit code of 'yes' via '( yes || true ) | ...' - # see https://github.com/rear/rear/issues/2820#issuecomment-1162804476 - # because 'yes' may get terminated by SIGPIPE when plain 'yes | ...' is used - # see https://github.com/rear/rear/issues/2820#issuecomment-1162772415 - # and suppress needless "yes: standard output: Broken pipe" stderr messages - # that appear at least with newer 'yes' in coreutils-8.32 in openSUSE Leap 15.3 cat >> "$LAYOUT_CODE" </dev/null || true ) | lvm lvcreate $lvopts $vg ; then + if ! lvm lvcreate -y $lvopts $vg ; then LogPrintError "Failed to create LVM volume '$vg/$lvname' with lvcreate $lvopts $vg" - if ( yes 2>/dev/null || true ) | lvm lvcreate $fallbacklvopts $vg ; then + if lvm lvcreate -y $fallbacklvopts $vg ; then LogPrintError "Created LVM volume '$vg/$lvname' using fallback options lvcreate $fallbacklvopts $vg" else LogPrintError "Also failed to create LVM volume '$vg/$lvname' with lvcreate $fallbacklvopts $vg" From 7a9fc70ff95bda2d4409d45d70a52cb5f1c290fd Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Thu, 14 Jul 2022 15:58:23 +0200 Subject: [PATCH 184/215] Uodate a comment that became obsolete --- usr/share/rear/conf/GNU/Linux.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/conf/GNU/Linux.conf b/usr/share/rear/conf/GNU/Linux.conf index 820077195a..7e47b91264 100644 --- a/usr/share/rear/conf/GNU/Linux.conf +++ b/usr/share/rear/conf/GNU/Linux.conf @@ -5,7 +5,7 @@ ip less parted readlink -# For noninteractive confirmation in lvm commands during layout recreation +# For noninteractive confirmation in commands yes ) From 4cc3c58c7ef31572889f63cee89ffaad1525873d Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Mon, 18 Jul 2022 10:50:35 +0200 Subject: [PATCH 185/215] Update 380_copy_usb_fs_module.sh Typo fix in comment 'needd' -> 'needed' --- usr/share/rear/prep/USB/default/380_copy_usb_fs_module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/prep/USB/default/380_copy_usb_fs_module.sh b/usr/share/rear/prep/USB/default/380_copy_usb_fs_module.sh index b6531c5e78..485a1406ca 100644 --- a/usr/share/rear/prep/USB/default/380_copy_usb_fs_module.sh +++ b/usr/share/rear/prep/USB/default/380_copy_usb_fs_module.sh @@ -6,7 +6,7 @@ # moduels in /etc/modules get loaded during recovery system startup via .../system-setup.d/40-start-udev-or-load-modules.sh local usb_fs -# TODO: When 'lsblk' is not too old (it must support the needd options like '-o FSTYPE') +# TODO: When 'lsblk' is not too old (it must support the needed options like '-o FSTYPE') # then "lsblk -no FSTYPE $USB_DEVICE" could show the USB filesystem directly # in its last output line also when it is called with a direct kernel parent device: # For example my encrypted root filesystem on my openSUSE Leap 15.2 laptop is 'ext4': From f36bfe9bb6a13b1945a8e23e1cecac00ad9bf32f Mon Sep 17 00:00:00 2001 From: Pavel Cahyna Date: Wed, 27 Jul 2022 11:41:06 +0200 Subject: [PATCH 186/215] Update lvm commands shown in log messages to match what gets actually executed. --- .../rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh b/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh index 45b1a344a1..6089cc096b 100644 --- a/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh +++ b/usr/share/rear/layout/prepare/GNU/Linux/110_include_lvm_code.sh @@ -292,11 +292,11 @@ $ifline LogPrint "Creating LVM volume '$vg/$lvname' (some properties may not be preserved)" $warnraidline if ! lvm lvcreate -y $lvopts $vg ; then - LogPrintError "Failed to create LVM volume '$vg/$lvname' with lvcreate $lvopts $vg" + LogPrintError "Failed to create LVM volume '$vg/$lvname' with lvcreate -y $lvopts $vg" if lvm lvcreate -y $fallbacklvopts $vg ; then - LogPrintError "Created LVM volume '$vg/$lvname' using fallback options lvcreate $fallbacklvopts $vg" + LogPrintError "Created LVM volume '$vg/$lvname' using fallback options lvcreate -y $fallbacklvopts $vg" else - LogPrintError "Also failed to create LVM volume '$vg/$lvname' with lvcreate $fallbacklvopts $vg" + LogPrintError "Also failed to create LVM volume '$vg/$lvname' with lvcreate -y $fallbacklvopts $vg" # Explicit 'false' is needed to let the whole 'if then else fi' command exit with non zero exit state # to let diskrestore.sh abort here as usual when a command fails (diskrestore.sh runs with 'set -e'): false From 5c920d770bfd1f15958325feb70689c8a602667b Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 28 Jul 2022 11:06:01 +0200 Subject: [PATCH 187/215] Update 290_kernel_cmdline.sh In rescue/GNU/Linux/290_kernel_cmdline.sh show when not adding a kernel option to KERNEL_CMDLINE as LogPrintError to match the LogPrint that is used to show when adding a kernel option to KERNEL_CMDLINE and fixed a typo 'supperseeded' -> 'superseded', cf. https://github.com/rear/rear/pull/2749#issuecomment-1197843273 --- usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh index b66fee96b3..0e11204b34 100644 --- a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh +++ b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh @@ -19,7 +19,7 @@ for new_kernel_option in "${new_kernel_options_to_add[@]}" ; do for rear_kernel_option in $KERNEL_CMDLINE ; do # Check if a kernel option key without value parameter (everything before =) is not already present in rear KERNEL_CMDLINE array. if test "$new_kernel_option_keyword" = "${rear_kernel_option%%=*}" ; then - Log "Current kernel option [$new_kernel_option] supperseeded by [$rear_kernel_option] in your rear configuration: (KERNEL_CMDLINE)" + LogPrintError "Not adding '$new_kernel_option' (superseded by existing '$rear_kernel_option' in KERNEL_CMDLINE)" # Continue with the next new_kernel_option (i.e. continue the outer 'for' loop): continue 2 fi From d2d2300b680a3e84637bca10ceed7bf1f8a7ebec Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 28 Jul 2022 11:28:07 +0200 Subject: [PATCH 188/215] Update default.conf Better description of COPY_KERNEL_PARAMETERS in default.conf, cf. https://github.com/rear/rear/pull/2749#issuecomment-1197843273 --- usr/share/rear/conf/default.conf | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/usr/share/rear/conf/default.conf b/usr/share/rear/conf/default.conf index 9c549c61e3..c191d3d9c9 100644 --- a/usr/share/rear/conf/default.conf +++ b/usr/share/rear/conf/default.conf @@ -105,12 +105,13 @@ KERNEL_VERSION="${KERNEL_VERSION:-$( uname -r )}" # but for the latter using USE_DHCLIENT="yes" (see below) is probably easier. # See also the 'RESCUE IMAGE KERNEL COMMAND LINE OPTIONS' section in the ReaR man page ("man rear"): KERNEL_CMDLINE="" -# The COPY_KERNEL_PARAMETERS array lists kernel parameter that should be part of the KERNEL_CMDLINE (in rescue image) -# if present on the current system (/proc/cmdline). -# COPY_KERNEL_PARAMETERS should only store kernel parameter key, not the value (ex: net.ifnames but not net.ifnames=0) -# If the key-value kernel parameter is already set in KERNEL_CMDLINE variable it will always superseed -# the one detected on the current system (if any). -# - Check net.ifnames and biosdevname kernel parameter as it may impact the network interface name during recovery/migration. +# The COPY_KERNEL_PARAMETERS array lists kernel parameters that should be part +# of the KERNEL_CMDLINE (in rescue image) if present on the current system (/proc/cmdline). +# COPY_KERNEL_PARAMETERS specifies kernel parameter keys without value (e.g. net.ifnames but not net.ifnames=0). +# If such a kernel parameter key is already specified in the KERNEL_CMDLINE variable +# the already specified setting supersedes the one detected on the current system (if any). +# - Check net.ifnames and biosdevname kernel parameter as it may impact +# the network interface name during recovery/migration. COPY_KERNEL_PARAMETERS=( 'net.ifnames' 'biosdevname' ) # These variables are used to include arch/os/version specific stuff: From 8a04aed776f9a9e331ae1b60fa926e7af71ee462 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 28 Jul 2022 13:54:44 +0200 Subject: [PATCH 189/215] Update 290_kernel_cmdline.sh Overhauled rescue/GNU/Linux/290_kernel_cmdline.sh in particular to make it possible to add several kernel options by this script with same kernel option keyword like console=ttyS0,9600 console=tty0 cf. https://github.com/rear/rear/pull/2749#issuecomment-1197843273 --- .../rescue/GNU/Linux/290_kernel_cmdline.sh | 73 +++++++++++-------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh index 0e11204b34..996e34357c 100644 --- a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh +++ b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh @@ -1,51 +1,60 @@ -# purpose of the script is to detect some important KERNEL CMDLINE options on the current system -# we should also use in rescue mode (automatically update KERNEL_CMDLINE array variable). +# Purpose of the script is to get the COPY_KERNEL_PARAMETERS on the current system +# to be used in the rescue/recovery system via automated update of KERNEL_CMDLINE. -# Scanning current kernel cmdline to look for important option ($COPY_KERNEL_PARAMETERS) to include in KERNEL_CMDLINE -for current_kernel_option in $( cat /proc/cmdline ); do +# Also protect the rescue/recovery system by removing net.ifnames=0 from KERNEL_CMDLINE +# if net.ifnames=0 is in KERNEL_CMDLINE but persistent network interface naming is used: +local persistent_naming='no' +is_persistent_ethernet_name $( ip route | awk '$2 == "dev" && $8 == "src" { print $3 }' | sort -u | head -n1 ) && persistent_naming='yes' + +# Scan current kernel cmdline for options in COPY_KERNEL_PARAMETERS to be included in KERNEL_CMDLINE: +local current_kernel_option +local new_kernel_options_to_add=() +for current_kernel_option in $( cat /proc/cmdline ) ; do # Get the current kernel option name (part before leftmost "=") and # add the whole option (with value) to new_kernel_options_to_add array # if the option name is part of COPY_KERNEL_PARAMETERS array: - if IsInArray "${current_kernel_option%%=*}" "${COPY_KERNEL_PARAMETERS[@]}" ; then - new_kernel_options_to_add+=( "$current_kernel_option" ) - fi + IsInArray "${current_kernel_option%%=*}" "${COPY_KERNEL_PARAMETERS[@]}" && new_kernel_options_to_add+=( "$current_kernel_option" ) done -# Verify if the kernel option we want to add to KERNEL_CMDLINE are not already set/force by the user in the rear configuration. -# If yes, the parameter set in the configuration file have the priority and superseed the current kernel option. +# Check if the kernel options we want to add to KERNEL_CMDLINE are already set by the user in KERNEL_CMDLINE. +# If yes, the user setting has priority and superseds the kernel option from the current system. +# For the check use the existing KERNEL_CMDLINE when this script is started +# and not tha modified KERNEL_CMDLINE with already added kernel options +# to make it possible to add several kernel options by this script +# with same kernel option keyword like console=ttyS0,9600 console=tty0 +# cf. https://github.com/rear/rear/pull/2749#issuecomment-1197843273 +local existing_kernel_cmdline="$KERNEL_CMDLINE" +local existing_kernel_option new_kernel_option new_kernel_option_keyword for new_kernel_option in "${new_kernel_options_to_add[@]}" ; do new_kernel_option_keyword="${new_kernel_option%%=*}" - - for rear_kernel_option in $KERNEL_CMDLINE ; do - # Check if a kernel option key without value parameter (everything before =) is not already present in rear KERNEL_CMDLINE array. - if test "$new_kernel_option_keyword" = "${rear_kernel_option%%=*}" ; then - LogPrintError "Not adding '$new_kernel_option' (superseded by existing '$rear_kernel_option' in KERNEL_CMDLINE)" + for existing_kernel_option in $existing_kernel_cmdline ; do + if test "$new_kernel_option_keyword" = "${existing_kernel_option%%=*}" ; then + LogPrint "Not adding '$new_kernel_option' (superseded by existing '$existing_kernel_option' in KERNEL_CMDLINE)" # Continue with the next new_kernel_option (i.e. continue the outer 'for' loop): continue 2 fi done - + # If we are using persistent naming do not add net.ifnames to KERNEL_CMDLINE + # see https://github.com/rear/rear/pull/1874 + # and continue with the next new_kernel_option: if test "net.ifnames" = "$new_kernel_option_keyword" ; then - # If we are using persistent naming do not add net.ifnames to KERNEL_CMDLINE - # see https://github.com/rear/rear/pull/1874 - # and continue with the next new_kernel_option: - is_persistent_ethernet_name $( ip r | awk '$2 == "dev" && $8 == "src" { print $3 }' | sort -u | head -1 ) && continue + if is_true $persistent_naming ; then + LogPrint "Not adding '$new_kernel_option' (persistent network interface naming is used)" + continue + fi fi - LogPrint "Adding $new_kernel_option to KERNEL_CMDLINE" - KERNEL_CMDLINE="$KERNEL_CMDLINE $new_kernel_option" + KERNEL_CMDLINE+=" $new_kernel_option" done -# In case we added 'KERNEL_CMDLINE="$KERNEL_CMDLINE net.ifnames=0"' to /etc/rear/local.conf, but we have no idea if we -# are using persistent naming or not then we should protect the rescue image from doing stupid things and remove -# the keyword (and value) in a preventive way in case "persistent naming is in use". +# In case we added 'KERNEL_CMDLINE="$KERNEL_CMDLINE net.ifnames=0"' to /etc/rear/local.conf +# but we have no idea if we are using persistent naming or not +# then we should protect the rescue image from doing stupid things +# and remove the keyword (and value) in a preventive way in case "persistent naming is in use". # And, to be clear the /proc/cmdline did not contain the keyword net.ifnames - -if is_persistent_ethernet_name $( ip r | awk '$2 == "dev" && $8 == "src" { print $3 }' | sort -u | head -1 ) ; then - # persistent naming is in use - # When the KERNEL_CMDLINE does NOT contain net.ifnames=0 silently return - echo $KERNEL_CMDLINE | grep -q 'net.ifnames=0' || return - # Remove net.ifnames=0 from KERNEL_CMDLINE - KERNEL_CMDLINE=$( echo $KERNEL_CMDLINE | sed -e 's/net.ifnames=0//' ) - LogPrint "Removing net.ifnames=0 from KERNEL_CMDLINE" +if is_true $persistent_naming ; then + if echo $KERNEL_CMDLINE | grep -q 'net.ifnames=0' ; then + KERNEL_CMDLINE=$( echo $KERNEL_CMDLINE | sed -e 's/net.ifnames=0//' ) + LogPrint "Removing 'net.ifnames=0' from KERNEL_CMDLINE (persistent network interface naming is used)" + fi fi From a24fed4ae58d61c01855ce2a77152f31a90929cd Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 28 Jul 2022 15:33:36 +0200 Subject: [PATCH 190/215] Update 290_kernel_cmdline.sh Fixed typo in comment 'tha' -> 'the' and added link to https://github.com/rear/rear/pull/2844 --- usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh index 996e34357c..5fb1f3e3db 100644 --- a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh +++ b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh @@ -19,10 +19,11 @@ done # Check if the kernel options we want to add to KERNEL_CMDLINE are already set by the user in KERNEL_CMDLINE. # If yes, the user setting has priority and superseds the kernel option from the current system. # For the check use the existing KERNEL_CMDLINE when this script is started -# and not tha modified KERNEL_CMDLINE with already added kernel options +# and not the modified KERNEL_CMDLINE with already added kernel options # to make it possible to add several kernel options by this script # with same kernel option keyword like console=ttyS0,9600 console=tty0 -# cf. https://github.com/rear/rear/pull/2749#issuecomment-1197843273 +# see https://github.com/rear/rear/pull/2749#issuecomment-1197843273 +# and https://github.com/rear/rear/pull/2844 local existing_kernel_cmdline="$KERNEL_CMDLINE" local existing_kernel_option new_kernel_option new_kernel_option_keyword for new_kernel_option in "${new_kernel_options_to_add[@]}" ; do From 0edf9f288ba6bcffd761aabe7e3dbeda2ce2065c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 29 Jul 2022 12:06:12 +0200 Subject: [PATCH 191/215] Update 290_kernel_cmdline.sh Consistently show kernel options within single quotes in messages and more clear comment about protecting the recovery system from falsely using 'net.ifnames=0' when actually persistent naming is used. --- .../rear/rescue/GNU/Linux/290_kernel_cmdline.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh index 5fb1f3e3db..255fecf344 100644 --- a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh +++ b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh @@ -44,18 +44,17 @@ for new_kernel_option in "${new_kernel_options_to_add[@]}" ; do continue fi fi - LogPrint "Adding $new_kernel_option to KERNEL_CMDLINE" + LogPrint "Adding '$new_kernel_option' to KERNEL_CMDLINE" KERNEL_CMDLINE+=" $new_kernel_option" done -# In case we added 'KERNEL_CMDLINE="$KERNEL_CMDLINE net.ifnames=0"' to /etc/rear/local.conf -# but we have no idea if we are using persistent naming or not -# then we should protect the rescue image from doing stupid things -# and remove the keyword (and value) in a preventive way in case "persistent naming is in use". -# And, to be clear the /proc/cmdline did not contain the keyword net.ifnames +# The user may hav added 'net.ifnames=0' to KERNEL_CMDLINE in /etc/rear/local.conf +# but he does not know whether or not persistent naming is used. +# So we should protect the rescue/recovery system from doing "stupid things" +# and remove 'net.ifnames=0' in a preventive way when persistent naming is used: if is_true $persistent_naming ; then if echo $KERNEL_CMDLINE | grep -q 'net.ifnames=0' ; then KERNEL_CMDLINE=$( echo $KERNEL_CMDLINE | sed -e 's/net.ifnames=0//' ) - LogPrint "Removing 'net.ifnames=0' from KERNEL_CMDLINE (persistent network interface naming is used)" + LogPrint "Removed 'net.ifnames=0' from KERNEL_CMDLINE (persistent network interface naming is used)" fi fi From 5970a6d955ee9532ae1e0d83fef255a40e6e8391 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Fri, 29 Jul 2022 13:14:06 +0200 Subject: [PATCH 192/215] Update 290_kernel_cmdline.sh Typo fix in comment 'hav' -> 'have' --- usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh index 255fecf344..02aebecc8e 100644 --- a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh +++ b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh @@ -48,8 +48,8 @@ for new_kernel_option in "${new_kernel_options_to_add[@]}" ; do KERNEL_CMDLINE+=" $new_kernel_option" done -# The user may hav added 'net.ifnames=0' to KERNEL_CMDLINE in /etc/rear/local.conf -# but he does not know whether or not persistent naming is used. +# The user may have added 'net.ifnames=0' to KERNEL_CMDLINE in /etc/rear/local.conf +# but he may not know whether or not persistent naming is used. # So we should protect the rescue/recovery system from doing "stupid things" # and remove 'net.ifnames=0' in a preventive way when persistent naming is used: if is_true $persistent_naming ; then From 4b4efc580a5ac7c4b8670b2ca4e511ccceec03bd Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 3 Aug 2022 10:34:21 +0200 Subject: [PATCH 193/215] Update 290_kernel_cmdline.sh Keep the 'if then fi' block even for its one line assignment body to increase readability because the 'if' condition is a bit complicated with its implicitly changed variable value via bash parameter expansion so it is better to have that on a separated line for easier understanding. A new named variable 'current_kernel_option_keyword' would be best but because all is well explained in the comment it is OK this way. Cf. https://github.com/rear/rear/pull/2844#discussion_r936340652 --- usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh index 02aebecc8e..eb6c45deb2 100644 --- a/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh +++ b/usr/share/rear/rescue/GNU/Linux/290_kernel_cmdline.sh @@ -13,7 +13,9 @@ for current_kernel_option in $( cat /proc/cmdline ) ; do # Get the current kernel option name (part before leftmost "=") and # add the whole option (with value) to new_kernel_options_to_add array # if the option name is part of COPY_KERNEL_PARAMETERS array: - IsInArray "${current_kernel_option%%=*}" "${COPY_KERNEL_PARAMETERS[@]}" && new_kernel_options_to_add+=( "$current_kernel_option" ) + if IsInArray "${current_kernel_option%%=*}" "${COPY_KERNEL_PARAMETERS[@]}" ; then + new_kernel_options_to_add+=( "$current_kernel_option" ) + fi done # Check if the kernel options we want to add to KERNEL_CMDLINE are already set by the user in KERNEL_CMDLINE. From 7952b51184dccdeb0e0af6358f5ed731d644925f Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:15:44 +0200 Subject: [PATCH 194/215] Use STRING+=" additional words" everywhere Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/backup/NETFS/default/500_make_backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/backup/NETFS/default/500_make_backup.sh b/usr/share/rear/backup/NETFS/default/500_make_backup.sh index 229300c625..fa3becf5e6 100644 --- a/usr/share/rear/backup/NETFS/default/500_make_backup.sh +++ b/usr/share/rear/backup/NETFS/default/500_make_backup.sh @@ -11,7 +11,7 @@ function set_tar_features () { local tar_version=$( get_version tar --version ) if version_newer "$tar_version" 1.23 ; then FEATURE_TAR_WARNINGS="y" - TAR_OPTIONS="$TAR_OPTIONS --warning=no-xdev" + TAR_OPTIONS+=" --warning=no-xdev" fi FEATURE_TAR_IS_SET=1 } From 7121c43adc654c161b06a63ab141242b8b8e6ee5 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:19:22 +0200 Subject: [PATCH 195/215] Update 501_check_ssh_keys.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/build/default/501_check_ssh_keys.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/build/default/501_check_ssh_keys.sh b/usr/share/rear/build/default/501_check_ssh_keys.sh index c4d84f424b..2c7462667c 100644 --- a/usr/share/rear/build/default/501_check_ssh_keys.sh +++ b/usr/share/rear/build/default/501_check_ssh_keys.sh @@ -88,7 +88,7 @@ for key_file in "${key_files[@]}" ; do if ssh-keygen -q -p -P '' -N '' -f "$ROOTFS_DIR/$key_file" >/dev/null 2>&1 ; then rm -v "$ROOTFS_DIR/$key_file" 1>&2 Log "Removed SSH key file '$key_file' from recovery system because it has no passphrase" - removed_key_files="$removed_key_files $key_file" + removed_key_files+=" $key_file" else Log "SSH key file '$key_file' has a passphrase and is allowed in the recovery system" fi From 0e7b7e5935bf5286dcbc4618b37bf8460f969b77 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:22:11 +0200 Subject: [PATCH 196/215] Update 990_verify_rootfs.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/build/default/990_verify_rootfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/build/default/990_verify_rootfs.sh b/usr/share/rear/build/default/990_verify_rootfs.sh index 1e201cb3ef..095d176726 100644 --- a/usr/share/rear/build/default/990_verify_rootfs.sh +++ b/usr/share/rear/build/default/990_verify_rootfs.sh @@ -231,7 +231,7 @@ for program in "${PROGS[@]}" ; do # Use the basename because the path within the recovery system is usually different compared to the path on the original system: program=$( basename $program ) # Redirected stdin for login shell avoids motd welcome message, cf. https://github.com/rear/rear/issues/2120. - chroot $ROOTFS_DIR /bin/bash --login -c "type $program" < /dev/null || missing_programs="$missing_programs $program" + chroot $ROOTFS_DIR /bin/bash --login -c "type $program" < /dev/null || missing_programs+=" $program" done # Report programs in the PROGS array that cannot be found as executable command within the recovery system: @@ -256,7 +256,7 @@ for required_program in "${REQUIRED_PROGS[@]}" ; do # Use the basename because the path within the recovery system is usually different compared to the path on the original system: required_program=$( basename $required_program ) # Redirected stdin for login shell avoids motd welcome message, cf. https://github.com/rear/rear/issues/2120. - chroot $ROOTFS_DIR /bin/bash --login -c "type $required_program" < /dev/null || missing_required_programs="$missing_required_programs $required_program" + chroot $ROOTFS_DIR /bin/bash --login -c "type $required_program" < /dev/null || missing_required_programs+=" $required_program" done # Report programs in the REQUIRED_PROGS array that cannot be found as executable command within the recovery system: if contains_visible_char "$missing_required_programs" ; then From 26044c10a1ba2b10b915b7e397f485447e516e7a Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:26:22 +0200 Subject: [PATCH 197/215] Update 250_migrate_lun_wwid.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/finalize/GNU/Linux/250_migrate_lun_wwid.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/finalize/GNU/Linux/250_migrate_lun_wwid.sh b/usr/share/rear/finalize/GNU/Linux/250_migrate_lun_wwid.sh index e1bdd2363a..39f8086390 100644 --- a/usr/share/rear/finalize/GNU/Linux/250_migrate_lun_wwid.sh +++ b/usr/share/rear/finalize/GNU/Linux/250_migrate_lun_wwid.sh @@ -10,7 +10,7 @@ Log "TAG-15-migrate-wwid: $LUN_WWID_MAP" local sed_script="" local old_wwid new_wwid device while read old_wwid new_wwid device ; do - sed_script="$sed_script;/${old_wwid}/s/${old_wwid}/${new_wwid}/g" + sed_script+=";/${old_wwid}/s/${old_wwid}/${new_wwid}/g" done < <( sort -u $LUN_WWID_MAP ) # debug line: Debug "$sed_script" From 0894e04dc1f600dd1e7455768a354899d45db858 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:27:29 +0200 Subject: [PATCH 198/215] Update 280_migrate_uuid_tags.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh b/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh index d994ce8ef2..ea6371a7f5 100644 --- a/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh +++ b/usr/share/rear/finalize/GNU/Linux/280_migrate_uuid_tags.sh @@ -10,7 +10,7 @@ Log "TAG-15-migrate: $FS_UUID_MAP" local sed_script="" local old_uuid new_uuid device while read old_uuid new_uuid device ; do - sed_script="$sed_script;/${old_uuid}/s/${old_uuid}/${new_uuid}/g" + sed_script+=";/${old_uuid}/s/${old_uuid}/${new_uuid}/g" done < <( sort -u $FS_UUID_MAP ) # debug line: Debug "$sed_script" From 9994e96dc67f503480e13c205c5c710617bea061 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:36:16 +0200 Subject: [PATCH 199/215] Update 136_include_btrfs_subvolumes_SLES_code.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- .../prepare/GNU/Linux/136_include_btrfs_subvolumes_SLES_code.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/prepare/GNU/Linux/136_include_btrfs_subvolumes_SLES_code.sh b/usr/share/rear/layout/prepare/GNU/Linux/136_include_btrfs_subvolumes_SLES_code.sh index c3a26084b7..babeff62fd 100644 --- a/usr/share/rear/layout/prepare/GNU/Linux/136_include_btrfs_subvolumes_SLES_code.sh +++ b/usr/share/rear/layout/prepare/GNU/Linux/136_include_btrfs_subvolumes_SLES_code.sh @@ -77,7 +77,7 @@ btrfs_subvolumes_setup_SLES() { # When there is a non-empty subvolume_path, btrfs snapshot subvolume handling is needed: Log "Handling snapshot subvolume $subvolume_path for $device at $mountpoint" # Remember btrfs snapshot subvolumes to exclude them when mounting all btrfs normal subvolumes below: - snapshot_subvolumes_devices_and_paths="$snapshot_subvolumes_devices_and_paths $device,$subvolume_path" + snapshot_subvolumes_devices_and_paths+=" $device,$subvolume_path" # Be verbose if there are active btrfs snapshot subvolumes entries (snapshot subvolumes cannot be recreated). # In this case inform the user that nothing can be done for activated snapshot subvolumes entries: if test "btrfssnapshotsubvol" = "$keyword" ; then From 95066de3f895b87c97e4f88ac0ffe192ce53cc7e Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:43:24 +0200 Subject: [PATCH 200/215] Update 200_partition_layout.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh index a581adc064..52a4b142c8 100644 --- a/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/200_partition_layout.sh @@ -262,7 +262,7 @@ extract_partitions() { flags="" for flag in $flaglist ; do if [[ "$flag" = boot || "$flag" = esp || "$flag" = root || "$flag" = swap || "$flag" = hidden || "$flag" = raid || "$flag" = lvm || "$flag" = lba || "$flag" = palo || "$flag" = legacy_boot || "$flag" = bios_grub || "$flag" = prep ]] ; then - flags="$flags$flag," + flags+="$flag," elif [[ "$flag" = "type=06" ]] ; then flags="${flags}prep," fi @@ -293,7 +293,7 @@ extract_partitions() { flags="" for flag in $flaglist ; do if [[ "$flag" = boot || "$flag" = root || "$flag" = swap || "$flag" = hidden || "$flag" = raid || "$flag" = lvm || "$flag" = lba || "$flag" = palo || "$flag" = legacy_boot || "$flag" = bios_grub || "$flag" = prep ]] ; then - flags="$flags$flag," + flags+="$flag," elif [[ "$flag" = "type=06" ]] ; then flags="${flags}prep," fi From 8ee82a90b087c02298538d21d9ee4de78a8184f8 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:48:47 +0200 Subject: [PATCH 201/215] Update 230_filesystem_layout.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- .../rear/layout/save/GNU/Linux/230_filesystem_layout.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh b/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh index 6aded348ee..f05a397acb 100644 --- a/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/230_filesystem_layout.sh @@ -48,7 +48,7 @@ fi # /dev/sda2 / btrfs rw,relatime,space_cache # The sorting relies on that mount and findmnt output the first mounted thing first # so that in particular what is mounted at '/' is output before other stuff. -read_filesystems_command="$read_filesystems_command | sort -t ' ' -k 1,1 -u" +read_filesystems_command+=" | sort -t ' ' -k 1,1 -u" # The Docker daemon mounts file systems for its Docker containers, see also # https://docs.docker.com/storage/storagedriver/device-mapper-driver/#configure-direct-lvm-mode-for-production @@ -232,7 +232,7 @@ fi ;; (btrfs) # Remember devices and mountpoints of the btrfs filesystems for the btrfs subvolume layout stuff below: - btrfs_devices_and_mountpoints="$btrfs_devices_and_mountpoints $device,$mountpoint" + btrfs_devices_and_mountpoints+=" $device,$mountpoint" uuid=$( btrfs filesystem show $device | grep -o 'uuid: .*' | cut -d ':' -f 2 | tr -d '[:space:]' ) label=$( btrfs filesystem show $device | grep -o 'Label: [^ ]*' | cut -d ':' -f 2 | tr -d '[:space:]' ) test "none" = "$label" && label= @@ -349,7 +349,7 @@ fi # for all devices except '/dev/sda3' where btrfs_subvolumes_setup_SLES() is called to setup that btrfs filesystem # cf. https://github.com/rear/rear/pull/2080#discussion_r265046317 and see the code in the # usr/share/rear/layout/prepare/GNU/Linux/133_include_mount_filesystem_code.sh script: - IsInArray "$btrfs_device" "${BTRFS_SUBVOLUME_SLES_SETUP[@]}" || btrfs_subvolume_sles_setup_devices="$btrfs_subvolume_sles_setup_devices $btrfs_device" + IsInArray "$btrfs_device" "${BTRFS_SUBVOLUME_SLES_SETUP[@]}" || btrfs_subvolume_sles_setup_devices+=" $btrfs_device" # SLES 12 SP1 (or later) normal subvolumes that belong to snapper are excluded from being recreated: # Snapper's base subvolume '/@/.snapshots' is excluded because during "rear recover" # that one will be created by "snapper/installation-helper --step 1" which fails if it already exists From 2ec68fdcdc97253276ad0f52dfab8e1f6349e8b2 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:50:32 +0200 Subject: [PATCH 202/215] Update 280_multipath_layout.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh b/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh index e78355f2df..7d0a889a55 100644 --- a/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh @@ -56,7 +56,7 @@ if grep -q ^multipath $DISKLAYOUT_FILE ; then # depending to the linux distro and arch, libaio can be located in different dir. (ex: /lib/powerpc64le-linux-gnu) for libdir in $(ldconfig -p | awk '/libaio.so/ { print $NF }' | xargs -n1 dirname | sort -u); do - libaio2add="$libaio2add $libdir/libaio*" + libaio2add+=" $libdir/libaio*" done LIBS+=( $libaio2add ) fi From fa343139047a07c7896bfb31fc0f330139cee256 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:52:27 +0200 Subject: [PATCH 203/215] Update 280_multipath_layout.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh b/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh index 7d0a889a55..c5d696516b 100644 --- a/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh +++ b/usr/share/rear/layout/save/GNU/Linux/280_multipath_layout.sh @@ -30,7 +30,7 @@ while read dm_name junk ; do slaves="" for slave in /sys/block/$name/slaves/* ; do - slaves="$slaves$(get_device_name ${slave##*/})," + slaves+="$(get_device_name ${slave##*/})," done dm_disktype=$(parted -s $dev_name print | grep -E "Partition Table|Disk label" | cut -d ":" -f "2" | tr -d " ") From 25079a6ff972108fe7bc01d067cb75230a722405 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Tue, 9 Aug 2022 13:54:34 +0200 Subject: [PATCH 204/215] Update 510_current_disk_usage.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/layout/save/GNU/Linux/510_current_disk_usage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/layout/save/GNU/Linux/510_current_disk_usage.sh b/usr/share/rear/layout/save/GNU/Linux/510_current_disk_usage.sh index 0711ded3d0..a3295367c9 100644 --- a/usr/share/rear/layout/save/GNU/Linux/510_current_disk_usage.sh +++ b/usr/share/rear/layout/save/GNU/Linux/510_current_disk_usage.sh @@ -11,7 +11,7 @@ local rear_USB_ESP_partition="$( readlink -f /dev/disk/by-label/REAR-EFI )" local egrep_pattern="" test "$rear_USB_data_partition" && egrep_pattern="^$rear_USB_data_partition" if test "$rear_USB_ESP_partition" ; then - test "$egrep_pattern" && egrep_pattern="$egrep_pattern|^$rear_USB_ESP_partition" || egrep_pattern="^$rear_USB_ESP_partition" + test "$egrep_pattern" && egrep_pattern+="|^$rear_USB_ESP_partition" || egrep_pattern="^$rear_USB_ESP_partition" fi # The disk usage must be in MiB units '-BM' (and not in arbitrary human readable units via '-h') # because the values are used in 420_autoresize_last_partitions.sh to calculate whether or not From 191dd466599d8e56803f749c1e5c9fe991ce34ae Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 12:16:05 +0200 Subject: [PATCH 205/215] Update _input-output-functions.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/lib/_input-output-functions.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/usr/share/rear/lib/_input-output-functions.sh b/usr/share/rear/lib/_input-output-functions.sh index 7b3c29508f..1360f82bc5 100644 --- a/usr/share/rear/lib/_input-output-functions.sh +++ b/usr/share/rear/lib/_input-output-functions.sh @@ -158,7 +158,7 @@ function terminate_descendants_from_grandchildren_to_children () { if kill -0 $descendant_pid ; then # Keep the current ordering also in not_yet_terminated_pids # i.e. grandchildren before children: - not_yet_terminated_pids="$not_yet_terminated_pids $descendant_pid" + not_yet_terminated_pids+=" $descendant_pid" LogPrint "Descendant process $descendant_pid not yet terminated" fi done @@ -251,7 +251,7 @@ function terminate_descendants_from_children_to_grandchildren () { if kill -0 $descendant_pid ; then # Keep the current ordering also in not_yet_terminated_pids # i.e. children before grandchildren: - not_yet_terminated_pids="$not_yet_terminated_pids $descendant_pid" + not_yet_terminated_pids+=" $descendant_pid" LogPrint "Child process $descendant_pid not yet terminated" fi done @@ -1167,7 +1167,7 @@ function UserInput () { # Avoid stderr if timeout is not set or empty or not an integer value: if test "$timeout" -ge 1 2>/dev/null ; then if test "$default_and_timeout" ; then - default_and_timeout="$default_and_timeout timeout $timeout seconds" + default_and_timeout+=" timeout $timeout seconds" else default_and_timeout="timeout $timeout seconds" fi @@ -1198,18 +1198,18 @@ function UserInput () { fi # Prepare the 'read' call: local read_options_and_arguments="" - is_true "$raw_input" && read_options_and_arguments="$read_options_and_arguments -r" - is_true "$silent_input" && read_options_and_arguments="$read_options_and_arguments -s" + is_true "$raw_input" && read_options_and_arguments+=" -r" + is_true "$silent_input" && read_options_and_arguments+=" -s" # When a zero timeout was specified (via -t 0) do not use it. # Avoid stderr if timeout is not set or empty or not an integer value: - test "$timeout" -ge 1 2>/dev/null && read_options_and_arguments="$read_options_and_arguments -t $timeout" + test "$timeout" -ge 1 2>/dev/null && read_options_and_arguments+=" -t $timeout" # When no input_words_array_name was specified (via -a myarr) do not use it: - test "$input_words_array_name" && read_options_and_arguments="$read_options_and_arguments -a $input_words_array_name" + test "$input_words_array_name" && read_options_and_arguments+=" -a $input_words_array_name" # When zero input_max_chars was specified (via -n 0) do not use it. # Avoid stderr if input_max_chars is not set or empty or not an integer value: - test "$input_max_chars" -ge 1 2>/dev/null && read_options_and_arguments="$read_options_and_arguments -n $input_max_chars" + test "$input_max_chars" -ge 1 2>/dev/null && read_options_and_arguments+=" -n $input_max_chars" # When no input_delimiter was specified (via -d x) do not use it: - test "$input_delimiter" && read_options_and_arguments="$read_options_and_arguments -d $input_delimiter" + test "$input_delimiter" && read_options_and_arguments+=" -d $input_delimiter" # Get the actual user input value: local input_string="" # When a predefined user input value exists use that as automated user input: From ea5f6340978fe56742d35f2ac370d4c7697b60e0 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 12:20:31 +0200 Subject: [PATCH 206/215] Update columns-functions.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/lib/columns-functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/rear/lib/columns-functions.sh b/usr/share/rear/lib/columns-functions.sh index 0140a3db6b..17c81486ff 100644 --- a/usr/share/rear/lib/columns-functions.sh +++ b/usr/share/rear/lib/columns-functions.sh @@ -41,7 +41,7 @@ init_columns() { let start=$i-${#word} word=$( echo "$word" | tr '[:upper:]' '[:lower:]') - columns="$columns${word%% }=$start;" + columns+="${word%% }=$start;" word="" fi else @@ -59,7 +59,7 @@ init_columns() { # last word let start=$i-${#word} word=$( echo "$word"| tr '[:upper:]' '[:lower:]') - columns="$columns${word%% }=$start;" + columns+="${word%% }=$start;" #echo "c:$columns" } @@ -129,7 +129,7 @@ get_columns() { local start=$(get_column_start "$header") local size=$(get_column_size "$header") #echo "$header $start $size" - value="$value${line:$start:$size};" + value+="${line:$start:$size};" done echo "$value" } From ad53d111e6d3017461e433f83023dda28c07eff3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 12:22:34 +0200 Subject: [PATCH 207/215] Update global-functions.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/lib/global-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index ad8cba15ae..df19b63065 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -418,7 +418,7 @@ function backup_path() { path="" ;; (file) # type file needs a local path (must be mounted by user) - path="$path/${NETFS_PREFIX}" + path+="/${NETFS_PREFIX}" ;; (iso) if [[ "$WORKFLOW" = "recover" ]]; then @@ -452,7 +452,7 @@ function output_path() { case $scheme in (file) # type file needs a local path (must be mounted by user) - path="$path/${OUTPUT_PREFIX}" + path+="/${OUTPUT_PREFIX}" ;; (*) # nfs, cifs, usb, a.o. need a temporary mount-path path="${BUILD_DIR}/outputfs/${OUTPUT_PREFIX}" From aad61e0b6bcadb21b10fc89fd65f9b20da177e1c Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 12:28:06 +0200 Subject: [PATCH 208/215] Update 240_include_multipath_tools.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/prep/GNU/Linux/240_include_multipath_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/prep/GNU/Linux/240_include_multipath_tools.sh b/usr/share/rear/prep/GNU/Linux/240_include_multipath_tools.sh index a823eeee6c..0b0871e6e9 100644 --- a/usr/share/rear/prep/GNU/Linux/240_include_multipath_tools.sh +++ b/usr/share/rear/prep/GNU/Linux/240_include_multipath_tools.sh @@ -9,6 +9,6 @@ COPY_AS_IS+=( /etc/multipath.conf /etc/multipath/* /lib*/multipath ) # depending to the linux distro and arch, libaio can be located in different dir. (ex: /lib/powerpc64le-linux-gnu) for libdir in $(ldconfig -p | awk '/libaio.so/ { print $NF }' | xargs -n1 dirname | sort -u); do - libaio2add="$libaio2add $libdir/libaio*" + libaio2add+=" $libdir/libaio*" done LIBS+=( $libaio2add ) From 7ed3bd30477e9f2c690e0847feb288cf1242f5f1 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 12:31:05 +0200 Subject: [PATCH 209/215] Update 070_set_backup_archive.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh b/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh index 4644b79bb0..b7cd8b4291 100644 --- a/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh +++ b/usr/share/rear/prep/NETFS/default/070_set_backup_archive.sh @@ -233,7 +233,7 @@ if test "$latest_full_backup" ; then # Tell the user what will be restored: local restore_archives_file_names="" for restore_archive in "${RESTORE_ARCHIVES[@]}" ; do - restore_archives_file_names="$restore_archives_file_names $( basename "$restore_archive" )" + restore_archives_file_names+=" $( basename "$restore_archive" )" done LogPrint "For backup restore using $restore_archives_file_names" fi From 2c1d26e78769e4dfde044f84e40376b99a9afaf3 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 12:58:02 +0200 Subject: [PATCH 210/215] Update 200_selinux_in_use.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 Additionally fixed indentation and cleaned up the logic that "just returns if SELinux is not in use". --- .../RSYNC/GNU/Linux/200_selinux_in_use.sh | 114 +++++++++--------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh b/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh index 845000399f..671b8aa8f1 100644 --- a/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh +++ b/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh @@ -1,70 +1,76 @@ -# check if SELinux is in use, if not, just silently return -[[ -f /selinux/enforce || -f /sys/fs/selinux/enforce ]] || return 0 -if [ -f /selinux/enforce ]; then - SELINUX_ENFORCE=/selinux/enforce -elif [ -f /sys/fs/selinux/enforce ]; then - SELINUX_ENFORCE=/sys/fs/selinux/enforce +# check if SELinux is in use, if not, just return +if [ -f /selinux/enforce ] ; then + SELINUX_ENFORCE=/selinux/enforce +elif [ -f /sys/fs/selinux/enforce ] ; then + SELINUX_ENFORCE=/sys/fs/selinux/enforce else - SELINUX_ENFORCE= - BugError "SELinux enforce file is not found. Please enhance this script." + Log "SELinux is not in use (neither /selinux/enforce nor /sys/fs/selinux/enforce exists)" + return fi # check global settings (see default.conf) if is_true "$BACKUP_SELINUX_DISABLE" ; then - cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode - RSYNC_SELINUX= - return + cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode + RSYNC_SELINUX= + return fi #PROGS+=( setfiles chcon restorecon ) -# SELinux is found to be available on this system; depending on backup program we may need to do different things -# So far, only rsync and tar has special options for selinux. Others, just disable SELinux during backup only! +# SELinux is found to be available on this system; +# depending on backup program we may need to do different things +# So far, only rsync and tar has special options for selinux. +# Others, just disable SELinux during backup only! case $(basename $BACKUP_PROG) in - (rsync) - if grep -q "no xattrs" "$TMP_DIR/rsync_protocol"; then - local host - host="$(rsync_host "$BACKUP_URL")" - # no xattrs compiled in remote rsync, so saving SELinux attributes are not possible - Log "WARNING: --xattrs not possible on system ($host) (no xattrs compiled in rsync)" - # $TMP_DIR/selinux.mode is a trigger during backup to disable SELinux - cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode - RSYNC_SELINUX= # internal variable used in recover mode (empty means disable SELinux) - touch $TMP_DIR/force.autorelabel # after reboot the restored system do a forced SELinux relabeling - else - # if --xattrs is already set; no need to do it again - if ! grep -q xattrs <<< "${BACKUP_RSYNC_OPTIONS[*]}" ; then - BACKUP_RSYNC_OPTIONS+=( --xattrs ) - fi - RSYNC_SELINUX=1 # variable used in recover mode (means using xattr and not disable SELinux) - fi - ;; + (rsync) + if grep -q "no xattrs" "$TMP_DIR/rsync_protocol" ; then + local host + host="$(rsync_host "$BACKUP_URL")" + # no xattrs compiled in remote rsync, so saving SELinux attributes are not possible + Log "WARNING: --xattrs not possible on system ($host) (no xattrs compiled in rsync)" + # $TMP_DIR/selinux.mode is a trigger during backup to disable SELinux + cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode + # internal variable used in recover mode (empty means disable SELinux) + RSYNC_SELINUX= + # after reboot the restored system do a forced SELinux relabeling + touch $TMP_DIR/force.autorelabel + else + # if --xattrs is already set; no need to do it again + if ! grep -q xattrs <<< "${BACKUP_RSYNC_OPTIONS[*]}" ; then + BACKUP_RSYNC_OPTIONS+=( --xattrs ) + fi + # variable used in recover mode (means using xattr and not disable SELinux) + RSYNC_SELINUX=1 + fi + ;; - (tar) - if tar --usage | grep -q selinux ; then - # during backup we will NOT disable SELinux - BACKUP_PROG_OPTIONS+=( "--selinux" ) + (tar) + if tar --usage | grep -q selinux ; then + # during backup we will NOT disable SELinux + BACKUP_PROG_OPTIONS+=( "--selinux" ) + # include SELinux utilities and /etc/selinux directory + # so rescue/restore ReaR image can run with SELinux enabled + PROGS+=( getenforce setenforce sestatus setfiles chcon restorecon ) + COPY_AS_IS+=( /etc/selinux ) + # alter kernel command line to explicitly enable SELinux + # replace 'selinux=0' with 'selinux=1' if 'selinux=0' exists + # or append 'selinux=1' if no 'selinux=1' exists + KERNEL_CMDLINE=$( echo $KERNEL_CMDLINE | sed -e 's/selinux=0/selinux=1/' ) + echo $KERNEL_CMDLINE | grep -q 'selinux=1' || KERNEL_CMDLINE+=" selinux=1" + else + # during backup we will disable SELinux + cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode + touch $TMP_DIR/force.autorelabel + # after reboot the restored system does a SELinux relabeling + fi + ;; - # include SELinux utilities and /etc/selinux directory so rescue/restore ReaR image can run with SELinux enabled - PROGS+=( getenforce setenforce sestatus setfiles chcon restorecon ) - COPY_AS_IS+=( /etc/selinux ) - # alter kernel command line to explicitly enable SELinux (append "selinux=1" if no selinux=0 exists) - KERNEL_CMDLINE=$(echo $KERNEL_CMDLINE | sed -e 's/selinux=0/selinux=1/') - echo $KERNEL_CMDLINE | grep -q 'selinux=1' || KERNEL_CMDLINE="$KERNEL_CMDLINE selinux=1" - else - # during backup we will disable SELinux - cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode - touch $TMP_DIR/force.autorelabel - # after reboot the restored system does a SELinux relabeling - fi - ;; - - (*) - # disable SELinux for unlisted BACKUP_PROGs - cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode - touch $TMP_DIR/force.autorelabel - ;; + (*) + # disable SELinux for unlisted BACKUP_PROGs + cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode + touch $TMP_DIR/force.autorelabel + ;; esac From 43f0834ad424bb04c6bc31b14d40abfa36b96495 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 13:00:02 +0200 Subject: [PATCH 211/215] Update 610_save_capabilities.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/rescue/NETFS/default/610_save_capabilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/rescue/NETFS/default/610_save_capabilities.sh b/usr/share/rear/rescue/NETFS/default/610_save_capabilities.sh index 9240447e5c..461f049ca0 100644 --- a/usr/share/rear/rescue/NETFS/default/610_save_capabilities.sh +++ b/usr/share/rear/rescue/NETFS/default/610_save_capabilities.sh @@ -16,7 +16,7 @@ has_binary getcap && has_binary setcap || Error "getcap and setcap are needed wh # Empty values must be avoided for egrep -v because egrep -v '' or egrep -v 'something|' matches all: exclude_directories="$BUILD_DIR" -test "$ISO_DIR" && exclude_directories="$exclude_directories|$ISO_DIR" +test "$ISO_DIR" && exclude_directories+="|$ISO_DIR" # Be backward compatible: is_true "$NETFS_RESTORE_CAPABILITIES" && NETFS_RESTORE_CAPABILITIES=( '/' ) From bf57866250e27054f592b48b65de3775f0dd6c28 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 13:19:24 +0200 Subject: [PATCH 212/215] Update 400_restore_with_tsm.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/restore/TSM/default/400_restore_with_tsm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/restore/TSM/default/400_restore_with_tsm.sh b/usr/share/rear/restore/TSM/default/400_restore_with_tsm.sh index 4f30d9f382..db7762f5f5 100644 --- a/usr/share/rear/restore/TSM/default/400_restore_with_tsm.sh +++ b/usr/share/rear/restore/TSM/default/400_restore_with_tsm.sh @@ -32,7 +32,7 @@ for num in $TSM_RESTORE_FILESPACE_NUMS ; do cat /dev/null >$backup_restore_log_file UserOutput "Filespace '$filespace' restore progress can be followed with 'tail -f $backup_restore_log_file'" # Make sure filespace has a trailing / (for dsmc): - test "${filespace:0-1}" == "/" || filespace="$filespace/" + test "${filespace:0-1}" == "/" || filespace+="/" Log "Running 'LC_ALL=$LANG_RECOVER dsmc restore $filespace $TARGET_FS_ROOT/$filespace -subdir=yes -replace=all -tapeprompt=no -errorlogname=\"$backup_restore_log_file\" ${TSM_DSMC_RESTORE_OPTIONS[@]}'" # Regarding things like '0<&6 1>&7 2>&8' see "What to do with stdin, stdout, and stderr" in https://github.com/rear/rear/wiki/Coding-Style # Both stdout and stderr are redirected into the backup restore log file From feccd3a96737eac476055c9832a91a85bd834f27 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 13:23:49 +0200 Subject: [PATCH 213/215] Update 400_restore_packages.sh Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/restore/YUM/default/400_restore_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/restore/YUM/default/400_restore_packages.sh b/usr/share/rear/restore/YUM/default/400_restore_packages.sh index ce127988b1..816941a459 100644 --- a/usr/share/rear/restore/YUM/default/400_restore_packages.sh +++ b/usr/share/rear/restore/YUM/default/400_restore_packages.sh @@ -18,7 +18,7 @@ LogPrint "Adding rpm repositories from $yum_backup_dir" local repoList="" for repo in $(cat $yum_backup_dir/rpm_repositories.dat) do - repoList="$repoList --enablerepo=$repo" + repoList+=" --enablerepo=$repo" done mkdir -p $TARGET_FS_ROOT/etc/yum.repos.d cp -a /etc/yum.repos.d/* $TARGET_FS_ROOT/etc/yum.repos.d/ From a338e7c75d22abc38ff0c898d3df2d1a1e50e3bb Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Wed, 10 Aug 2022 13:31:08 +0200 Subject: [PATCH 214/215] Update system-setup Use simpler and faster syntax STRING+=" additional words" see https://github.com/rear/rear/issues/2848 --- usr/share/rear/skel/default/etc/scripts/system-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/rear/skel/default/etc/scripts/system-setup b/usr/share/rear/skel/default/etc/scripts/system-setup index 1dc7f1e8ab..7243ecce33 100755 --- a/usr/share/rear/skel/default/etc/scripts/system-setup +++ b/usr/share/rear/skel/default/etc/scripts/system-setup @@ -99,7 +99,7 @@ if test -s "/md5sums.txt" ; then # because it seems it can be modified even before this md5sum verification here runs, # see https://github.com/rear/rear/issues/1883#issuecomment-409875733 egrep_pattern="/etc/issue|/etc/ssh/sshd_config|/etc/udev/rules.d/70-persistent-net.rules" - test "$EXCLUDE_MD5SUM_VERIFICATION" && egrep_pattern="$egrep_pattern|$EXCLUDE_MD5SUM_VERIFICATION" + test "$EXCLUDE_MD5SUM_VERIFICATION" && egrep_pattern+="|$EXCLUDE_MD5SUM_VERIFICATION" # Regardless of '--quiet' md5sum shows "FAILED" messages nevertheless (cf. 'man md5sum'): if grep -E -v "$egrep_pattern" md5sums.txt | md5sum --quiet --check ; then echo -e "md5sums are OK\n" From 83a6976b0c977a583067916e636ad11cad3643f8 Mon Sep 17 00:00:00 2001 From: Johannes Meixner Date: Thu, 11 Aug 2022 15:01:57 +0200 Subject: [PATCH 215/215] Update global-functions.sh Better syntax of the explanation comments for the is_true and is_false functions. --- usr/share/rear/lib/global-functions.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index df19b63065..77263cb1d6 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -92,19 +92,19 @@ function contains_visible_char () { test "$( tr -d -c '[:graph:]' <<<"$*" )" } -# Two explicit functions to be able to test explicitly for true and false (see issue #625) +# Two functions to be able to test explicitly for true and false (see issue #625) # because "tertium non datur" (cf. https://en.wikipedia.org/wiki/Law_of_excluded_middle) -# does not hold for variables because variables could be unset or have empty value -# and to test if a variable is true or false its value is tested by that functions -# but the variable may not have a real value (i.e. be unset or have empty value) and -# because both functions test explicitly '! is_true' is not the same as 'is_false' +# does not hold for variables because variables could be unset or have empty value. +# To test if a variable is true or false its value is tested by that functions +# but the variable may not have a real value (i.e. be unset or have empty value). +# Because both functions test explicitly '! is_true' is not the same as 'is_false' # and '! is_false' is not the same as 'is_true' (see both function comments below): function is_true () { - # The argument is usually the value of a variable which needs to be tested - # only if there is explicitly a 'true' value then is_true returns true - # so that an unset variable or an empty value is not true - # and also for any other value that is not recognized as a 'true' value + # The argument is usually the value of a variable which needs to be tested. + # Only if there is explicitly a 'true' value then is_true returns true + # so that an unset variable or an empty value is not true. + # Also for any other value that is not recognized as a 'true' value # by the is_true function the is_true function results false: case "$1" in ([tT] | [yY] | [yY][eE][sS] | [tT][rR][uU][eE] | 1) @@ -114,11 +114,11 @@ function is_true () { } function is_false () { - # The argument is usually the value of a variable which needs to be tested - # only if there is explicitly a 'false' value then is_false returns true + # The argument is usually the value of a variable which needs to be tested. + # Only if there is explicitly a 'false' value then is_false returns true # so that an unset variable or an empty value is not false - # (caution: for unset or empty variables is_false is false) - # and also for any other value that is not recognized as a 'false' value + # (caution: for unset or empty variables is_false is false). + # Also for any other value that is not recognized as a 'false' value # by the is_false function the is_false function results false: case "$1" in ([fF] | [nN] | [nN][oO] | [fF][aA][lL][sS][eE] | 0)