Linux系统运维日志

说明

备份很重要，我们使用一台备份服务器来对gitlab服务器数据进行备份，而测试备份文件是否可用也很重要，要不然用到备份文件，而此文件无法正常还原就损失大了。所以我们还对备份文件进行还原测试及使用zabbix监控备份和还原是否成功。
备份服务器系统使用centos7。下面是备份服务器的配置过程。

配置备份服务器环境

安装docker

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager 
    --add-repo 
    https://download.docker.com/linux/centos/docker-ce.repo
yum -y install docker-ce    
systemctl start docker

拉取gitlab镜像

docker pull hub-mirror.c.163.com/gitlab/gitlab-ce:8.17.2-ce.0
docker tag hub-mirror.c.163.com/gitlab/gitlab-ce:8.17.2-ce.0 gitlab/gitlab-ce:8.17.2-ce.0

配置密钥免密码登录gitlab服务器

ssh-keygen
ssh-copy-id -i ~/.ssh/id_rsa.pub 192.168.93.5

关闭selinux

sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/sysconfig/selinux
setenforce 0

安装zabbix-agent

rpm -ivh http://repo.zabbix.com/zabbix/3.2/rhel/7/x86_64/zabbix-release-3.2-1.el7.noarch.rpm 
yum -y install zabbix-agent zabbix-sender
service zabbix-agent start
chkconfig zabbix-agent on

配置agent

server_ip=192.168.93.202
sed -i "s/Server=.*/Server=$server_ip/"  /etc/zabbix/zabbix_agentd.conf
sed -i "s/ServerActive=.*/Server=$server_ip/"  /etc/zabbix/zabbix_agentd.conf
service zabbix-agent restart

导入模板

<?xml version="1.0" encoding="UTF-8"?>
<zabbix_export>
    <version>3.2</version>
    <date>2017-09-06T15:39:00Z</date>
    <groups>
        <group>
            <name>Templates</name>
        </group>
    </groups>
    <templates>
        <template>
            <template>Template Gitlab Backup</template>
            <name>Template Gitlab Backup</name>
            <description/>
            <groups>
                <group>
                    <name>Templates</name>
                </group>
            </groups>
            <applications>
                <application>
                    <name>gitlab-backup</name>
                </application>
            </applications>
            <items>
                <item>
                    <name>gitlab backup result</name>
                    <type>7</type>
                    <snmp_community/>
                    <multiplier>0</multiplier>
                    <snmp_oid/>
                    <key>gitlab-backup</key>
                    <delay>86400</delay>
                    <history>90</history>
                    <trends>0</trends>
                    <status>0</status>
                    <value_type>4</value_type>
                    <allowed_hosts/>
                    <units/>
                    <delta>0</delta>
                    <snmpv3_contextname/>
                    <snmpv3_securityname/>
                    <snmpv3_securitylevel>0</snmpv3_securitylevel>
                    <snmpv3_authprotocol>0</snmpv3_authprotocol>
                    <snmpv3_authpassphrase/>
                    <snmpv3_privprotocol>0</snmpv3_privprotocol>
                    <snmpv3_privpassphrase/>
                    <formula>1</formula>
                    <delay_flex/>
                    <params/>
                    <ipmi_sensor/>
                    <data_type>0</data_type>
                    <authtype>0</authtype>
                    <username/>
                    <password/>
                    <publickey/>
                    <privatekey/>
                    <port/>
                    <description/>
                    <inventory_link>0</inventory_link>
                    <applications>
                        <application>
                            <name>gitlab-backup</name>
                        </application>
                    </applications>
                    <valuemap/>
                    <logtimefmt/>
                </item>
            </items>
            <discovery_rules/>
            <httptests/>
            <macros/>
            <templates/>
            <screens/>
        </template>
    </templates>
    <triggers>
        <trigger>
            <expression>{Template Gitlab Backup:gitlab-backup.strlen()}&lt;&gt;0</expression>
            <recovery_mode>0</recovery_mode>
            <recovery_expression/>
            <name>gitlab backup failed</name>
            <correlation_mode>0</correlation_mode>
            <correlation_tag/>
            <url/>
            <status>0</status>
            <priority>2</priority>
            <description/>
            <type>0</type>
            <manual_close>0</manual_close>
            <dependencies/>
            <tags/>
        </trigger>
        <trigger>
            <expression>{Template Gitlab Backup:gitlab-backup.nodata(25h)}=1</expression>
            <recovery_mode>0</recovery_mode>
            <recovery_expression/>
            <name>no gitlab backup message</name>
            <correlation_mode>0</correlation_mode>
            <correlation_tag/>
            <url/>
            <status>0</status>
            <priority>2</priority>
            <description/>
            <type>0</type>
            <manual_close>0</manual_close>
            <dependencies/>
            <tags/>
        </trigger>
    </triggers>
</zabbix_export>

配置备份脚本

mkdir -p /data/sh
vi /data/sh/backup-gitlab.sh

脚本内容：

#!/bin/bash
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin"

GITLAB_HOST="192.168.93.5"
ZABBIX_SERVER="192.168.93.202"
BACKUP_SERVER_HOST_NAME="backup server"

# 开始ssh远程执行备份
echo "开始ssh远程执行备份"
if ! msg=`ssh $GITLAB_HOST gitlab-rake gitlab:backup:create 2>&1`;then
    zabbix_sender -z $ZABBIX_SERVER -s "$BACKUP_SERVER_HOST_NAME" -k gitlab-backup -o "backup failed: $msg"
    exit 1
fi
# 获取备份文件名
echo "获取备份文件名"
backup_filename=`ssh $GITLAB_HOST ls -t /var/opt/gitlab/backups/ | head -1`
mkdir -p /data/backup/gitlab/

# 拉取备份文件
echo "拉取备份文件"
if ! msg=`scp $GITLAB_HOST:/var/opt/gitlab/backups/$backup_filename /data/backup/gitlab/ 2>&1`;then
    zabbix_sender -z $ZABBIX_SERVER -s "$BACKUP_SERVER_HOST_NAME" -k gitlab-backup -o "scp backup file failed: $msg"
    exit 1  
fi

# 拉取配置文件
echo "拉取配置文件"
mkdir -p /data/backup/gitlab/etc/
if ! msg=`scp -r $GITLAB_HOST:/etc/gitlab/* /data/backup/gitlab/etc/ 2>&1`;then
    zabbix_sender -z $ZABBIX_SERVER -s "$BACKUP_SERVER_HOST_NAME" -k gitlab-backup -o "scp etc file failed: $msg"
    exit 1  
fi
# 备份.ssh
mkdir -p /data/backup/gitlab/etc/ssh/
scp -r $GITLAB_HOST:/var/opt/gitlab/.ssh/ /data/backup/gitlab/etc/ssh/

timestamp=`echo $backup_filename  | sed 's/_gitlab_backup.tar//'`
# 启动一个gitlab容器进行还原测试
echo "启动一个gitlab容器进行还原测试"
docker run --detach 
    --hostname gitlab.example.com 
    --name gitlab 
    --restart always 
    -p 443:443 
    -p 80:80 
    --volume /data/backup/gitlab/etc:/etc/gitlab 
    -v /data/backup/gitlab:/var/opt/gitlab/backups/ 
    gitlab/gitlab-ce:8.17.2-ce.0

# 停止unicorn和sidekiq
echo "等待gitlab启动"
while true;do
    if [[ `docker exec gitlab gitlab-ctl status | grep -c ^run` -eq 7  ]];then
        break
    fi
    sleep 5    
done
echo "重新配置"
docker exec gitlab gitlab-ctl reconfigure
docker exec gitlab gitlab-ctl restart
echo "停止unicorn和sidekiq"
docker exec gitlab gitlab-ctl stop unicorn
docker exec gitlab gitlab-ctl stop sidekiq
chmod -R 777 /data/backup/gitlab/

# 开始还原备份
echo "开始还原备份"
if ! msg=`docker exec gitlab gitlab-rake gitlab:backup:restore BACKUP=$timestamp force=yes 2>&1`;then
    zabbix_sender -z $ZABBIX_SERVER -s "$BACKUP_SERVER_HOST_NAME" -k gitlab-backup -o "rescore failed: $msg"
    exit 1
fi
docker exec gitlab gitlab-ctl start
docker stop gitlab
docker rm gitlab

# 清除备份
find /data/backup/gitlab/*.tar -mtime +7 -exec rm -f {} ;
zabbix_sender -z $ZABBIX_SERVER -s "$BACKUP_SERVER_HOST_NAME" -k gitlab-backup -o ""

加定时任务

01 03 * * * /data/sh/backup-gitlab.sh