logo
Bendi新闻
>
干货—— ES 备份恢复

干货—— ES 备份恢复

5月前

转自：https://www.cnblogs.com/sxy-blog/p/18128248

1.官网提供snap快照备份恢复

 

https://www.elastic.co/guide/en/elasticsearch/reference/7.9/snapshot-restore.html

环境要求

 

要求所有es节点，挂载同一个共享目录，可以使用NFS。

注意背景
[root@devops01 ~]#id elasticsearch 
uid=998(elasticsearch) gid=996(elasticsearch) groups=996(elasticsearch)
[root@devops01 ~]#

搭建NFS机器，服务端

yum install nfs-utils -y

# 根据es的用户信息，创建挂载用户
groupadd elasticsearch -g 996
useradd elasticsearch -g 996 -u 998 -M -s /sbin/nologin

cat > /etc/exports << 'EOF'
/es-nfs-data 10.0.0.0/24(rw,sync,all_squash,anonuid=998,anongid=996)
EOF

# 目录创建授权
[root yc-k8s ~]#mkdir -p /es-nfs-data
[root yc-k8s ~]#chown -R elasticsearch.elasticsearch /es-nfs-data/

[root yc-k8s ~]#ll -d  /es-nfs-data/
drwxr-xr-x. 2 elasticsearch elasticsearch 6 Dec  3 02:16 /es-nfs-data/
[root yc-k8s ~]#


[root yc-k8s ~]#systemctl restart nfs


[root yc-k8s ~]#showmount -e 10.0.0.122
Export list for 10.0.0.122:
/es-nfs-data 10.0.0.0/24

es节点安装nfs插件

cat > nfs-client.sh <<'EOF'
yum install nfs-utils -y
mkdir /es-client-data -p
mount -t nfs 10.0.0.122:/es-nfs-data  /es-client-data
EOF


sh nfs-client.sh

# 检查挂载
df -h |grep es-client-data

es节点修改config，开启snapshot

每个节点config.yml添加配置
path.repo: /es-client-data/


# 如
[es-node3 root ~]#cat /etc/elasticsearch/elasticsearch.yml 
cluster.name: yuchao_es
node.name: es-node3
path.data: /var/lib/elasticsearch/
path.logs: /var/log/elasticsearch/
bootstrap.memory_lock: true
network.host: 127.0.0.1,10.0.0.20
http.port: 9200
discovery.seed_hosts: ["10.0.0.18","10.0.0.19","10.0.0.20"]
cluster.initial_master_nodes: ["10.0.0.18"]
path.repo: /es-client-data/
[es-node3 root ~]#

# 重启
systemctl restart elasticsearch.service

确保重启正确

 

1.先注册快照仓库

 

PUT /_snapshot/my_backup
{
  "type": "fs",
  "settings": {
    "location": "/es-client-data/my_backup_location",
    "compress": true
  }
}

# 可以查询仓库信息
GET /_snapshot/my_backup

 

2.创建一个快照

PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=true

 

检查快照

 

图解nfs快照

 

3.针对具体index的快照

# 创建第二个快照，名字snapshot_2 

PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true
{
  "indices": "t1,t2",
  "ignore_unavailable": true,
  "include_global_state": false
  }
}

 

4.查看快照信息

 


GET /_snapshot

GET /_snapshot/my_backup/


GET /_snapshot/my_backup/snapshot_1

GET /_snapshot/my_backup/snapshot_2

 

5.查看正在运行的快照

GET /_snapshot/my_backup/_current

{
  "snapshots" : [ ]
}

6.演练恢复索引

1. 删除 t2库

2.基于有该t2 库的快照
POST /_snapshot/my_backup/snapshot_2/_restore

3.默认恢复，发现有同名index，导致恢复错误

4.可以单独还原某个index，或者删除es数据，全库恢复。

GET /_snapshot

GET /_snapshot/my_backup/


GET /_snapshot/my_backup/snapshot_1

GET /_snapshot/my_backup/snapshot_2


GET /_snapshot/my_backup/_current

POST /_snapshot/my_backup/snapshot_2/_restore

# 恢复指定的索引
POST /_snapshot/my_backup/snapshot_2/_restore
{
  "indices": "t2",
  "ignore_unavailable": true,
  "include_global_state": false,              
  "rename_pattern": "index_(.+)",             
  "rename_replacement": "restored_index_$1",  
  "include_aliases": false
}

# 参数解释
上面的indices, 表示只恢复索引’index_1’
rename_pattern: 表示重命名索引以’index_’开头的索引.（注意我们这里索引叫做t2，所以没作用）
rename_replacement: 表示将所有的索引重命名为’restored_index_xxx’.如index_1会被重命名为restored_index_1.

 

指定index恢复

 

恢复且修改index名


# 如果想修改恢复后的index名字
POST /_snapshot/my_backup/snapshot_2/_restore
{
  "indices": "t2",
  "ignore_unavailable": true,
  "include_global_state": false,              
  "rename_pattern": "t(.+)",             
  "rename_replacement": "restored_index_$1",  
  "include_aliases": false
}

 

7.用日期命名快照

不建议去用。

PUT /_snapshot/my_backup/<snapshot-{now/d}>
PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E

 

8.记录es练习接口

PUT /_snapshot/my_backup
{
  "type": "fs",
  "settings": {
    "location": "/es-client-data/my_backup_location",
    "compress": true
  }
}


GET /_snapshot/my_backup


PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=true

PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true
{
  "indices": "t1,t2",
  "ignore_unavailable": true,
  "include_global_state": false
  }
}


GET /_snapshot/

GET /_snapshot/my_backup/


GET /_snapshot/my_backup/_all


GET /_snapshot/my_backup/snapshot_1

GET /_snapshot/my_backup/snapshot_2


GET /_snapshot/my_backup/_current

POST /_snapshot/my_backup/snapshot_2/_restore

POST /_snapshot/my_backup/snapshot_2/_restore
{
  "indices": "t2",
  "ignore_unavailable": true,
  "include_global_state": false,              
  "rename_pattern": "index_(.+)",             
  "rename_replacement": "restored_index_$1",  
  "include_aliases": false
}


POST /_snapshot/my_backup/snapshot_2/_restore
{
  "indices": "t2",
  "ignore_unavailable": true,
  "include_global_state": false,              
  "rename_pattern": "t(.+)",             
  "rename_replacement": "restored_index_$1",  
  "include_aliases": false
}


PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E

2.第三方备份工具

1.安装node环境，基于nodejs开发的备份工具
# 注意node版本要求和于超老师一致

2.软件官网
https://www.npmjs.com/package/elasticdump

2.1 安装node

wget https://nodejs.org/dist/v10.16.3/node-v10.16.3-linux-x64.tar.xz

tar -xf node-v10.16.3-linux-x64.tar.xz 


[root yc-k8s /opt]#
[root yc-k8s /opt]#ln -s node-v10.16.3-linux-x64/ node
[root yc-k8s /opt]#
[root yc-k8s /opt]#echo 'export PATH=/opt/node/bin:$PATH' >> /etc/profile
[root yc-k8s /opt]#
[root yc-k8s /opt]#source /etc/profile
[root yc-k8s /opt]#
[root yc-k8s /opt]#node -v
v10.16.3
[root yc-k8s /opt]#
[root yc-k8s /opt]#npm -v
6.9.0
[root yc-k8s /opt]#

# 设置淘宝源
# 于超老师这里测试，腾讯源快一点

npm config set registry https://registry.npm.taobao.org
npm config set registry http://mirrors.cloud.tencent.com/npm/


npm config get registry

# 安装elasticdump备份工具

[root yc-k8s /opt]#npm install elasticdump -g

[root yc-k8s /opt]#elasticdump --version
6.94.1

2.2 备份命令

备份es数据为可读的json文件



[root yc-k8s /opt]#elasticdump \
> --input=http://10.0.0.18:9200/t1 \
> --output=/es-nfs-data/t1.json \
> --type=data
Sat, 03 Dec 2022 14:35:45 GMT | starting dump
Sat, 03 Dec 2022 14:35:45 GMT | got 8 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:35:45 GMT | sent 8 objects to destination file, wrote 8
Sat, 03 Dec 2022 14:35:45 GMT | got 0 objects from source elasticsearch (offset: 8)
Sat, 03 Dec 2022 14:35:45 GMT | Total Writes: 8
Sat, 03 Dec 2022 14:35:45 GMT | dump complete
[root yc-k8s /opt]#


[root yc-k8s /es-nfs-data]#cat t1.json 
{"_index":"t1","_type":"_doc","_id":"7HNg1IQBRcFJVNOeWVxR","_score":1,"_source":{"name":"yu1","age":"28","address":"JS","job":"dev"}}
{"_index":"t1","_type":"_doc","_id":"7XNg1IQBRcFJVNOeWVyL","_score":1,"_source":{"name":"yu2","age":"27","address":"BJ","job":"dev"}}
{"_index":"t1","_type":"_doc","_id":"7nNg1IQBRcFJVNOeWVyd","_score":1,"_source":{"name":"yu3","age":"26","address":"SD","job":"ops"}}
{"_index":"t1","_type":"_doc","_id":"73Ng1IQBRcFJVNOeWVyq","_score":1,"_source":{"name":"yu4","age":"25","address":"JX","job":"ops"}}
{"_index":"t1","_type":"_doc","_id":"8HNg1IQBRcFJVNOeWVy5","_score":1,"_source":{"name":"jack01","age":"19","address":"YN","job":"test"}}
{"_index":"t1","_type":"_doc","_id":"8XNg1IQBRcFJVNOeWVzD","_score":1,"_source":{"name":"tom02","age":"30","address":"DB","job":"test"}}
{"_index":"t1","_type":"_doc","_id":"8nNg1IQBRcFJVNOeWVzN","_score":1,"_source":{"name":"david03","age":"30","address":"BJ","job":"test"}}
{"_index":"t1","_type":"_doc","_id":"83Ng1IQBRcFJVNOeWVzY","_score":1,"_source":{"name":"xiaohei01","age":"17","address":"BJ","job":"ops"}}
[root yc-k8s /es-nfs-data]#

备份且压缩

elasticdump \
--input=http://10.0.0.18:9200/t2 \
--output=$ \
  | gzip > /es-nfs-data/t2.json.gz


[root yc-k8s /es-nfs-data]#gzip -d t2.json.gz 
[root yc-k8s /es-nfs-data]#ll
total 12
drwxr-xr-x. 3 elasticsearch elasticsearch 4096 Dec  3 04:50 my_backup_location
-rw-r--r--. 1 root          root          1090 Dec  3 22:35 t1.json
-rw-r--r--. 1 root          root           402 Dec  3 22:37 t2.json
[root yc-k8s /es-nfs-data]#cat t2.json 
{"_index":"t2","_type":"_doc","_id":"9HNg1IQBRcFJVNOe-1xE","_score":1,"_source":{"name":"yu1","age":"28","address":"JS","job":"dev"}}
{"_index":"t2","_type":"_doc","_id":"9XNg1IQBRcFJVNOe-1yA","_score":1,"_source":{"name":"yu2","age":"27","address":"BJ","job":"dev"}}
{"_index":"t2","_type":"_doc","_id":"9nNg1IQBRcFJVNOe-1yS","_score":1,"_source":{"name":"yu3","age":"26","address":"SD","job":"ops"}}
[root yc-k8s /es-nfs-data]#

2.3 恢复数据

# 恢复数据，就是input，output反过来

[root yc-k8s /es-nfs-data]#elasticdump \
>  --input=/es-nfs-data/t2.json \
>  --output=http://10.0.0.18:9200/t2
Sat, 03 Dec 2022 14:44:23 GMT | starting dump
Sat, 03 Dec 2022 14:44:23 GMT | got 3 objects from source file (offset: 0)
Sat, 03 Dec 2022 14:44:23 GMT | sent 3 objects to destination elasticsearch, wrote 3
Sat, 03 Dec 2022 14:44:23 GMT | got 0 objects from source file (offset: 3)
Sat, 03 Dec 2022 14:44:23 GMT | Total Writes: 3
Sat, 03 Dec 2022 14:44:23 GMT | dump complete
[root yc-k8s /es-nfs-data]#

2.4 批量备份

# 去掉以点开头的index 
curl -s 10.0.0.18:9200/_cat/indices | awk '{print $3}' | grep -v '^\.'

# 脚本
#!/bin/bash
# author : www.yuchaoit.cn

indexs=$(curl -s 10.0.0.18:9200/_cat/indices | awk '{print $3}' | grep -v '^\.')

for i in $indexs
do
    elasticdump \
    --input=http://10.0.0.18:9200/${i} \
    --output=/es-nfs-data/${i}.json \
    --type=data
done

备份结果

root yc-k8s /es-nfs-data]#
[root yc-k8s /es-nfs-data]#sh for-index.sh 
Sat, 03 Dec 2022 14:54:57 GMT | starting dump
Sat, 03 Dec 2022 14:54:57 GMT | got 3 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:54:57 GMT | sent 3 objects to destination file, wrote 3
Sat, 03 Dec 2022 14:54:57 GMT | got 0 objects from source elasticsearch (offset: 3)
Sat, 03 Dec 2022 14:54:57 GMT | Total Writes: 3
Sat, 03 Dec 2022 14:54:57 GMT | dump complete
Sat, 03 Dec 2022 14:54:57 GMT | starting dump
Sat, 03 Dec 2022 14:54:57 GMT | got 8 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:54:57 GMT | sent 8 objects to destination file, wrote 8
Sat, 03 Dec 2022 14:54:57 GMT | got 0 objects from source elasticsearch (offset: 8)
Sat, 03 Dec 2022 14:54:57 GMT | Total Writes: 8
Sat, 03 Dec 2022 14:54:57 GMT | dump complete
Sat, 03 Dec 2022 14:54:57 GMT | starting dump
Sat, 03 Dec 2022 14:54:57 GMT | got 3 objects from source elasticsearch (offset: 0)
Sat, 03 Dec 2022 14:54:57 GMT | sent 3 objects to destination file, wrote 3
Sat, 03 Dec 2022 14:54:57 GMT | got 0 objects from source elasticsearch (offset: 3)
Sat, 03 Dec 2022 14:54:57 GMT | Total Writes: 3
Sat, 03 Dec 2022 14:54:57 GMT | dump complete
[root yc-k8s /es-nfs-data]#ll
\total 20
-rw-r--r--. 1 root          root           248 Dec  3 22:54 for-index.sh
drwxr-xr-x. 3 elasticsearch elasticsearch 4096 Dec  3 04:50 my_backup_location
-rw-r--r--. 1 root          root           444 Dec  3 22:54 restored_index_2.json
-rw-r--r--. 1 root          root          1090 Dec  3 22:54 t1.json
-rw-r--r--. 1 root          root           402 Dec  3 22:54 t2.json
[root yc-k8s /es-nfs-data]#

2.5 密码认证

es有密码时，备份命令

    elasticdump \
    --input=http://name:[email protected]:9200/t2 \
    --output=/es-nfs-data/t2.json \
    --type=data

2.6 建议用法

1. 想分析es数据，用elasticdump导出json
2. 仅仅备份数据，用es官网工具
3. 恢复es数据，如果数据冲突会被覆盖

3.es安全认证

 

逻辑图

 

官网资料

https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-stack-security.html

 

1.创建证书

1. 创建证书，输入密码
[root@es-node1 ~]#
[root@es-node1 ~]#
[root@es-node1 ~]#/usr/share/elasticsearch/bin/elasticsearch-certutil ca
This tool assists you in the generation of X.509 certificates and certificate
signing requests for use with SSL/TLS in the Elastic stack.

The 'ca' mode generates a new 'certificate authority'
This will create a new X.509 certificate and private key that can be used
to sign certificate when running in 'cert' mode.

Use the 'ca-dn' option if you wish to configure the 'distinguished name'
of the certificate authority

By default the 'ca' mode produces a single PKCS#12 output file which holds:
    * The CA certificate
    * The CA's private key

If you elect to generate PEM format certificates (the -pem option), then the output will
be a zip file containing individual files for the CA certificate and private key

Please enter the desired output file [elastic-stack-ca.p12]: 
Enter password for elastic-stack-ca.p12 : 
[root@es-node1 ~]#
[root@es-node1 ~]#
[root@es-node1 ~]#


[root@es-node1 ~]#file /usr/share/elasticsearch/elastic-stack-ca.p12 
/usr/share/elasticsearch/elastic-stack-ca.p12: data



2.根据ca整数，创建私钥，也可以二次设置密码
/usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12


3.检查
[root@es-node1 /usr/share/elasticsearch]#ll
total 576
drwxr-xr-x  2 root root   4096 Nov 18 23:53 bin
-rw-------  1 root root   3443 Dec  3 23:10 elastic-certificates.p12
-rw-------  1 root root   2527 Dec  3 23:08 elastic-stack-ca.p12
drwxr-xr-x  9 root root    107 Nov 18 23:53 jdk
drwxr-xr-x  3 root root   4096 Nov 18 23:53 lib
-rw-r--r--  1 root root  13675 Sep  2  2020 LICENSE.txt
drwxr-xr-x 52 root root   4096 Nov 18 23:53 modules
-rw-rw-r--  1 root root 544318 Sep  2  2020 NOTICE.txt
drwxr-xr-x  2 root root      6 Sep  2  2020 plugins
-rw-r--r--  1 root root   7007 Sep  2  2020 README.asciidoc


4.复制证书到固定位置，同步es节点

[root@es-node1 /usr/share/elasticsearch]#scp -r /etc/elasticsearch/certs [email protected]:/etc/elasticsearch/
elastic-certificates.p12                                                                                                                                                                                    100% 3443     5.1MB/s   00:00    
elastic-stack-ca.p12                                                                                                                                                                                        100% 2527     3.1MB/s   00:00    
[root@es-node1 /usr/share/elasticsearch]#scp -r /etc/elasticsearch/certs [email protected]:/etc/elasticsearch/
elastic-certificates.p12                                                                                                                                                                                    100% 3443     5.3MB/s   00:00    
elastic-stack-ca.p12                                                                                                                                                                                        100% 2527     2.8MB/s   00:00    
[root@es-node1 /usr/share/elasticsearch]#


5.修改es节点配置文件，开启安全功能
# es安全组件，xpack，开启即可。
# 来自于官网的教程，复制粘贴即可，有机会再阅读es数据，逐步深入学习。



xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: certs/elastic-stack-ca.p12
xpack.security.transport.ssl.truststore.path: certs/elastic-stack-ca.p12

6.注意修改es目录权限，重启进程
chown -R elasticsearch.elasticsearch /etc/elasticsearch/
systemctl restart elasticsearch.service 

7.检查es状态
netstat -tunlp|grep 9200

此时就已连不上了，需要账密了

 

创建账户，密码

默认会给一些列的内置用户，创建密码，统一123123

[root@es-node1 ~]#/usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,kibana_system,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y


Enter password for [elastic]: 
passwords must be at least [6] characters long
Try again.
Enter password for [elastic]: 
Reenter password for [elastic]: 
Enter password for [apm_system]: 
Reenter password for [apm_system]: 
Enter password for [kibana_system]: 
Reenter password for [kibana_system]: 
Enter password for [logstash_system]: 
Reenter password for [logstash_system]: 
Enter password for [beats_system]: 
Reenter password for [beats_system]: 
Enter password for [remote_monitoring_user]: 
Reenter password for [remote_monitoring_user]: 
Changed password for user [apm_system]
Changed password for user [kibana_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]

内置角色解释

账户elastic为elasticsearch超级管理员，拥有所有权限
账户kibana用于kibana组件获取相关信息用于web展示
账户logstash_system用于logstash服务获取elasticsearch的监控数据
账户beats_system用于存储es监控信息时用
账户kibana_system，用于kibana安全与es集群通讯用。


es官网，内置角色
https://www.elastic.co/guide/en/elasticsearch/reference/7.9/built-in-roles.html

2.创建kibana登录用户

 

务必注意，这里kibana使用刚才es创建的 账户，密码，来连接es集群。

密码保证一致，否则kibana，无法和es通信了。

[root@es-node1 ~]#vim /etc/kibana/kibana.yml 

 42 # If your Elasticsearch is protected with basic authentication, these settings provide
 43 # the username and password that the Kibana server uses to perform maintenance on the Kibana
 44 # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
 45 # is proxied through the Kibana server.
 46 elasticsearch.username: "kibana_system"
 47 elasticsearch.password: "123123"


重启kibana
[root@es-node1 ~]#systemctl restart kibana

kibana正确安全连接es

 

elastic
123123

普通用户kibana 123123 也可以登录

es-head登录认证

 

3.创建用户

kibana先创建es的索引

 

便于查询数据。

kibana功能space

 

需求：1.限制开发小于，登录kibana，只能使用某个功能，如discover,且只能看t2 索引

实践：
1. 创建space ，dev，允许只能看某些内容

创建roles

 

创建用户，开发小于，绑定dev角色

 

yu
yu123123

开发小于登录kibana

只能访问t1 index数据

 

4.角色访问index区别

END

官方站点：www.linuxprobe.com

Linux命令大全：www.linuxcool.com

刘遄老师QQ：5604215

Linux技术交流群：2636170

（新群，火热加群中……）

想要学习Linux系统的读者可以点击"阅读原文"按钮来了解书籍《Linux就该这么学》，同时也非常适合专业的运维人员阅读，成为辅助您工作的高价值工具书！


微信扫码关注该文公众号作者

来源：Linux就该这么学

相关新闻

[干货] “中国航天员”的专属单词——宏观市场 | 采矿业、基础工业拉动利润恢复——评2024年6月工业企业利润数据ES备份恢复固定收益 | 恢复的主基调——高速公路2023年报点评2024教育工作怎么干？听听各地教育部门负责人怎么说——建议大家：不管房子大小，装修时“这几样”要买贵的，别不当回事【装修干货】8月社没福利 | 人生教练入门干货开学倒计时！7-12年级KPI与攻克难点干货指南[干货] “皮蛋”的英语是什么？上厕所如何“压水花”？10米跳台的水花，还没你家马桶的大！【装修干货】[干货] in the beginning 和 at the beginning 有何区别？这届年轻夫妻，就流行分房睡！【装修干货】干货! 加拿大出车祸处理步骤| 注意事项| 需要报警吗？ | 保险如何理赔？赞！中科院博士教你用AI写论文！Discussion部分要怎么写……（干货+实操）我认可的装修风水不多，比如这条【装修干货】[干货]“喉结”的英文为啥是 Adam's apple？厨房怎么设计？开放式厨房好不好？看完这篇你就懂了！【装修干货】《2024旧金山首届求职及爬藤高峰论坛》来湾区啦！大咖云集干货满满～赶快报名参加[干货] 为啥 carpet(地毯)还可以表示“训斥”？半开放式设计如何重塑家居空间？【装修干货】求职干货 | MBB 2025秋招已开！海归求职：咨询（战略、管理、人力、IT等）客厅挂画选的好，品味提升少不了【装修干货】这才是设计师该有的收纳思维观！【装修干货】[干货] “文化差异”究竟用 culture shock 还是cultural shock？
logo
联系我们隐私协议©2024 bendi.news
Bendi新闻
Bendi.news刊载任何文章，不代表同意其说法或描述，仅为提供更多信息，也不构成任何建议。文章信息的合法性及真实性由其作者负责，与Bendi.news及其运营公司无关。欢迎投稿，如发现稿件侵权，或作者不愿在本网发表文章，请版权拥有者通知本网处理。