Elasticsearch压测工具esrally

Elasticsearch压测工具esrally

配置参数 配置值
GIT_HOME /opt/git
PYENV_HOME /opt/pyenv

环境要求

python3.8 pip3
jdk8(可选)
git 1.9+

环境变量信息

# 系统环境变量
cat << EOF >> /etc/profile.d/01-esrally.sh
export GIT_HOME=/opt/git
export PYENV_HOME=/opt/pyenv
EOF

git部署

使用安装包安装:

#1.下载安装包:https://github.com/git/git/releases
tar -zxvf git-2.9.5.tar.gz
#2:安装git需要的依赖
yum install curl-devel expat-devel gettext-devel openssl-devel zlib-devel gcc perl-ExtUtils-MakeMaker
#3:如果已经安装编译源码所需依赖的时候,yum自动安装了git,需要先卸载这个旧版git
yum -y remove git
#4:编译git源码:
make prefix=${GIT_HOME} all
#5:安装git至/opt/git路径
make prefix=${GIT_HOME} install
#5:环境变量配置
cat << EOF >> /etc/profile
export PATH=$PATH:${GIT_HOME}/bin
EOF
source /etc/profile
#6:验证
git --version

#================================================================================================================
#需要离线部署情况
#1:安装pyenv需要的依赖
yum -y install openssl-devel zlib-devel bzip2-devel sqlite-devel readline-devel libffi-devel systemtap-sdt-devel curl-devel perl-devel
#2.git clone项目
git clone https://github.com/pyenv/pyenv.git
#3.环境变量配置
cat << EOF >> /etc/profile.d/pyenv.sh
export PYENV_ROOT="${PYENV_HOME}"
export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"
EOF
#4.virtualenv环境部署
git clone https://github.com/pyenv/pyenv-virtualenv.git /opt/pyenv/plugins/pyenv-virtualenv
#5.环境创建 由于在线速度较慢,使用离线部署
mv Python-3.8.8.tar.xz -P ${PYENV_HOME}/cache
pyenv install 3.8.8
pyenv virtualenv 3.8.8 python_3.8.8

pyenv部署

#1:安装pyenv需要的依赖
yum -y install openssl-devel zlib-devel bzip2-devel sqlite-devel readline-devel libffi-devel systemtap-sdt-devel curl-devel perl-devel
#2.git clone项目
git clone https://github.com/pyenv/pyenv.git
#3.环境变量配置
cat << EOF >> /etc/profile.d/pyenv.sh
export PYENV_ROOT="${PYENV_HOME}"
export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"
EOF
#4.virtualenv环境部署
git clone https://github.com/pyenv/pyenv-virtualenv.git /opt/pyenv/plugins/pyenv-virtualenv
#5.环境创建 由于在线速度较慢,使用离线部署
mv Python-3.8.8.tar.xz -P ${PYENV_HOME}/cache
pyenv install 3.8.8
pyenv virtualenv 3.8.8 python_3.8.8

#================================================================================================================
#需要离线部署情况
#进入pyenv环境
pyenv activate python_3.8.8
#pyenv安装
pip3 install --upgrade pip
pip3 install esrally
#离线测试 数据包部署
nv ~/.rally/rally.ini ~/.rally/rally.ini.bk
cat << EOF >> ~/.rally/rally.ini
[meta]
config.version = 17

[system]
env.name = local

[node]
root.dir = $(eval echo ~$user)/.rally/benchmarks
src.root.dir = $(eval echo ~$user)/.rally/benchmarks/src

[source]
remote.repo.url = https://github.com/elastic/elasticsearch.git
elasticsearch.src.subdir = elasticsearch

[benchmarks]
local.dataset.cache = $(eval echo ~$user)/.rally/benchmarks/data

[reporting]
datastore.type = in-memory
datastore.host =
datastore.port = 
datastore.secure = False
datastore.user =
datastore.password =

[tracks]
#default.url = https://github.com/elastic/rally-tracks
default.url = $(eval echo ~$user)/.rally/benchmarks/tracks/default

[teams]
#default.url = https://github.com/elastic/rally-teams
default.url = $(eval echo ~$user)/.rally/benchmarks/teams/defaul

[defaults]
preserve_benchmark_candidate = false

[distributions]
release.cache = true
EOF
#数据集下载
#以geonames为例
#访问https://github.com/elastic/rally-tracks/blob/master/download.sh
#可看到readonly URL="http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora"
#通过查看https://github.com/elastic/rally-tracks/blob/master/geonames/files.txt文件,该数据集名为 documents-2.json.bz2 和 documents-2-1k.json.bz2
#尝试进行组合
#http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames/documents-2.json.bz2
wget http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames/documents-2.json.bz2
mv documents-2.json.bz2 ~/.rally/benchmarks/data/geonames
#离线测试
#ES集群配置xpath情况下
esrally race --pipeline=benchmark-only --target-hosts=10.0.131.8:9200,10.0.131.9:9200,10.0.131.10:9200 --offline --track=geonames --client-options="use_ssl:false,verify_certs:false,basic_auth_user:'elastic',basic_auth_password:'Changingit124'" --challenge=append-no-conflicts-index-only
#ES集群未配置xpath情况下
esrally race --pipeline=benchmark-only --target-hosts=10.0.131.8:9200,10.0.131.9:9200,10.0.131.10:9200 --offline --track=geonames --challenge=append-no-conflicts-index-only

esrally部署测试

#进入pyenv环境
pyenv activate python_3.8.8
#pyenv安装
pip3 install --upgrade pip
pip3 install esrally
#进行测试
#ES集群配置xpath情况下
esrally race --pipeline=benchmark-only --target-hosts=10.0.131.8:9200,10.0.131.9:9200,10.0.131.10:9200 --track=geonames --client-options="use_ssl:false,verify_certs:false,basic_auth_user:'elastic',basic_auth_password:'Changingit124'" --challenge=append-no-conflicts-index-only
#ES集群未配置xpath情况下
esrally race --pipeline=benchmark-only --target-hosts=10.0.131.8:9200,10.0.131.9:9200,10.0.131.10:9200 --track=geonames --challenge=append-no-conflicts-index-only
#参数报表
------------------------------------------------------
    _______             __   _____
   / ____(_)___  ____ _/ /  / ___/_________  ________
  / /_  / / __ \/ __ `/ /   \__ \/ ___/ __ \/ ___/ _ \
 / __/ / / / / / /_/ / /   ___/ / /__/ /_/ / /  /  __/
/_/   /_/_/ /_/\__,_/_/   /____/\___/\____/_/   \___/
------------------------------------------------------

|                                                        Metric |         Task |     Value |   Unit |
|--------------------------------------------------------------:|-------------:|----------:|-------:|
|                    Cumulative indexing time of primary shards |              |   25.5145 |    min |
|            Min cumulative indexing time across primary shards |              |         0 |    min |
|         Median cumulative indexing time across primary shards |              |         0 |    min |
|            Max cumulative indexing time across primary shards |              |   5.30113 |    min |
|           Cumulative indexing throttle time of primary shards |              |         0 |    min |
|   Min cumulative indexing throttle time across primary shards |              |         0 |    min |
|Median cumulative indexing throttle time across primary shards |              |         0 |    min |
|   Max cumulative indexing throttle time across primary shards |              |         0 |    min |
|                       Cumulative merge time of primary shards |              |   1.47403 |    min |
|                      Cumulative merge count of primary shards |              |        15 |        |
|               Min cumulative merge time across primary shards |              |         0 |    min |
|            Median cumulative merge time across primary shards |              |         0 |    min |
|               Max cumulative merge time across primary shards |              |   1.40015 |    min |
|              Cumulative merge throttle time of primary shards |              |   0.15585 |    min |
|      Min cumulative merge throttle time across primary shards |              |         0 |    min |
|   Median cumulative merge throttle time across primary shards |              |         0 |    min |
|      Max cumulative merge throttle time across primary shards |              |   0.15585 |    min |
|                     Cumulative refresh time of primary shards |              |   7.72502 |    min |
|                    Cumulative refresh count of primary shards |              |       324 |        |
|             Min cumulative refresh time across primary shards |              |         0 |    min |
|          Median cumulative refresh time across primary shards |              |         0 |    min |
|             Max cumulative refresh time across primary shards |              |    1.9067 |    min |
|                       Cumulative flush time of primary shards |              |     0.589 |    min |
|                      Cumulative flush count of primary shards |              |        53 |        |
|               Min cumulative flush time across primary shards |              |         0 |    min |
|            Median cumulative flush time across primary shards |              |         0 |    min |
|               Max cumulative flush time across primary shards |              |  0.179733 |    min |
|                                       Total Young Gen GC time |              |    15.772 |      s |
|                                      Total Young Gen GC count |              |       185 |        |
|                                         Total Old Gen GC time |              |         0 |      s |
|                                        Total Old Gen GC count |              |         0 |        |
|                                                    Store size |              |   4.16667 |     GB |
|                                                 Translog size |              | 0.0390588 |     GB |
|                                        Heap used for segments |              |   2.19177 |     MB |
|                                      Heap used for doc values |              |  0.726276 |     MB |
|                                           Heap used for terms |              |   1.24084 |     MB |
|                                           Heap used for norms |              | 0.0759888 |     MB |
|                                          Heap used for points |              |         0 |     MB |
|                                   Heap used for stored fields |              |  0.148659 |     MB |
|                                                 Segment count |              |       271 |        |
|                                                Min Throughput | index-append |    6753.2 | docs/s |
|                                               Mean Throughput | index-append |   6968.04 | docs/s |
|                                             Median Throughput | index-append |   6860.72 | docs/s |
|                                                Max Throughput | index-append |    7344.1 | docs/s |
|                                       50th percentile latency | index-append |   5298.43 |     ms |
|                                       90th percentile latency | index-append |   8082.31 |     ms |
|                                       99th percentile latency | index-append |   12970.7 |     ms |
|                                     99.9th percentile latency | index-append |     15998 |     ms |
|                                      100th percentile latency | index-append |   16156.2 |     ms |
|                                  50th percentile service time | index-append |   5298.43 |     ms |
|                                  90th percentile service time | index-append |   8082.31 |     ms |
|                                  99th percentile service time | index-append |   12970.7 |     ms |
|                                99.9th percentile service time | index-append |     15998 |     ms |
|                                 100th percentile service time | index-append |   16156.2 |     ms |
|                                                    error rate | index-append |         0 |      % |

#报告指标介绍
Cumulative indexing time of primary shards:主分片的累计索引时间,用于索引的累积时间
Cumulative indexing time across primary shards:跨主分片的累计索引时间,跨主碎片建立索引的最小、中值和最大累计时间
Cumulative indexing throttle time across primary shards:跨主分片的累计索引节流时间,已限制索引的累积时间
Cumulative merge time of primary shards:主分片的累计合并时间
Cumulative merge count of primary shards:主分片的累计合并个数
Cumulative merge time across primary shards:跨主分片的累计合并时间
Cumulative merge throttle time of primary shards:跨主分片的累计节流时间
Cumulative refresh time of primary shards:主分片的累计刷新时间
Cumulative refresh count of primary shards:主分片的累计刷新次数
Cumulative refresh time across primary shards:跨主分片的累计刷新时间
Cumulative flush time of primary shards:主分片的累计冲洗时间
Cumulative flush count of primary shards:主分片的累计冲洗次数
Cumulative flush time across primary shards:跨主分片的累计冲洗时间
Total Young Gen GC:整个集群中Young Gen GC的总运行时间
Total Old Gen GC:整个集群中Old Gen GC的总运行时间
Store size:索引大小(不包括事务日志),以字节为单位
Translog size:事务日志的大小(以字节为单位)
Index size:索引大小
Total written:总共写入
Heap used for *:堆用于相应项目的字节数
Segment count:细分总数
Throughput:吞吐量
Latency:延迟,提交请求到接收完整响应之间的时间段。它还包括等待时间,即请求花费等待直到准备好由Elasticsearch服务的时间。
Service Time:服务时间,ES处理请求的时间
error rate:错误率,错误响应相对于响应总数的比率。Python Elasticsearch客户端抛出的任何异常都被认为是错误的(例如HTTP响应代码4xx,5xx或网络错误(网络不可达))
点赞
  1. 鸟叔说道:
    Google Chrome Windows 7
    鸟叔来串门,通过虫洞穿梭至此,期待回访!

鸟叔进行回复 取消回复

电子邮件地址不会被公开。必填项已用 * 标注