nomad es
作者：互联网
Nomad 部署 Elasticsearch
参考文档 https://itnext.io/elasticsearch-on-nomad-ae685b762779
准备es镜像
FROM centos:7
ENV APP_HOME /opt/elasticsearch
ENV ES_TMPDIR=${APP_HOME}/temp
ENV ES_USER elasticsearch
RUN useradd -ms /bin/bash -d ${APP_HOME} ${ES_USER}
RUN yum install nmap-ncat -y
RUN set -ex \
&& ES_PACK_NAME=elasticsearch-7.8.1 \
&& FILE_NAME=${ES_PACK_NAME}-linux-x86_64.tar.gz \
&& curl -O https://artifacts.elastic.co/downloads/elasticsearch/${FILE_NAME} \
&& tar -zxf ${FILE_NAME} \
&& cp -R ${ES_PACK_NAME}/* ${APP_HOME} \
&& rm -rf ${APP_HOME}/config/elasticsearch.yml \
&& rm -rf ${APP_HOME}/config/jvm.options \
&& rm -rf ${FILE_NAME} \
&& rm -rf ${ES_PACK_NAME}
RUN mkdir -p ${APP_HOME}/temp
RUN ls -al ${APP_HOME}
RUN chmod a+x -R ${APP_HOME}/bin
RUN chown ${ES_USER}: -R ${APP_HOME}
RUN set -ex \
&& echo 'elasticsearch - nofile 65536' >> /etc/security/limits.d/elasticsearch.conf \
&& echo 'elasticsearch - nproc 65536' >> /etc/security/limits.d/elasticsearch.conf \
&& echo 'root soft memlock unlimited' >> /etc/security/limits.d/elasticsearch.conf \
&& echo 'root hard memlock unlimited' >> /etc/security/limits.d/elasticsearch.conf
RUN cat /etc/security/limits.d/elasticsearch.conf
USER ${ES_USER}
WORKDIR ${APP_HOME}
EXPOSE 9200 9300
推送镜像
#登录dockerhub
docker login
#推送
export ES_IMAGE=x602/elasticsearch:7.8.1
docker build . -t ${ES_IMAGE};
docker push ${ES_IMAGE};
在Nomad上部署Elasticsearch
创建命名空间
nomad namespace apply -description "Elasticsearch Cluster" elasticsearch;
创建和注册卷
# 1.2 ceph-csi es
########## register volume: es-master-0
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacae2 --size 2048 --pool myPool --image-feature layering;
## es volume.
cat <<EOF > es-volume-master-0.hcl
type = "csi"
id = "es-master-0"
name = "es-master-0"
external_id = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacae2"
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
mount_options {
fs_type = "ext4"
}
plugin_id = "ceph-csi"
secrets {
userID = "admin"
userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
pool = "myPool"
imageFeatures = "layering"
}
EOF
# register volume.
nomad volume register -namespace=elasticsearch es-volume-master-0.hcl;
########## register volume: es-master-1
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacac3 --size 2048 --pool myPool --image-feature layering;
## es volume.
cat <<EOF > es-volume-master-1.hcl
type = "csi"
id = "es-master-1"
name = "es-master-1"
external_id = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacac3"
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
mount_options {
fs_type = "ext4"
}
plugin_id = "ceph-csi"
secrets {
userID = "admin"
userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
pool = "myPool"
imageFeatures = "layering"
}
EOF
# register volume.
nomad volume register -namespace=elasticsearch es-volume-master-1.hcl;
########## register volume: es-master-2
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacac4 --size 2048 --pool myPool --image-feature layering;
## es volume.
cat <<EOF > es-volume-master-2.hcl
type = "csi"
id = "es-master-2"
name = "es-master-2"
external_id = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacac4"
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
mount_options {
fs_type = "ext4"
}
plugin_id = "ceph-csi"
secrets {
userID = "admin"
userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
pool = "myPool"
imageFeatures = "layering"
}
EOF
# register volume.
nomad volume register -namespace=elasticsearch es-volume-master-2.hcl;
########## register volume: es-data-0
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacaf0 --size 2048 --pool myPool --image-feature layering;
## es volume.
cat <<EOF > es-volume-data-0.hcl
type = "csi"
id = "es-data-0"
name = "es-data-0"
external_id = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacaf0"
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
mount_options {
fs_type = "ext4"
}
plugin_id = "ceph-csi"
secrets {
userID = "admin"
userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
pool = "myPool"
imageFeatures = "layering"
}
EOF
# register volume.
nomad volume register -namespace=elasticsearch es-volume-data-0.hcl;
########## register volume: es-data-1
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf1 --size 2048 --image-feature layering;
## es volume.
cat <<EOF > es-volume-data-1.hcl
type = "csi"
id = "es-data-1"
name = "es-data-1"
external_id = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacaf1"
capability {
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
mount_options {
fs_type = "ext4"
}
plugin_id = "ceph-csi"
secrets {
userID = "admin"
userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
pool = "myPool"
imageFeatures = "layering"
}
EOF
# register volume.
nomad volume register -namespace=elasticsearch es-volume-data-1.hcl;
检查状态
nomad volume status -namespace elasticsearch;
Container Storage Interface
ID Name Plugin ID Schedulable Access Mode
es-data-0 es-data-0 ceph-csi true single-node-writer
es-data-1 es-data-1 ceph-csi true single-node-writer
es-master-0 es-master-0 ceph-csi true single-node-writer
es-master-1 es-master-1 ceph-csi true single-node-writer
es-master-2 es-master-2 ceph-csi true single-node-writer
job
# 1.2 ceph 7.8
job "elasticsearch" {
namespace = "elasticsearch"
datacenters = ["dc1"]
type = "service"
update {
max_parallel = 1
health_check = "checks"
min_healthy_time = "30s"
healthy_deadline = "5m"
auto_revert = true
canary = 0
stagger = "30s"
}
##################################### master-0 ############################################
group "master-0" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-master-0"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
initial_master_nodes:
- {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
node:
name: es-master-0
master: true
data: false
ingest: false
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-master-0-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
##################################### master-1 ############################################
group "master-1" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-master-1"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "await-es-master-0-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
initial_master_nodes:
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
node:
name: es-master-1
master: true
data: false
ingest: false
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-master-1-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
##################################### master-2 ############################################
group "master-2" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-master-2"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "await-es-master-0-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-1-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
initial_master_nodes:
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
node:
name: es-master-2
master: true
data: false
ingest: false
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-master-2-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
##################################### data-0 ############################################
group "data-0" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-data-0"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "await-es-master-0-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-1-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-2-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
node:
name: es-data-0
master: false
data: true
ingest: true
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-data-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
##################################### data-1 ############################################
group "data-1" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-data-1"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "await-es-master-0-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-1-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-2-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
node:
name: es-data-1
master: false
data: true
ingest: true
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-data-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
##################################### add data-2 ############################################
group "data-2" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-data-2"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "await-es-master-0-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-1-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-2-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
node:
name: es-data-2
master: false
data: true
ingest: true
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-data-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
##################################### add data-3 ############################################
group "data-3" {
count = 1
restart {
attempts = 3
delay = "30s"
interval = "5m"
mode = "fail"
}
network {
port "request" {
}
port "communication" {
}
}
volume "ceph-volume" {
type = "csi"
read_only = false
source = "es-data-3"
access_mode = "single-node-writer"
attachment_mode = "file-system"
}
task "await-es-master-0-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-1-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "await-es-master-2-comm" {
driver = "docker"
config {
image = "busybox:1.28"
command = "sh"
args = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
network_mode = "host"
}
resources {
cpu = 200
memory = 128
}
lifecycle {
hook = "prestart"
sidecar = false
}
}
task "elasticsearch" {
driver = "docker"
kill_timeout = "300s"
kill_signal = "SIGTERM"
volume_mount {
volume = "ceph-volume"
destination = "/srv"
read_only = false
}
env {
ES_TMPDIR = "/opt/elasticsearch/temp"
}
template {
data = <<EOF
cluster:
name: my-cluster
publish:
timeout: 300s
join:
timeout: 300s
node:
name: es-data-3
master: false
data: true
ingest: true
network:
host: 0.0.0.0
discovery:
seed_hosts:
- {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
- {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
data:
- /srv/data
logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
destination = "local/elasticsearch.yml"
}
template {
data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
destination = "local/jvm.options"
}
config {
image = "x602/elasticsearch:7.8.1"
force_pull = false
volumes = [
"./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
"./local/jvm.options:/opt/elasticsearch/config/jvm.options"
]
command = "bin/elasticsearch"
args = [
"-Enetwork.publish_host=${NOMAD_IP_request}",
"-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
"-Ehttp.port=${NOMAD_PORT_request}",
"-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
"-Etransport.tcp.port=${NOMAD_PORT_communication}"
]
ports = [
"request",
"communication"
]
ulimit {
memlock = "-1"
nofile = "65536"
nproc = "65536"
}
}
resources {
cpu = 100
memory = 1024
}
service {
name = "es-req"
port = "request"
check {
name = "rest-tcp"
type = "tcp"
interval = "10s"
timeout = "2s"
}
check {
name = "rest-http"
type = "http"
path = "/"
interval = "5s"
timeout = "4s"
}
}
service {
name = "es-data-comm"
port = "communication"
check {
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
}
获取服务中所有节点的 IP 地址和端口。
cat <<EOF > service.tpl
{{ range service "es-req" }}
- {{ .Address }}:{{ .Port }}{{ end }}
EOF
cat <<EOF > service.tpl
{{ range service "es-req" }}
- {{ .Address }}:{{ .Port }}{{ end }}
EOF
consul-template -template service.tpl -dry;
>
- 10.103.3.41:20102
- 10.103.3.41:28131
- 10.103.3.41:29898
- 10.103.3.41:28413
- 10.103.3.41:28185
- 10.103.3.42:27690
- 10.103.3.42:29942
验证
curl http://10.103.3.41:20102/_cluster/health?pretty
{
"cluster_name" : "my-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 7,
"number_of_data_nodes" : 4,
"active_primary_shards" : 5,
"active_shards" : 5,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
进行 基准测试
https://esrally.readthedocs.io/en/stable/docker.html
# target-hosts 写3主节点ip（考虑是否写数据节点ip）
docker run --rm elastic/rally race --track=geonames --pipeline=benchmark-only --challenge=append-no-conflicts --target-hosts=10.103.3.42:29942,10.103.3.41:28413,10.103.3.42:27690
#docker run --rm -v $PWD/myrally:/rally/.rally elastic/rally race --track=geonames --pipeline=benchmark-only --challenge=append-no-conflicts --target-hosts=10.103.3.42:29942,10.103.3.41:28413,10.103.3.42:27690
tailf myrally/logs/rally.log 查看日志
#其他测试
docker run elastic/rally race --track=nyc_taxis --test-mode --pipeline=benchmark-only --target-hosts=10.103.3.40:24497
rbd --pool myPool list
create an image in the pool of ceph.
sudo rbd --pool myPool resize csi-vol-00000000-1111-2222-bbbb-cacacacacaf1 --size 20480 #调整大小
删除 rbd
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacac3
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacac4
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacae2
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf0
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf1
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf2
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf3
创建
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacac3 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacac4 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacae2 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf0 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf1 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf2 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf3 --size 204800 --image-feature layering;
标签：csi,--,volume,nomad,elasticsearch,master,es 来源： https://www.cnblogs.com/x602/p/15872274.html