其他分享
首页 > 其他分享> > nomad es

nomad es

作者:互联网

Nomad 部署 Elasticsearch

参考文档 https://itnext.io/elasticsearch-on-nomad-ae685b762779

准备es镜像

FROM centos:7

ENV APP_HOME /opt/elasticsearch
ENV ES_TMPDIR=${APP_HOME}/temp
ENV ES_USER elasticsearch

RUN useradd -ms /bin/bash -d ${APP_HOME} ${ES_USER}

RUN yum install nmap-ncat -y

RUN set -ex \
    && ES_PACK_NAME=elasticsearch-7.8.1 \
    && FILE_NAME=${ES_PACK_NAME}-linux-x86_64.tar.gz \
    && curl -O https://artifacts.elastic.co/downloads/elasticsearch/${FILE_NAME} \
    && tar -zxf ${FILE_NAME} \
    && cp -R ${ES_PACK_NAME}/* ${APP_HOME} \
    && rm -rf ${APP_HOME}/config/elasticsearch.yml \
    && rm -rf ${APP_HOME}/config/jvm.options \
    && rm -rf ${FILE_NAME} \
    && rm -rf ${ES_PACK_NAME}

RUN mkdir -p ${APP_HOME}/temp
RUN ls -al ${APP_HOME}

RUN chmod a+x -R ${APP_HOME}/bin
RUN chown ${ES_USER}: -R ${APP_HOME}

RUN set -ex \
    && echo 'elasticsearch   - nofile 65536' >> /etc/security/limits.d/elasticsearch.conf \
    && echo 'elasticsearch   - nproc  65536' >> /etc/security/limits.d/elasticsearch.conf \
    && echo 'root  soft  memlock unlimited' >> /etc/security/limits.d/elasticsearch.conf \
    && echo 'root  hard  memlock unlimited' >> /etc/security/limits.d/elasticsearch.conf
RUN cat /etc/security/limits.d/elasticsearch.conf

USER ${ES_USER}
WORKDIR ${APP_HOME}
EXPOSE 9200 9300

推送镜像

#登录dockerhub
docker login

#推送
export ES_IMAGE=x602/elasticsearch:7.8.1
docker build . -t ${ES_IMAGE};
docker push ${ES_IMAGE};

在Nomad上部署Elasticsearch

创建命名空间

nomad namespace apply -description "Elasticsearch Cluster" elasticsearch;

创建和注册卷

# 1.2 ceph-csi es

########## register volume: es-master-0
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacae2 --size 2048 --pool myPool --image-feature layering;

## es volume.
cat <<EOF > es-volume-master-0.hcl
type = "csi"
id   = "es-master-0"
name = "es-master-0"
external_id     = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacae2"
capability {
  access_mode     = "single-node-writer"
  attachment_mode = "file-system"
}
mount_options {
  fs_type = "ext4"
}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
  clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
  pool      = "myPool"
  imageFeatures = "layering"
}
EOF

# register volume.
nomad volume register -namespace=elasticsearch es-volume-master-0.hcl;



########## register volume: es-master-1
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacac3 --size 2048 --pool myPool --image-feature layering;

## es volume.
cat <<EOF > es-volume-master-1.hcl
type = "csi"
id   = "es-master-1"
name = "es-master-1"
external_id     = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacac3"
capability {
  access_mode     = "single-node-writer"
  attachment_mode = "file-system"
}
mount_options {
  fs_type = "ext4"
}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
  clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
  pool      = "myPool"
  imageFeatures = "layering"
}
EOF

# register volume.
nomad volume register -namespace=elasticsearch es-volume-master-1.hcl;



########## register volume: es-master-2
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacac4 --size 2048 --pool myPool --image-feature layering;

## es volume.
cat <<EOF > es-volume-master-2.hcl
type = "csi"
id   = "es-master-2"
name = "es-master-2"
external_id     = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacac4"
capability {
  access_mode     = "single-node-writer"
  attachment_mode = "file-system"
}
mount_options {
  fs_type = "ext4"
}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
  clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
  pool      = "myPool"
  imageFeatures = "layering"
}
EOF

# register volume.
nomad volume register -namespace=elasticsearch es-volume-master-2.hcl;




########## register volume: es-data-0
## create an image in the pool of ceph.
sudo rbd create csi-vol-00000000-1111-2222-bbbb-cacacacacaf0 --size 2048 --pool myPool --image-feature layering;

## es volume.
cat <<EOF > es-volume-data-0.hcl
type = "csi"
id   = "es-data-0"
name = "es-data-0"
external_id     = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacaf0"
capability {
  access_mode     = "single-node-writer"
  attachment_mode = "file-system"
}
mount_options {
  fs_type = "ext4"
}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
  clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
  pool      = "myPool"
  imageFeatures = "layering"
}
EOF

# register volume.
nomad volume register -namespace=elasticsearch es-volume-data-0.hcl;



########## register volume: es-data-1
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf1 --size 2048 --image-feature layering;


## es volume.
cat <<EOF > es-volume-data-1.hcl
type = "csi"
id   = "es-data-1"
name = "es-data-1"
external_id     = "0001-0024-ef8394c8-2b17-435d-ae47-9dff271875d1-0000000000000001-00000000-1111-2222-bbbb-cacacacacaf1"
capability {
  access_mode     = "single-node-writer"
  attachment_mode = "file-system"
}
mount_options {
  fs_type = "ext4"
}
plugin_id       = "ceph-csi"
secrets {
  userID  = "admin"
  userKey = "AQBWYPVhs0XZIhAACN0PAHEJTitvq514oVC12A=="
}
context {
  clusterID = "ef8394c8-2b17-435d-ae47-9dff271875d1"
  pool      = "myPool"
  imageFeatures = "layering"
}
EOF

# register volume.
nomad volume register -namespace=elasticsearch es-volume-data-1.hcl;

检查状态

nomad volume status -namespace elasticsearch;
Container Storage Interface
ID           Name         Plugin ID  Schedulable  Access Mode
es-data-0    es-data-0    ceph-csi   true         single-node-writer
es-data-1    es-data-1    ceph-csi   true         single-node-writer
es-master-0  es-master-0  ceph-csi   true         single-node-writer
es-master-1  es-master-1  ceph-csi   true         single-node-writer
es-master-2  es-master-2  ceph-csi   true         single-node-writer

job

# 1.2 ceph 7.8
job "elasticsearch" {
  namespace = "elasticsearch"
  datacenters = ["dc1"]
  type        = "service"
  update {
    max_parallel     = 1
    health_check     = "checks"
    min_healthy_time = "30s"
    healthy_deadline = "5m"
    auto_revert      = true
    canary           = 0
    stagger          = "30s"
  }
  ##################################### master-0 ############################################
  group "master-0" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-master-0"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
  initial_master_nodes:
    - {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
node:
  name: es-master-0
  master: true
  data: false
  ingest: false
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-master-0-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }
  ##################################### master-1 ############################################
  group "master-1" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-master-1"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "await-es-master-0-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
  initial_master_nodes:
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
node:
  name: es-master-1
  master: true
  data: false
  ingest: false
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-master-1-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }
  ##################################### master-2 ############################################
  group "master-2" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-master-2"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "await-es-master-0-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-1-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
  initial_master_nodes:
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
node:
  name: es-master-2
  master: true
  data: false
  ingest: false
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ env "NOMAD_IP_communication" }}:{{ env "NOMAD_HOST_PORT_communication" }}
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-master-2-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }
  ##################################### data-0 ############################################
  group "data-0" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-data-0"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "await-es-master-0-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-1-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-2-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
node:
  name: es-data-0
  master: false
  data: true
  ingest: true
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-data-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }
  ##################################### data-1 ############################################
  group "data-1" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-data-1"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "await-es-master-0-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-1-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-2-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
node:
  name: es-data-1
  master: false
  data: true
  ingest: true
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-data-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }

##################################### add data-2 ############################################
  group "data-2" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-data-2"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "await-es-master-0-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-1-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-2-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
node:
  name: es-data-2
  master: false
  data: true
  ingest: true
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-data-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }

##################################### add data-3 ############################################
  group "data-3" {
    count = 1
    restart {
      attempts = 3
      delay = "30s"
      interval = "5m"
      mode = "fail"
    }
    network {
      port "request" {
      }
      port "communication" {
      }
    }
    volume "ceph-volume" {
      type = "csi"
      read_only = false
      source = "es-data-3"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }
    task "await-es-master-0-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-0-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-1-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-1-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "await-es-master-2-comm" {
      driver = "docker"
      config {
        image        = "busybox:1.28"
        command      = "sh"
        args         = ["-c", "echo -n 'Waiting for service'; until nslookup es-master-2-comm.service.consul 2>&1 >/dev/null; do echo '.'; sleep 2; done"]
        network_mode = "host"
      }
      resources {
        cpu    = 200
        memory = 128
      }
      lifecycle {
        hook    = "prestart"
        sidecar = false
      }
    }
    task "elasticsearch" {
      driver = "docker"
      kill_timeout = "300s"
      kill_signal = "SIGTERM"
      volume_mount {
        volume      = "ceph-volume"
        destination = "/srv"
        read_only   = false
      }
      env {
        ES_TMPDIR = "/opt/elasticsearch/temp"
      }
      template {
        data = <<EOF
cluster:
  name: my-cluster
  publish:
    timeout: 300s
  join:
    timeout: 300s
node:
  name: es-data-3
  master: false
  data: true
  ingest: true
network:
  host: 0.0.0.0
discovery:
  seed_hosts:
    - {{ range service "es-master-0-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-1-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
    - {{ range service "es-master-2-comm" }}{{ .Address }}:{{ .Port }}{{ end }}
path:
  data:
    - /srv/data
  logs: /srv/log
bootstrap.memory_lock: true
indices.query.bool.max_clause_count: 10000
EOF
        destination = "local/elasticsearch.yml"
      }
      template {
        data = <<EOF
-Xms512m
-Xmx512m
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
-Djava.io.tmpdir=${ES_TMPDIR}
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=data
-XX:ErrorFile=logs/hs_err_pid%p.log
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:logs/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
EOF
        destination = "local/jvm.options"
      }
      config {
        image = "x602/elasticsearch:7.8.1"
        force_pull = false
        volumes = [
          "./local/elasticsearch.yml:/opt/elasticsearch/config/elasticsearch.yml",
          "./local/jvm.options:/opt/elasticsearch/config/jvm.options"
        ]
        command = "bin/elasticsearch"
        args = [
          "-Enetwork.publish_host=${NOMAD_IP_request}",
          "-Ehttp.publish_port=${NOMAD_HOST_PORT_request}",
          "-Ehttp.port=${NOMAD_PORT_request}",
          "-Etransport.publish_port=${NOMAD_HOST_PORT_communication}",
          "-Etransport.tcp.port=${NOMAD_PORT_communication}"
        ]
        ports = [
          "request",
          "communication"
        ]
        ulimit {
          memlock = "-1"
          nofile = "65536"
          nproc = "65536"
        }
      }
      resources {
        cpu = 100
        memory = 1024
      }
      service {
        name = "es-req"
        port = "request"
        check {
          name = "rest-tcp"
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
        check {
          name     = "rest-http"
          type     = "http"
          path     = "/"
          interval = "5s"
          timeout  = "4s"
        }
      }
      service {
        name = "es-data-comm"
        port = "communication"
        check {
          type = "tcp"
          interval = "10s"
          timeout = "2s"
        }
      }
    }
  }
}

获取服务中所有节点的 IP 地址和端口。

cat <<EOF > service.tpl
{{ range service "es-req" }}
  - {{ .Address }}:{{ .Port }}{{ end }}
EOF

cat <<EOF > service.tpl
{{ range service "es-req" }}
  - {{ .Address }}:{{ .Port }}{{ end }}
EOF

consul-template -template service.tpl -dry;
> 
  - 10.103.3.41:20102
  - 10.103.3.41:28131
  - 10.103.3.41:29898
  - 10.103.3.41:28413
  - 10.103.3.41:28185
  - 10.103.3.42:27690
  - 10.103.3.42:29942

验证

curl http://10.103.3.41:20102/_cluster/health?pretty
{
  "cluster_name" : "my-cluster",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 7,
  "number_of_data_nodes" : 4,
  "active_primary_shards" : 5,
  "active_shards" : 5,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

进行 基准测试

https://esrally.readthedocs.io/en/stable/docker.html

# target-hosts 写3主节点ip(考虑是否写数据节点ip)
docker run --rm   elastic/rally race --track=geonames  --pipeline=benchmark-only --challenge=append-no-conflicts --target-hosts=10.103.3.42:29942,10.103.3.41:28413,10.103.3.42:27690

#docker run --rm  -v $PWD/myrally:/rally/.rally  elastic/rally race --track=geonames  --pipeline=benchmark-only --challenge=append-no-conflicts --target-hosts=10.103.3.42:29942,10.103.3.41:28413,10.103.3.42:27690
tailf myrally/logs/rally.log 查看日志

#其他测试
docker run   elastic/rally race --track=nyc_taxis  --test-mode --pipeline=benchmark-only --target-hosts=10.103.3.40:24497

rbd --pool myPool list

create an image in the pool of ceph.

sudo rbd --pool myPool resize csi-vol-00000000-1111-2222-bbbb-cacacacacaf1 --size 20480 #调整大小

删除 rbd

sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacac3
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacac4
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacae2
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf0
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf1
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf2
sudo rbd --pool myPool rm csi-vol-00000000-1111-2222-bbbb-cacacacacaf3

创建

sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacac3 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacac4 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacae2 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf0 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf1 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf2 --size 204800 --image-feature layering;
sudo rbd --pool myPool create csi-vol-00000000-1111-2222-bbbb-cacacacacaf3 --size 204800 --image-feature layering;

标签:csi,--,volume,nomad,elasticsearch,master,es
来源: https://www.cnblogs.com/x602/p/15872274.html