13.prometheus監控tengine（無用）

一、環境準備

1.1 docker安裝tengine帶nginx-module-vts模組(二選一)

mkdir /data/ -p
cd /data/

# 透過git clone下載已經建立好的docker-compose.yaml檔案
 git clone https://gitee.com/linge365/docker-compose.git

version: '3'
services:
  tengine:
    image: registry.cn-shenzhen.aliyuncs.com/linge365/tengine:2.4.0-geoip2
    container_name: tengine               # 容器名,相當於docker run命令中的--name
    restart: unless-stopped               # 之前是什麼狀態，docker重啟後，保持之前的狀態（如果之前是stop，那docker重啟時，也是stop狀態）
    volumes:                              # 資料卷掛載路徑設定,將本機目錄對映到容器目錄,相當於docker run命令中的-v
      - ./conf/nginx.conf:/etc/nginx/nginx.conf
      - ./conf/conf.d:/etc/nginx/conf.d
      - ./html:/usr/share/nginx/html
      - ./log:/var/log/nginx
      - ./cert:/etc/nginx/cert
      #- ./geoip2:/etc/nginx/geoip2
    environment:                        # 設定環境變數,相當於docker run命令中的-e
      TZ: Asia/Shanghai
      LANG: en_US.UTF-8
    ports:                              # 對映埠，相當於docker run 命令中的-p
      - "80:80"
      - "443:443"

cd docker-compose/tengine
docker-compose up -d

1.2 編譯安裝的tengine安裝nginx-module-vts模組（二選一）

# 如果tengine是自行編譯安裝的，安裝nginx-module-vts模組，如下
#下載nginx-module-vts外掛

cd /usr/local/src/
git clone git://github.com/vozlt/nginx-module-vts.git

# 重新編譯nginx，指定nginx-module-vts目錄(編譯前記得備份原來的tengine目錄)
cd /usr/local/src/tengine-2.4.0  #進入到tengine原始碼目錄
./configure  ...  --add-module=/usr/local/src/nginx-module-vts
make
make install

二、監控tengine

2.1 準備tengine環境

tengine開啟status(docker安裝的tengine)

注：
●監控tengine我將使用第三方nginx-module-vts模組。
檢查是否安裝有nginx-module-vts模組，nginx-module-vts模組已在上面的環境搭建安裝好。

[root@test tengine]# docker exec -it tengine nginx -V 2>&1 | grep -o nginx-module-vts
nginx-module-vts

修改配置檔案

cd /data/tengine/docker-compose/tengine
vim conf/nginx.conf
# 在http標籤中加入vhost_traffic_status_zone配置
http {
    vhost_traffic_status_zone;

    ...
}

vim conf/nginx_prod.conf
http {
    vhost_traffic_status_zone;

    ...

    server {

        ...

        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }
}

vim conf/conf.d/default.conf
    server {

        ...

        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }

# 檢查配置
[root@test tengine]# docker exec -it tengine nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

# 重啟
docker restart tengine

# web檢查
curl http://192.168.11.62/status/format/json
http://192.168.10.100/status

2.2 二進位制安裝nginx-vts-exporter(二選一)

nginx-vts-exporter下載地址: https://github.com/hnlq715/nginx-vts-exporter/releases

wget https://github.com/hnlq715/nginx-vts-exporter/releases/download/v0.10.8/nginx-vtx-exporter_0.10.8_linux_amd64.tar.gz

mkdir /opt/prometheus/nginx_vtx_exporter -p
tar xvf nginx-vtx-exporter_0.10.8_linux_amd64.tar.gz -C /opt/prometheus/nginx_vtx_exporter
ls -l /opt/prometheus/nginx_vtx_exporter

# 建立使用者，修改資料夾許可權
useradd -M -s /usr/sbin/nologin prometheus
chown prometheus:prometheus -R /opt/prometheus

# 建立systemd服務
cat > /etc/systemd/system/nginx_vtx_exporter.service <<"EOF"
[Unit]
Description=nginx_vtx_exporter
After=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
Restart=always
ExecStart=/opt/prometheus/nginx_vtx_exporter/nginx-vtx-exporter -nginx.scrape_uri=http://192.168.11.62/status/format/json

[Install]
WantedBy=multi-user.target
EOF

# 啟動服務
systemctl daemon-reload
systemctl start nginx_vtx_exporter.service
systemctl enable nginx_vtx_exporter.service

2.3 docker安裝nginx-vts-exporter

docker-compose方式

mkdir /data/nginx_vts_exporter

cd /data/nginx_vts_exporter

cat >docker-compose.yaml <<EOF
version: '3.3'
services:
  nginx_vts_exporter:
    image: sophos/nginx-vts-exporter:latest
    container_name: nginx_vts_exporter
    environment:
      NGINX_STATUS: http://192.168.11.62/status/format/json
    restart: always
    ports:
      - "9913:9913"
EOF

# 啟動
docker-compose up -d

# metrics地址
http://192.168.10.100:9913/metrics

3.Prometheus配置

cd /data/docker-prometheus 

#在scrape_configs(搜刮配置):下面增加如下配置：

cat >> prometheus/prometheus.yml << "EOF"
  - job_name: 'nginx_vts_exporter'
    static_configs:
    - targets: ['192.168.10.100:9913']
      labels:
        instance: test伺服器
EOF

# 重新載入配置
curl -X POST http://localhost:9090/-/reload

# 檢查
http://192.168.10.14:9090/targets

3.1 常用監控指標

nginx_server_connections{status="accepted"} 接收請求數
nginx_server_connections{status="active"} 活動連線數
nginx_server_connections{status="handled"} 成功處理請求數
nginx_server_connections{status="reading"} 正在進行讀操作的請求數
nginx_server_connections{status="requests"} 總請求數
nginx_server_connections{status="waiting"} 正在等待的請求數
nginx_server_connections{status="writing"} 正在進行寫操作的請求數

nginx_upstream_requests{backend="xx:8080",code="4xx"} 後端節點請求狀態

nginx_server_requests{code="4xx",host="*"}          狀態碼為4xx,所有主機
nginx_server_requests{code="4xx",host="localhost"}  狀態碼為4xx,localhost主機
nginx_server_requests{code="5xx",host="*"}          狀態碼為5xx,所有主機     
nginx_server_requests{code="5xx",host="localhost"}  狀態碼為5xx,localhost主機

4. 觸發器

cat >>prometheus/rules/tengine.yml <<"EOF"
groups:
- name: tengine
  rules:
  - alert: 客戶端請求tengine非200
    expr: sum by (code) (rate(nginx_server_requests{code=~"4xx|5xx"}[2m])) > 0
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "客戶端http請求異常"
      description: "HTTP狀態碼非 200-399"
  - alert: tengine請求後端節點非200
    expr: sum by (code) (rate(nginx_upstream_requests{code=~"4xx|5xx"}[2m])) > 0
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "後端節點http請求異常"
      description: "後端節點請求狀態碼非 200-399"
EOF

注

• 2個觸發器大部分重複（如：tengine請求後端節點返回500，那tengine也會給客戶端返回500）

檢查配置並載入

docker exec -it prometheus promtool check config /etc/prometheus/prometheus.yml

curl -X POST http://localhost:9090/-/reload

# 檢查
http://192.168.10.14:9090/rules
http://192.168.10.14:9090/alerts?search=

5. dashboard

grafana展示prometheus從nginx_exporter收集到的的資料

https://grafana.com/grafana/dashboards/2949