使用Prometheus+Grafana搭建监控系统实践

# 下载
[root@prometheus src]# cd /usr/local/src/
[root@prometheus src]# wget https://github.com/prometheus/prometheus/releases/download/v2.0.0/prometheus-2.0.0.linux-amd64.tar.gz

# 部署到 /usr/local/ 目录
# promethus 不用编译安装，解压目录中有配置文件与启动文件
[root@prometheus src]# tar -zxvf prometheus-2.0.0.linux-amd64.tar.gz -C /usr/local/
[root@prometheus src]# cd /usr/local/
[root@prometheus local]# mv prometheus-2.0.0.linux-amd64/ prometheus/

# 验证
[root@prometheus local]# cd prometheus/
[root@prometheus prometheus]# ./prometheus --version

# 解压目录中的 prometheus.yml
# 简单验证，主要配置采用默认文件配置，有修改 / 新增处用红色标示
[root@prometheus prometheus]# vim prometheus.yml
# 全局配置
global:
  scrape_interval:     15s # 设置抓取 (pull) 时间间隔，默认是 1m
  evaluation_interval: 15s # 设置 rules 评估时间间隔，默认是 1m
  # scrape_timeout is set to the global default (10s).

# 告警管理配置，暂未使用，默认配置
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093

# 加载 rules，并根据设置的时间间隔定期评估，暂未使用，默认配置
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# 抓取(pull)，即监控目标配置
# 默认只有主机本身的监控配置
scrape_configs:
  # 监控目标的 label（这里的监控目标只是一个 metric，而不是指某特定主机，可以在特定主机取多个监控目标），在抓取的每条时间序列表中都会添加此 label
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    # 可覆盖全局配置设置的抓取间隔，由 15 秒重写成 5 秒。scrape_interval: 5s

    # 静态指定监控目标，暂不涉及使用一些服务发现机制发现目标
static_configs:
      - targets: ['localhost:9090']
        # (opentional)再添加一个 label，标识了监控目标的主机
labels:
          instance: prometheus

  - job_name: 'linux'
    scrape_interval: 10s
static_configs:
  # 采用 node_exporter 默认开放的端口
      - targets: ['172.20.1.212:9100']
labels:
          instance: node1

# 添加用户，后期用此账号启动服务
[root@prometheus prom etheus]# groupadd prometheus
[root@prometheus prometheus]# useradd -g prometheus -s /sbin/nologin prometheus

# 赋权
[root@prometheus prometheus]# cd ~
[root@prometheus ~]# chown -R prometheus:prometheus /usr/local/prometheus/

# 创建 prometheus 运行数据目录
[root@prometheus ~]# mkdir -p /var/lib/prometheus
[root@prometheus ~]# chown -R prometheus:prometheus /var/lib/prometheus/

[root@prometheus ~]# touch /usr/lib/systemd/system/prometheus.service 
[root@prometheus ~]# chown prometheus:prometheus /usr/lib/systemd/system/prometheus.service

[root@prometheus ~]# vim /usr/lib/systemd/system/prometheus.service
[Unit]
Description=Prometheus
Documentation=https://prometheus.io/
After=network.target

[Service]
# Type 设置为 notify 时，服务会不断重启
Type=simple
User=prometheus
# --storage.tsdb.path 是可选项，默认数据目录在运行目录的./dada 目录中
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus
Restart=on-failure

[Install]
WantedBy=multi-user.target

# 设置开机启动
[root@prometheus ~]# systemctl enable Prometheus
[root@prometheus ~]# systemctl start prometheus

[root@prometheus ~]# vim /etc/sysconfig/iptables
-A INPUT -p tcp -m state --state NEW -m tcp --dport 9090 -j ACCEPT

[root@prometheus ~]# service iptables restart

[root@prometheus ~]# systemctl status prometheus

[root@prometheus ~]# netstat -tunlp | grep 9090

# 下载
[root@node1 ~]# cd /usr/local/src/
[root@node1 src]# wget https://github.com/prometheus/node_exporter/releases/download/v0.15.1/node_exporter-0.15.1.linux-amd64.tar.gz

# 部署
[root@node1 src]# tar -zxvf node_exporter-0.15.1.linux-amd64.tar.gz -C /usr/local/
[root@node1 src]# cd /usr/local/
[root@node1 local]# mv node_exporter-0.15.1.linux-amd64/ node_exporter/

[root@node1 ~]# groupadd prometheus
[root@node1 ~]# useradd -g prometheus -s /sbin/nologin prometheus
[root@node1 ~]# chown -R prometheus:prometheus /usr/local/node_exporter/

[root@node1 ~]# vim /usr/lib/systemd/system/node_exporter.service
[Unit]
Description=node_exporter
Documentation=https://prometheus.io/
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=/usr/local/node_exporter/node_exporter
Restart=on-failure

[Install]
WantedBy=multi-user.target

[root@node1 ~]# systemctl enable node_exporter
[root@node1 ~]# systemctl start node_exporter

# 官方 node_exporter 默认使用 9100 端口
[root@node1 ~]# vim /etc/sysconfig/iptables
-A INPUT -p tcp -m state --state NEW -m tcp --dport 9100 -j ACCEPT

[root@node1 ~]# service iptables restart

# 下载
[root@prometheus ~]# cd /usr/local/src/
[root@prometheus src]# wget https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-4.6.2-1.x86_64.rpm

# 安装
[root@prometheus src]# yum localinstall grafana-4.6.2-1.x86_64.rpm

[root@prometheus src]# systemctl enable grafana-server
[root@prometheus src]# systemctl start grafana-server

# grafana-server 默认使用 3000 端口
[root@prometheus src]# vim /etc/sysconfig/iptables
-A INPUT -p tcp -m state --state NEW -m tcp --dport 3000 -j ACCEPT

[root@prometheus src]# service iptables restart