Skip to content

刘航 / watchmen

Go to a project
Commit aa9d5016 by see.liuh@gmail.com
Options

【CHG】更新vproc，增加lld功能 

【CHG】更新文档：
1. 增加zabbix 监控docker说明
2. 增加《20181128整理需求以及实现》章节
1 parent 3ee4934c
Expand all
Inline Side-by-side
Showing 217 changed files with 1455 additions and 60 deletions
awesome_script/README.md
......@@ -34,3 +34,15 @@ vproc_testcase.py是简单的单元测试
- proxyname，template1、2默认是有值的，如果想为空，需指定，比如 --proxyname="" --template2=""
## elasticsearch_clean.py
删除30天前当天的日志。将其添加到linux crontab 任务计划中，每天执行一次。滚动清理
```
chmod +x elasticsearch_clean.py
copy elasticsearch_clean.py /usr/local/bin/
# 每天2点05执行一次
echo "05 2 * * * root elasticsearch_clean.py" >> /etc/crontab
/etc/init.d/cron reload
```
awesome_script/vproc/README.md
......@@ -16,9 +16,11 @@ vproc是实现上述功能的命令行工具，python编写
3. 提供类似top命令的状态查看功能，不联动zabbix，也能实时掌控自己应用程序的运行状态。
4. zabbix lld json串生成
## 使用方法
vproc工具有4个命令。
vproc工具有5个命令。
- 写：
......@@ -42,6 +44,18 @@ vproc工具有4个命令。
./vproc clean --path=app.group1
```
- lld json功能:
```
# 将path目录下第一级目录/文件名当做macro的value，生成json
# 假设有 app.group1.group2.item和app.group1.item1
./vproc lld --path=app.group1 --macro=DY
# 输出
{"data": [{"{#DY}": "group1"}, {"{#DY}": "item1"}]}
```
这个功能是配合zabbix的lld功能使用。详情见watchmen文档动态item章节
- 查看：
```
......
awesome_script/vproc/vproc.py
......@@ -3,7 +3,6 @@
#######################################################
# author: liuh 2018-10-24
# version: v1.0
# copy to device /usr/local/bin/vproc
#######################################################
......@@ -12,11 +11,13 @@ import getopt
import os
import shutil
import time
import json
#######################config#####################
ROOT_PATH="/root/UserApp/vproc"
# ROOT_PATH="/root/UserApp/vproc"
ROOT_PATH="/root/Log/vproc"
# ROOT_PATH="./vproc"
version="v1.1"
version="v1.2"
##################################################
def usage():
......@@ -27,12 +28,14 @@ def usage():
read read a key's value
write write a key's value
clean remove group of items
lld generate zabbix Low-level discovery json string
watch display items in the command line
write command options:
[+]-k any, --key=any | status item key.
[+]-v any, --value=any | status item value.
example:
vproc write --key=vaserver.license --value=1
vproc write --key=vaserver.va0.cameraname --value=camera1
vproc write --key=vaserver.va0.playok --value=1
......@@ -49,6 +52,15 @@ def usage():
vproc clean --path=vaserver.va0
if executed write command example, the dir:VPROC_ROOT/vaserver/va0 would be remove.
lld command options:
[+]-p any, --path=any | use the first level directory/file name in the [path] as the value of the macro
[+]-m any, --macro=any | macro name | defaule: MACRONAME
example:
vproc lld --path=vaserver --macro=MACRONAME
if executed write command example, print the json string:
{"data":[{"{#MACRONAME}":"va0"},{"{#MACRONAME}":"license"}]}
watch command options:
[+]-p any, --path=any | monitoring item groups path that need to be displayed
example:
......@@ -154,25 +166,60 @@ def watch_items(key_paths):
for file in filenames:
file_full_name = os.path.join(parent_dir, file)
filepaths.append(file_full_name)
for path in filepaths:
print_item(path)
def lld_json(paths, macro):
subobjs = []
for key_path in paths:
filepaths = []
watch_root = ROOT_PATH
midpath = ""
paths = key_path.split('.')
paths_len = len(paths)
if(paths_len>0):
for path in paths:
watch_root = watch_root+ "/" + path
midpath = midpath+ path + "/"
if(os.path.isfile(watch_root)):
return ""
else:
if not os.path.exists(watch_root):
print("no suck file or dir: {}".format(watch_root))
for parent_dir, dirnames, filenames in os.walk(watch_root):
for file in filenames:
if(watch_root == parent_dir):
file_full_name = os.path.join(parent_dir, file)
filepaths.append(file)
for dirname in dirnames:
if(watch_root == parent_dir):
filepaths.append(dirname)
for fp in filepaths:
objkey = "{{#{}}}".format(macro)
subobj = {objkey: fp}
subobjs.append(subobj)
obj = {'data': subobjs}
return json.dumps(obj)
def main():
argc = len(sys.argv)
if argc < 3 or (sys.argv[1]!= "read" and sys.argv[1]!= "write" and sys.argv[1]!= "clean" and sys.argv[1]!= "watch"):
if argc < 3 or (sys.argv[1]!= "read" and sys.argv[1]!= "write"
and sys.argv[1]!= "clean" and sys.argv[1]!= "watch" and sys.argv[1]!= "lld"):
usage()
sys.exit(-1)
try:
options,args = getopt.getopt(sys.argv[2:],"-h-k:-v:-p:",["help","key=","value=", "path="])
options,args = getopt.getopt(sys.argv[2:],"-h-k:-v:-p:-m:",["help","key=","value=", "path=", "macro="])
except getopt.GetoptError:
print("error: invalid args")
sys.exit(-1)
keys=[]
values=[]
key_path=[]
key_paths=[]
macros=[]
if(sys.argv[1] == "write"):
for name,value in options:
if name in ("-h","--help"):
......@@ -215,21 +262,34 @@ def main():
if name in ("-h","--help"):
usage()
if name in ("-p","--path"):
key_path.append(value)
for index in range(len(key_path)):
remove_items(key_path[index])
key_paths.append(value)
for index in range(len(key_paths)):
remove_items(key_paths[index])
sys.exit(0)
if(sys.argv[1] == "lld"):
for name,value in options:
if name in ("-h","--help"):
usage()
if name in ("-p","--path"):
key_paths.append(value)
if name in ("-m","--macro"):
macros.append(value)
if len(macros) > 0:
print(lld_json(key_paths, macros[0]))
else:
print(lld_json(key_paths, "MACRONAME"))
sys.exit(0)
if(sys.argv[1] == "watch"):
for name,value in options:
if name in ("-h","--help"):
usage()
if name in ("-p","--path"):
key_path.append(value)
key_paths.append(value)
while True:
try:
os.system("clear")
os.system("date")
watch_items(key_path)
watch_items(key_paths)
time.sleep(1)
except KeyboardInterrupt, e:
print ''
......
awesome_script/vproc/vproc_testcase.py
......@@ -2,34 +2,45 @@
import os
print("--------------------testcase write value-------------------")
res = os.popen("python vproc write --key=vaserver.va0.cameraname --value=name1").read();
res = os.popen("/usr/bin/python2.7 vproc.py write --key=vaserver.va0.cameraname --value=name1").read();
print(res)
res = os.popen("/usr/bin/python2.7 vproc.py write --key=vaserver.license --value=1").read();
print(res)
res = os.popen("/usr/bin/python2.7 vproc.py write --key=vaserver.va1.cameraname --value=name2").read();
print(res)
print("--------------------testcase read value-------------------")
res = os.popen("python vproc read --key=vaserver.va0.cameraname").read();
res = os.popen("/usr/bin/python2.7 vproc.py read --key=vaserver.va0.cameraname").read();
print(res)
print("--------------------testcase help, shoud print 3 times-------------------")
res = os.popen("python vproc --help").read();
res = os.popen("/usr/bin/python2.7 vproc.py --help").read();
print(res)
res = os.popen("python vproc read").read();
res = os.popen("/usr/bin/python2.7 vproc.py read").read();
print(res)
res = os.popen("python vproc").read();
res = os.popen("/usr/bin/python2.7 vproc.py").read();
print(res)
print("--------------------testcase read value, shoud print null-------------------")
res = os.popen("python vproc read --key=vaserver.lalalal").read();
res = os.popen("/usr/bin/python2.7 vproc.py read --key=vaserver.lalalal").read();
print(res)
print("--------------------testcase read value, shoud print invalid args 2 times-------------------")
res = os.popen("python vproc write --key=vaserver --value=123").read();
res = os.popen("/usr/bin/python2.7 vproc.py write --key=vaserver --value=123").read();
print(res)
res = os.popen("python vproc write --key=vaserver.fffff ").read();
res = os.popen("/usr/bin/python2.7 vproc.py write --key=vaserver.fffff ").read();
print(res)
print("--------------------testcase lld, should print {{\"data\": [{{\"{#MACRONAME}}\": \"license\"}}, {{\"{{#MACRONAME}}\": \"va0\"}}, {{\"{{#MACRONAME}}\": \"va1\"}}]}}-------------------")
res = os.popen("/usr/bin/python2.7 vproc.py lld --path=vaserver --macro=MACRONAME").read();
print(res)
print("--------------------testcase remove items-------------------")
res = os.popen("python vproc clean --path=vaserver.va0 ").read();
res = os.popen("/usr/bin/python2.7 vproc.py clean --path=vaserver.va0 ").read();
print(res)
doc/.vscode/settings.json 0 → 100644
{
"restructuredtext.confPath": "${workspaceFolder}"
}
\ No newline at end of file
doc/10-summary.rst
......@@ -7,7 +7,7 @@ watchmen试运行在繁星2代上，所以此文档内容与繁星2代深度耦
watchmen由3个组件组成： elk日志 + zabbix监测 + ansible批量控制 。 思路概述见 《自检系统思路概述ppt.pdf》
开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中
开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中
术语
--------
......
doc/20-ansible.rst
......@@ -139,3 +139,4 @@ ansible-doc 模块名 #查看某模块的使用方法，比如 ansible-doc apt
我这里的介绍只是快速入门方法，还有很多强大的功能，比如copy的backup特性，幂性，hosts的多级分组，定时任务等等 建议通读一遍中文说明文档：http://www.ansible.com.cn/docs/
doc/99-history.rst
修订历史
================
2018.12.04
-----------------------------
1. 增加zabbix 监控docker说明
2. 增加《20181128整理需求以及实现》章节
2018.11.23
------------------
1. 增加zabbix动态item功能（ Low-level discovery ），简称lld的操作文档。见章节4.6
2. vproc脚本增加生成lld所需json串功能。
3. vproc脚本根目录修改为/root/Log
2018.10.24
------------
1. 错别字修改，ansible的脚本叫playbook不是playback
......
doc/_build/doctrees/80-starnet-dev.doctree 0 → 100644
No preview for this file type
doc/_build/html/10-summary.html
......@@ -162,7 +162,7 @@
<p>文安自检系统（代号:watchmen）文档</p>
<p>watchmen试运行在繁星2代上，所以此文档内容与繁星2代深度耦合</p>
<p>watchmen由3个组件组成： elk日志 + zabbix监测 + ansible批量控制 。 思路概述见 《自检系统思路概述ppt.pdf》</p>
<p>开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中</p>
<p>开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中</p>
<div class="section" id="id2">
<h2>1.1. 术语<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h2>
<ul class="simple">
......
doc/_build/html/20-ansible.html
......@@ -93,7 +93,8 @@
</li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html#item">5. 动态item</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">6. 修订历史</a></li>
</ul>
......
doc/_build/html/30-elk.html
......@@ -103,7 +103,8 @@
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html#item">5. 动态item</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">6. 修订历史</a></li>
</ul>
......
doc/_build/html/99-history.html
......@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>5. 修订历史 &mdash; watchmen 1.0 documentation</title>
<title>6. 修订历史 &mdash; watchmen 1.0 documentation</title>
......@@ -28,7 +28,7 @@
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="prev" title="4. zabbix监测系统" href="40-zabbix.html" />
<link rel="prev" title="5. watchmen在繁星的应用与开发" href="80-starnet-dev.html" />
<script src="_static/js/modernizr.min.js"></script>
......@@ -87,11 +87,13 @@
<li class="toctree-l1"><a class="reference internal" href="20-ansible.html">2. ansible集群管理工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">5. 修订历史</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#id2">5.1. 2018.10.24</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id3">5.2. 2018.10.17</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id4">5.3. 2018.10.10</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id5">5.4. 2018.10.09</a></li>
<li class="toctree-l1"><a class="reference internal" href="80-starnet-dev.html">5. watchmen在繁星的应用与开发</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">6. 修订历史</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#id2">6.1. 2018.11.23</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id3">6.2. 2018.10.24</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id4">6.3. 2018.10.17</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id5">6.4. 2018.10.10</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id6">6.5. 2018.10.09</a></li>
</ul>
</li>
</ul>
......@@ -139,7 +141,7 @@
<li><a href="index.html">Docs</a> &raquo;</li>
<li>5. 修订历史</li>
<li>6. 修订历史</li>
<li class="wy-breadcrumbs-aside">
......@@ -159,13 +161,21 @@
<div itemprop="articleBody">
<div class="section" id="id1">
<h1>5. 修订历史<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h1>
<h1>6. 修订历史<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h1>
<div class="section" id="id2">
<h2>5.1. 2018.10.24<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h2>
<h2>6.1. 2018.11.23<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>增加zabbix动态item功能（ Low-level discovery ），简称lld的操作文档。见章节4.6</li>
<li>vproc脚本增加生成lld所需json串功能。</li>
<li>vproc脚本根目录修改为/root/Log</li>
</ol>
</div>
<div class="section" id="id3">
<h2>6.2. 2018.10.24<a class="headerlink" href="#id3" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>错别字修改，ansible的脚本叫playbook不是playback</li>
<li>增加4.4 4.5自定义监测项相关说明</li>
<li>增加vproc使用方法和原理</li>
<li>增加vproc使用方法和原理。vproc脚本源码在awesome_script/vproc/vproc.py</li>
<li>离线安装包更新，增加了vaserver的几个自定义监控项：</li>
</ol>
<ul class="simple">
......@@ -176,21 +186,21 @@
<li>增加zabbix监控模板： <a class="reference external" href="smb://192.168.9.1">smb://192.168.9.1</a>/公共模块/繁星/繁星二代/自检系统/zabbix_template 。新部署的zabbix将模板导入即可</li>
</ol>
</div>
<div class="section" id="id3">
<h2>5.2. 2018.10.17<a class="headerlink" href="#id3" title="Permalink to this headline"></a></h2>
<div class="section" id="id4">
<h2>6.3. 2018.10.17<a class="headerlink" href="#id4" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>zabbix agent和proxy配置文件增加 AllowRoot=1，使其可以用root账户运行</li>
<li>错别字修改，watchman改成watchmen，明明是想用DC漫画的watchmen :)</li>
</ol>
</div>
<div class="section" id="id4">
<h2>5.3. 2018.10.10<a class="headerlink" href="#id4" title="Permalink to this headline"></a></h2>
<div class="section" id="id5">
<h2>6.4. 2018.10.10<a class="headerlink" href="#id5" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>修改了zabbix_agent安装方法。由deb包安装改成编译安装。deb版本较低，一些定义好的item不支持。改成3.4版本源码编译的，与zabbix_proxy一起安装</li>
</ol>
</div>
<div class="section" id="id5">
<h2>5.4. 2018.10.09<a class="headerlink" href="#id5" title="Permalink to this headline"></a></h2>
<div class="section" id="id6">
<h2>6.5. 2018.10.09<a class="headerlink" href="#id6" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>新建文档，增加了watchmen各组件最基本的安装、配置、quickstart文档</li>
</ol>
......@@ -206,7 +216,7 @@
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="40-zabbix.html" class="btn btn-neutral" title="4. zabbix监测系统" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
<a href="80-starnet-dev.html" class="btn btn-neutral" title="5. watchmen在繁星的应用与开发" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
......
doc/_build/html/_sources/10-summary.rst.txt
......@@ -7,7 +7,7 @@ watchmen试运行在繁星2代上，所以此文档内容与繁星2代深度耦
watchmen由3个组件组成： elk日志 + zabbix监测 + ansible批量控制 。 思路概述见 《自检系统思路概述ppt.pdf》
开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中
开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中
术语
--------
......
doc/_build/html/_sources/20-ansible.rst.txt
......@@ -139,3 +139,4 @@ ansible-doc 模块名 #查看某模块的使用方法，比如 ansible-doc apt
我这里的介绍只是快速入门方法，还有很多强大的功能，比如copy的backup特性，幂性，hosts的多级分组，定时任务等等 建议通读一遍中文说明文档：http://www.ansible.com.cn/docs/
doc/_build/html/_sources/99-history.rst.txt
修订历史
================
2018.11.23
------------------
1. 增加zabbix动态item功能（ Low-level discovery ），简称lld的操作文档。见章节4.6
2. vproc脚本增加生成lld所需json串功能。
3. vproc脚本根目录修改为/root/Log
2018.10.24
------------
1. 错别字修改，ansible的脚本叫playbook不是playback
2. 增加4.4 4.5自定义监测项相关说明
3. 增加vproc使用方法和原理
3. 增加vproc使用方法和原理。vproc脚本源码在awesome_script/vproc/vproc.py
4. 离线安装包更新，增加了vaserver的几个自定义监控项：
- 增加zabbix_agent_proxy_tx1/bin/vproc 。
......
doc/_build/html/_sources/index.rst.txt
......@@ -15,6 +15,7 @@ Welcome to watchmen's documentation!
20-ansible
30-elk
40-zabbix
80-starnet-dev
99-history
Indices and tables
......
doc/_build/html/genindex.html
......@@ -87,7 +87,8 @@
<li class="toctree-l1"><a class="reference internal" href="20-ansible.html">2. ansible集群管理工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a></li>
<li class="toctree-l1"><a class="reference internal" href="80-starnet-dev.html">5. watchmen在繁星的应用与开发</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">6. 修订历史</a></li>
</ul>
......
doc/_build/html/index.html
......@@ -87,7 +87,8 @@
<li class="toctree-l1"><a class="reference internal" href="20-ansible.html">2. ansible集群管理工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a></li>
<li class="toctree-l1"><a class="reference internal" href="80-starnet-dev.html">5. watchmen在繁星的应用与开发</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">6. 修订历史</a></li>
</ul>
......@@ -210,13 +211,45 @@
<li class="toctree-l3"><a class="reference internal" href="40-zabbix.html#vaserver">4.5.1. VAServer</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="40-zabbix.html#item">4.6. 动态item</a><ul>
<li class="toctree-l3"><a class="reference internal" href="40-zabbix.html#id11">4.6.1. 概述</a></li>
<li class="toctree-l3"><a class="reference internal" href="40-zabbix.html#id14">4.6.2. 操作教程</a><ul>
<li class="toctree-l4"><a class="reference internal" href="40-zabbix.html#web">4.6.2.1. web上的配置</a></li>
<li class="toctree-l4"><a class="reference internal" href="40-zabbix.html#id15">4.6.2.2. 设备上配置文件</a></li>
<li class="toctree-l4"><a class="reference internal" href="40-zabbix.html#id16">4.6.2.3. 运作流程简述</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a><ul>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id2">5.1. 2018.10.24</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id3">5.2. 2018.10.17</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id4">5.3. 2018.10.10</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id5">5.4. 2018.10.09</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="80-starnet-dev.html">5. watchmen在繁星的应用与开发</a><ul>
<li class="toctree-l2"><a class="reference internal" href="80-starnet-dev.html#id1">5.1. 20181128整理需求以及实现</a><ul>
<li class="toctree-l3"><a class="reference internal" href="80-starnet-dev.html#id2">5.1.1. 原始需求以及应对方案</a></li>
<li class="toctree-l3"><a class="reference internal" href="80-starnet-dev.html#id3">5.1.2. 会话日志</a></li>
<li class="toctree-l3"><a class="reference internal" href="80-starnet-dev.html#id4">5.1.3. 相机状态</a></li>
<li class="toctree-l3"><a class="reference internal" href="80-starnet-dev.html#id5">5.1.4. 链路状态</a></li>
<li class="toctree-l3"><a class="reference internal" href="80-starnet-dev.html#id6">5.1.5. 任务状态</a></li>
<li class="toctree-l3"><a class="reference internal" href="80-starnet-dev.html#id7">5.1.6. 总结各服务需要做的事情</a><ul>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#sip">5.1.6.1. sip</a></li>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#mediaserver">5.1.6.2. MediaServer</a></li>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#vaserver">5.1.6.3. VAServer</a></li>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#dataexportservice">5.1.6.4. DataExportService</a></li>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#id8">5.1.6.5. 平台</a></li>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#operateserver">5.1.6.6. OperateServer</a></li>
<li class="toctree-l4"><a class="reference internal" href="80-starnet-dev.html#id9">5.1.6.7. 脚本工具</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">6. 修订历史</a><ul>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id2">6.1. 2018.11.23</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id3">6.2. 2018.10.24</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id4">6.3. 2018.10.17</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id5">6.4. 2018.10.10</a></li>
<li class="toctree-l2"><a class="reference internal" href="99-history.html#id6">6.5. 2018.10.09</a></li>
</ul>
</li>
</ul>
......
doc/_build/html/search.html
......@@ -86,7 +86,8 @@
<li class="toctree-l1"><a class="reference internal" href="20-ansible.html">2. ansible集群管理工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a></li>
<li class="toctree-l1"><a class="reference internal" href="80-starnet-dev.html">5. watchmen在繁星的应用与开发</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">6. 修订历史</a></li>
</ul>
......
doc/_build/watchmen_20181105.tar.gz 0 → 100644
No preview for this file type
doc/_build/watchmen_20181105/.buildinfo 0 → 100644
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
config: dfe5f3b56cb39b7b38b15961f34e3f20
tags: 645f666f9bcd5a90fca523b33c5a78b7
doc/_build/watchmen_20181105/10-summary.html 0 → 100644
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>1. 概述 &mdash; watchmen 1.0 documentation</title>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="2. ansible集群管理工具" href="20-ansible.html" />
<link rel="prev" title="Welcome to watchmen’s documentation!" href="index.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="index.html" class="icon icon-home"> watchmen
</a>
<div class="version">
1.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">1. 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#id2">1.1. 术语</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id3">1.2. 结构</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="20-ansible.html">2. ansible集群管理工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="99-history.html">5. 修订历史</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">watchmen</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li>1. 概述</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/10-summary.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="id1">
<h1>1. 概述<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h1>
<p>文安自检系统（代号:watchmen）文档</p>
<p>watchmen试运行在繁星2代上，所以此文档内容与繁星2代深度耦合</p>
<p>watchmen由3个组件组成： elk日志 + zabbix监测 + ansible批量控制 。 思路概述见 《自检系统思路概述ppt.pdf》</p>
<p>开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中</p>
<div class="section" id="id2">
<h2>1.1. 术语<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h2>
<ul class="simple">
<li>繁星2代：由40颗TX1芯片组成，每颗芯片有独立的操作系统，芯片之间用网络互通。一个繁星有4个网口，对应4颗主芯片，可与外部网络联通。主芯片有双网口，与其他子芯片组成内部子网络。</li>
</ul>
</div>
<div class="section" id="id3">
<h2>1.2. 结构<a class="headerlink" href="#id3" title="Permalink to this headline"></a></h2>
<p>从节点：</p>
<ul class="simple">
<li>filebeat （日志收集）</li>
<li>zabbix_agent （监测项状态上报）</li>
</ul>
<p>主节点：</p>
<ul class="simple">
<li>zabbix_agent （监测项状态上报）</li>
<li>zabbix_proxy （检测项转发代理）</li>
<li>haproxy （日志转发代理）</li>
<li>ansible （批量控制）</li>
</ul>
<p>服务器:</p>
<ul class="simple">
<li>zabbix_server （zabbix服务端）</li>
<li>zabbix_front （zabbix web）</li>
<li>postgres （zabbix数据库）</li>
<li>elasticsearch （日志系统服务端）</li>
<li>kibana （日志系统web）</li>
</ul>
<p>繁星2代网络隔离，只有主节点与外部网络联通，所以需要用zabbix_proxy和haproxy做代理，并且ansible部署在主节点，使用主节点对从节点批量控制</p>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="20-ansible.html" class="btn btn-neutral float-right" title="2. ansible集群管理工具" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="index.html" class="btn btn-neutral" title="Welcome to watchmen’s documentation!" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, liuhang.
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'1.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
\ No newline at end of file
doc/_build/watchmen_20181105/99-history.html 0 → 100644
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>5. 修订历史 &mdash; watchmen 1.0 documentation</title>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="prev" title="4. zabbix监测系统" href="40-zabbix.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="index.html" class="icon icon-home"> watchmen
</a>
<div class="version">
1.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="10-summary.html">1. 概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="20-ansible.html">2. ansible集群管理工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="30-elk.html">3. elk日志系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="40-zabbix.html">4. zabbix监测系统</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">5. 修订历史</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#id2">5.1. 2018.10.24</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id3">5.2. 2018.10.17</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id4">5.3. 2018.10.10</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id5">5.4. 2018.10.09</a></li>
</ul>
</li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">watchmen</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> &raquo;</li>
<li>5. 修订历史</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/99-history.rst.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="id1">
<h1>5. 修订历史<a class="headerlink" href="#id1" title="Permalink to this headline"></a></h1>
<div class="section" id="id2">
<h2>5.1. 2018.10.24<a class="headerlink" href="#id2" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>错别字修改，ansible的脚本叫playbook不是playback</li>
<li>增加4.4 4.5自定义监测项相关说明</li>
<li>增加vproc使用方法和原理</li>
<li>离线安装包更新，增加了vaserver的几个自定义监控项：</li>
</ol>
<ul class="simple">
<li>增加zabbix_agent_proxy_tx1/bin/vproc 。</li>
<li>zabbix_agent_proxy_tx1/etc/zabbix_agentd.conf和zabbix_agent_proxy_tx1/etc/zabbix_agentd.conf.d更新</li>
</ul>
<ol class="arabic simple" start="5">
<li>增加zabbix监控模板： <a class="reference external" href="smb://192.168.9.1">smb://192.168.9.1</a>/公共模块/繁星/繁星二代/自检系统/zabbix_template 。新部署的zabbix将模板导入即可</li>
</ol>
</div>
<div class="section" id="id3">
<h2>5.2. 2018.10.17<a class="headerlink" href="#id3" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>zabbix agent和proxy配置文件增加 AllowRoot=1，使其可以用root账户运行</li>
<li>错别字修改，watchman改成watchmen，明明是想用DC漫画的watchmen :)</li>
</ol>
</div>
<div class="section" id="id4">
<h2>5.3. 2018.10.10<a class="headerlink" href="#id4" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>修改了zabbix_agent安装方法。由deb包安装改成编译安装。deb版本较低，一些定义好的item不支持。改成3.4版本源码编译的，与zabbix_proxy一起安装</li>
</ol>
</div>
<div class="section" id="id5">
<h2>5.4. 2018.10.09<a class="headerlink" href="#id5" title="Permalink to this headline"></a></h2>
<ol class="arabic simple">
<li>新建文档，增加了watchmen各组件最基本的安装、配置、quickstart文档</li>
</ol>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="40-zabbix.html" class="btn btn-neutral" title="4. zabbix监测系统" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, liuhang.
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'1.0',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
\ No newline at end of file
doc/_build/watchmen_20181105/_sources/10-summary.rst.txt 0 → 100644
概述
=======
文安自检系统（代号:watchmen）文档
watchmen试运行在繁星2代上，所以此文档内容与繁星2代深度耦合
watchmen由3个组件组成： elk日志 + zabbix监测 + ansible批量控制 。 思路概述见 《自检系统思路概述ppt.pdf》
开发人员、测试人员、运维人员均会用到watchmen系统，使用svn或git共同维护此文档，将内容逐渐完善，将自己的使用实例、心得记录在文档中
术语
--------
- 繁星2代：由40颗TX1芯片组成，每颗芯片有独立的操作系统，芯片之间用网络互通。一个繁星有4个网口，对应4颗主芯片，可与外部网络联通。主芯片有双网口，与其他子芯片组成内部子网络。
结构
--------
从节点：
- filebeat （日志收集）
- zabbix_agent （监测项状态上报）
主节点：
- zabbix_agent （监测项状态上报）
- zabbix_proxy （检测项转发代理）
- haproxy （日志转发代理）
- ansible （批量控制）
服务器:
- zabbix_server （zabbix服务端）
- zabbix_front （zabbix web）
- postgres （zabbix数据库）
- elasticsearch （日志系统服务端）
- kibana （日志系统web）
繁星2代网络隔离，只有主节点与外部网络联通，所以需要用zabbix_proxy和haproxy做代理，并且ansible部署在主节点，使用主节点对从节点批量控制
doc/_build/watchmen_20181105/_sources/20-ansible.rst.txt 0 → 100644
ansible集群管理工具
=======================================
离线安装包: smb://192.168.9.1/公共模块/繁星/繁星二代/自检系统/ansible_offline_tx1
playbook实例在 ansible_playbook_example 目录。
在繁星中增加ansible，作为集群管理工具。ansible工作的流程是，读取/etc/ansible/hosts主机列表，批量的、自动的使用ssh远程到主机上执行指令，我们可以用它获得子节点的状态，批量配置、更新程序
我们有三种使用方法：
1. 写python或shell脚本，在脚本中编写复杂的指令组合，利用ansible的command模块将脚本和所需文件上传、执行。--此方法学习成本低，易上手。初级使用方法
2. 使用ansible提供的module+playbook。 module相当于shell命令(copy/ls/ps/apt-get等等)，playbook相当于shell脚本，将module组合执行。 --此方法需要需要学习各module的功能和参数,playbook语法（yml），优点是反馈更清晰，流程掌控更好，playbook比shell和python好写些。
3. 编写python脚本，使用ansible-api。 --此方法最高级，我能想到的需要使用此方法的应用途径：需要解析反馈信息并根据反馈做出不同反应；需要在自己的服务中利用ansible实现批量控制功能，比如批量升级服务，需要将各子节点升级进度，升级状态，错误信息反馈到web。
中文说明文档：http://www.ansible.com.cn/docs/
常用模块使用方法：https://www.cnblogs.com/zhaojiedi1992/p/zhaojiedi_linux_032_ansible02.html 。
安装
-------------------------------------------
- 安装
繁星的离线包已放在sdv，放在主节点上执行install.sh即可
在自己主机上安装: pip install ansible 。然后安装sshpass（支持在配置文件中增加密码免询问）
配置
------------------------------------------
- 配置主机列表
.. code-block:: shell
mkdir /etc/ansible
vim /etc/ansible/hosts
文件实例
.. code-block:: shell
[starnet1]
ubuntu@192.168.66.7
#指定密码，支持sudo不询问密码
ubuntu@192.168.66.12 ansible_sudo_pass='ubuntu'
ubuntu@192.168.66.27 ansible_sudo_pass='ubuntu'
ubuntu@192.168.66.32 ansible_sudo_pass='ubuntu'
ubuntu@192.168.66.4 ansible_sudo_pass='ubuntu'
[starnet2]
#公司内部使用这种配置方法。
192.168.66.[2:41] ansible_ssh_pass="ubuntu" ansible_ssh_user="ubuntu" ansible_sudo_pass='ubuntu'
- **非交互式用户名密码方式（推荐公司内部采用此方式，方便但安全性低）**
此方法需要安装sshpass。离线包已包含。密码需要已明文卸载配置中，不安全，在安全要求高的环境中禁止使用。但这个方法节省了对每个主机执行ssh-copy-id，是最方便的。参考配置文件中的starnet2组。在执行命令前，需要配置环境变量 export ANSIBLE_HOST_KEY_CHECKING=False，否则会报错。
执行 ansible starnet2 -m ping， 会向192.168.66.2，192.168.66.3 ... 192.168.66.41 测试是否连通。
- ssh公钥方式
.. code-block:: shell
ssh-keygen #一路回车
ssh-copy-id ubuntu@目标节点IP
#测试是否成功
ssh ubuntu@ip #不需要输入密码就能登录则说明成功
#也可以用root账户，输入密码。所有主机拷贝一次，后续就可以免密登陆了
#测试ping所有主机
ansible all -m ping
- 交互式用户名密码方式（询问密码）
.. code-block:: shell
ansible starnet1 -m ping --user=ubuntu -k
使用
-------------------------------------------
“上传文件夹，执行脚本” 。这种模式可以批量部署程序
- 执行多条指令方式
.. code-block:: shell
#拷贝文件/文件夹
`ansible starnet -m copy -a "src=./ansible_offline_tx1 /home/ubuntu/"`
#在主机上执行shell命令
`ansible starnet -m command -a "chmod +x /home/ubuntu/ansible_offline_tx1/install.sh"`
`ansible starnet -m command -a "bash /home/ubuntu/ansible_offline_tx1/install.sh"`
以上两条指令结合，可满足大部分需求。
- playbook方式
playbook文件内容:
.. code-block:: shell
---
- hosts: starnet
sudo: yes
tasks:•
- name: copy dir to remote
copy: src=ansible_offline_tx1 dest=/home/ubuntu/ansible_offline_tx1
- name: run script
command: chmod +x install.sh
args:
chdir: /home/ubuntu/ansible_offline_tx1/
command: bash install.sh
args:
chdir: /home/ubuntu/ansible_offline_tx1/
保存 ansible_offline_tx1.yml
执行playbook
.. code-block:: shell
ansible-playbook ansible_offline_tx1.yml
ansible提供了非常多的module，常用模块使用方法：https://www.cnblogs.com/zhaojiedi1992/p/zhaojiedi_linux_032_ansible02.html。
ansible-doc -l #可查看所有模块
ansible-doc 模块名 #查看某模块的使用方法，比如 ansible-doc apt ; ansible-doc pip ; ansible-doc ps
我这里的介绍只是快速入门方法，还有很多强大的功能，比如copy的backup特性，幂性，hosts的多级分组，定时任务等等 建议通读一遍中文说明文档：http://www.ansible.com.cn/docs/
doc/_build/watchmen_20181105/_sources/30-elk.rst.txt 0 → 100644
elk日志系统
===============
ELK组合是日志系统的经典组合。但是Logstash是java程序，占用内存资源比较大，前端将其换成轻量级的filebeat。haproxy放在主节点上做代理转发。
filebeat（日志收集）+ elasticsearch（日志存储搜索）+kibana（ui展示）+haproxy(代理转发)
安装
------------------------------------------
离线安装包: smb://192.168.9.1/公共模块/繁星/繁星二代/自检系统/(elk_tx1、elk_server、haproxy_tx1)
filebeat使用源码放在主机上编译。 elk三个工程使用官方下载的tar.gz的包。都是java程序，直接就能运行。
繁星2代已编译好filebeat，其他平台需按照下面步骤编译：
filebeat编译安装参考：
https://blog.csdn.net/lk142500/article/details/79535849
ubuntu14安装golang1.10：
https://www.cnblogs.com/senlinyang/p/8777384.html
配置
---------------------------------------------
elasticsearch-6.4.0配置
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
.. code-block:: shell
# 修改 elasticsearch-6.4.0/config/elasticsearch.yml
# 开放所有ip访问
network.host: 0.0.0.0
# 修改系统参数，否则运行会报错
vi /etc/sysctl.conf
vm.max_map_count=655360
sysctl -p
kibana-6.4.0-linux-x86_64配置
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
.. code-block:: shell
# 修改 kibana-6.4.0-linux-x86_64/config/kibana.yml
# 开放所有ip访问
server.host: "0.0.0.0"
# 配置elasticsearch的访问地址
elasticsearch.url: "http://127.0.0.1:9200"
filebeat配置
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
.. code-block:: shell
- type: log
# 改成true
enabled: true
# 配置日志路径，可配置多个，支持通配符但层级格式必须相符
paths:
- /root/Log/Ops/*/*.log
- /root/Log/VASvr/*/*.log
# 日志编码格式。如果不一致，配置两个《- type: log》块。
encoding: gbk
# 本机标签，在kibana搜索时可以以beat.name为过滤条件
name: "myname-192.168.5.99"
haproxy配置
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
.. code-block:: shell
# 编辑/etc/haproxy/haproxy.cfg 在末尾增加下面配置
# 将本机所有9200端口数据转发至服务器9200端口
listen elastic_proxy
bind 0.0.0.0:9200
mode tcp
server s1 192.168.9.133:9200
使用
------------------------------------------------------
kibana访问地址： http://ip:5601
quick start
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
Discover是查看、搜索日志的主要页面。如果有filebeat上报日志，第一次进入Discover会自动跳转，创建Index Patterns。
.. image:: ./images/elk-01.png
.. image:: ./images/elk-02.png
创建完毕后，在Discover中可以看到日志内容了，鼠标移到message，点add添加到“selected fields”中，日志内容看起来会清晰些。
.. image:: ./images/elk-03.png
Management->Elasticsearch->Index Management中会自动生成类似“filebeat-7.0.0-alpha1-2018.09.29”的行。
进入Management->Kibana->Index Patterns可查看、删除、重新创建Index Patterns
仅以上配置，利用搜索栏和Add a filter， 以主机、文件名、时间、关键字等等组合条件，就能满足绝大多数需求。
其他使用技巧在下面补充
kibana页面上的日志搜索技巧
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
kikana Discover页面的搜索栏支持的查询语法名叫Lucene query。elasticsearch底层使用Lucene，api和kibana都支持Lucene语法。
lucene query语法参考：https://blog.csdn.net/feifantiyan/article/details/54411183
下面列出几个使用实例：
- 查找beat.name为8.133的设备上日志文件路径中包含"Ops/2018-9-30"的日志
.. code-block:: shell
source:?*\/Ops\/2018-09-30?* AND beat.name:8.133
doc/_build/watchmen_20181105/_sources/99-history.rst.txt 0 → 100644
修订历史
================
2018.10.24
------------
1. 错别字修改，ansible的脚本叫playbook不是playback
2. 增加4.4 4.5自定义监测项相关说明
3. 增加vproc使用方法和原理
4. 离线安装包更新，增加了vaserver的几个自定义监控项：
- 增加zabbix_agent_proxy_tx1/bin/vproc 。
- zabbix_agent_proxy_tx1/etc/zabbix_agentd.conf和zabbix_agent_proxy_tx1/etc/zabbix_agentd.conf.d更新
5. 增加zabbix监控模板： smb://192.168.9.1/公共模块/繁星/繁星二代/自检系统/zabbix_template 。新部署的zabbix将模板导入即可
2018.10.17
------------
1. zabbix agent和proxy配置文件增加 AllowRoot=1，使其可以用root账户运行
2. 错别字修改，watchman改成watchmen，明明是想用DC漫画的watchmen :)
2018.10.10
------------
1. 修改了zabbix_agent安装方法。由deb包安装改成编译安装。deb版本较低，一些定义好的item不支持。改成3.4版本源码编译的，与zabbix_proxy一起安装
2018.10.09
------------
1. 新建文档，增加了watchmen各组件最基本的安装、配置、quickstart文档
doc/_build/watchmen_20181105/_sources/index.rst.txt 0 → 100644
.. face_camera_api documentation master file, created by
sphinx-quickstart on Fri Nov 10 10:22:03 2017.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to watchmen's documentation!
===========================================
.. toctree::
:maxdepth: 4
:numbered:
:caption: Contents:
10-summary
20-ansible
30-elk
40-zabbix
99-history
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
doc/_build/watchmen_20181105/_static/css/badge_only.css 0 → 100644
.fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{height:auto;max-height:100%}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}
doc/_build/watchmen_20181105/_static/css/theme.css 0 → 100644
This diff could not be displayed because it is too large.
doc/_build/watchmen_20181105/_static/doctools.js 0 → 100644
/*
* doctools.js
* ~~~~~~~~~~~
*
* Sphinx JavaScript utilities for all documentation.
*
* :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
* :license: BSD, see LICENSE for details.
*
*/
/**
* select a different prefix for underscore
*/
$u = _.noConflict();
/**
* make the code below compatible with browsers without
* an installed firebug like debugger
if (!window.console || !console.firebug) {
var names = ["log", "debug", "info", "warn", "error", "assert", "dir",
"dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace",
"profile", "profileEnd"];
window.console = {};
for (var i = 0; i < names.length; ++i)
window.console[names[i]] = function() {};
}
*/
/**
* small helper function to urldecode strings
*/
jQuery.urldecode = function(x) {
return decodeURIComponent(x).replace(/\+/g, ' ');
};
/**
* small helper function to urlencode strings
*/
jQuery.urlencode = encodeURIComponent;
/**
* This function returns the parsed url parameters of the
* current request. Multiple values per key are supported,
* it will always return arrays of strings for the value parts.
*/
jQuery.getQueryParameters = function(s) {
if (typeof s === 'undefined')
s = document.location.search;
var parts = s.substr(s.indexOf('?') + 1).split('&');
var result = {};
for (var i = 0; i < parts.length; i++) {
var tmp = parts[i].split('=', 2);
var key = jQuery.urldecode(tmp[0]);
var value = jQuery.urldecode(tmp[1]);
if (key in result)
result[key].push(value);
else
result[key] = [value];
}
return result;
};
/**
* highlight a given string on a jquery object by wrapping it in
* span elements with the given class name.
*/
jQuery.fn.highlightText = function(text, className) {
function highlight(node, addItems) {
if (node.nodeType === 3) {
var val = node.nodeValue;
var pos = val.toLowerCase().indexOf(text);
if (pos >= 0 &&
!jQuery(node.parentNode).hasClass(className) &&
!jQuery(node.parentNode).hasClass("nohighlight")) {
var span;
var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
if (isInSVG) {
span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
} else {
span = document.createElement("span");
span.className = className;
}
span.appendChild(document.createTextNode(val.substr(pos, text.length)));
node.parentNode.insertBefore(span, node.parentNode.insertBefore(
document.createTextNode(val.substr(pos + text.length)),
node.nextSibling));
node.nodeValue = val.substr(0, pos);
if (isInSVG) {
var bbox = span.getBBox();
var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
rect.x.baseVal.value = bbox.x;
rect.y.baseVal.value = bbox.y;
rect.width.baseVal.value = bbox.width;
rect.height.baseVal.value = bbox.height;
rect.setAttribute('class', className);
var parentOfText = node.parentNode.parentNode;
addItems.push({
"parent": node.parentNode,
"target": rect});
}
}
}
else if (!jQuery(node).is("button, select, textarea")) {
jQuery.each(node.childNodes, function() {
highlight(this, addItems);
});
}
}
var addItems = [];
var result = this.each(function() {
highlight(this, addItems);
});
for (var i = 0; i < addItems.length; ++i) {
jQuery(addItems[i].parent).before(addItems[i].target);
}
return result;
};
/*
* backward compatibility for jQuery.browser
* This will be supported until firefox bug is fixed.
*/
if (!jQuery.browser) {
jQuery.uaMatch = function(ua) {
ua = ua.toLowerCase();
var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
/(webkit)[ \/]([\w.]+)/.exec(ua) ||
/(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
/(msie) ([\w.]+)/.exec(ua) ||
ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
[];
return {
browser: match[ 1 ] || "",
version: match[ 2 ] || "0"
};
};
jQuery.browser = {};
jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
}
/**
* Small JavaScript module for the documentation.
*/
var Documentation = {
init : function() {
this.fixFirefoxAnchorBug();
this.highlightSearchWords();
this.initIndexTable();
},
/**
* i18n support
*/
TRANSLATIONS : {},
PLURAL_EXPR : function(n) { return n === 1 ? 0 : 1; },
LOCALE : 'unknown',
// gettext and ngettext don't access this so that the functions
// can safely bound to a different name (_ = Documentation.gettext)
gettext : function(string) {
var translated = Documentation.TRANSLATIONS[string];
if (typeof translated === 'undefined')
return string;
return (typeof translated === 'string') ? translated : translated[0];
},
ngettext : function(singular, plural, n) {
var translated = Documentation.TRANSLATIONS[singular];
if (typeof translated === 'undefined')
return (n == 1) ? singular : plural;
return translated[Documentation.PLURALEXPR(n)];
},
addTranslations : function(catalog) {
for (var key in catalog.messages)
this.TRANSLATIONS[key] = catalog.messages[key];
this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')');
this.LOCALE = catalog.locale;
},
/**
* add context elements like header anchor links
*/
addContextElements : function() {
$('div[id] > :header:first').each(function() {
$('<a class="headerlink">\u00B6</a>').
attr('href', '#' + this.id).
attr('title', _('Permalink to this headline')).
appendTo(this);
});
$('dt[id]').each(function() {
$('<a class="headerlink">\u00B6</a>').
attr('href', '#' + this.id).
attr('title', _('Permalink to this definition')).
appendTo(this);
});
},
/**
* workaround a firefox stupidity
* see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075
*/
fixFirefoxAnchorBug : function() {
if (document.location.hash && $.browser.mozilla)
window.setTimeout(function() {
document.location.href += '';
}, 10);
},
/**
* highlight the search words provided in the url in the text
*/
highlightSearchWords : function() {
var params = $.getQueryParameters();
var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : [];
if (terms.length) {
var body = $('div.body');
if (!body.length) {
body = $('body');
}
window.setTimeout(function() {
$.each(terms, function() {
body.highlightText(this.toLowerCase(), 'highlighted');
});
}, 10);
$('<p class="highlight-link"><a href="javascript:Documentation.' +
'hideSearchWords()">' + _('Hide Search Matches') + '</a></p>')
.appendTo($('#searchbox'));
}
},
/**
* init the domain index toggle buttons
*/
initIndexTable : function() {
var togglers = $('img.toggler').click(function() {
var src = $(this).attr('src');
var idnum = $(this).attr('id').substr(7);
$('tr.cg-' + idnum).toggle();
if (src.substr(-9) === 'minus.png')
$(this).attr('src', src.substr(0, src.length-9) + 'plus.png');
else
$(this).attr('src', src.substr(0, src.length-8) + 'minus.png');
}).css('display', '');
if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) {
togglers.click();
}
},
/**
* helper function to hide the search marks again
*/
hideSearchWords : function() {
$('#searchbox .highlight-link').fadeOut(300);
$('span.highlighted').removeClass('highlighted');
},
/**
* make the url absolute
*/
makeURL : function(relativeURL) {
return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL;
},
/**
* get the current relative url
*/
getCurrentURL : function() {
var path = document.location.pathname;
var parts = path.split(/\//);
$.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() {
if (this === '..')
parts.pop();
});
var url = parts.join('/');
return path.substring(url.lastIndexOf('/') + 1, path.length - 1);
},
initOnKeyListeners: function() {
$(document).keyup(function(event) {
var activeElementType = document.activeElement.tagName;
// don't navigate when in search box or textarea
if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') {
switch (event.keyCode) {
case 37: // left
var prevHref = $('link[rel="prev"]').prop('href');
if (prevHref) {
window.location.href = prevHref;
return false;
}
case 39: // right
var nextHref = $('link[rel="next"]').prop('href');
if (nextHref) {
window.location.href = nextHref;
return false;
}
}
}
});
}
};
// quick alias for translations
_ = Documentation.gettext;
$(document).ready(function() {
Documentation.init();
});
\ No newline at end of file
doc/_build/watchmen_20181105/_static/documentation_options.js 0 → 100644
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '',
VERSION: '1.0',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
\ No newline at end of file
doc/_build/watchmen_20181105/_static/fonts/fontawesome-webfont.svg 0 → 100644
This diff could not be displayed because it is too large.
doc/_build/watchmen_20181105/_static/jquery-3.2.1.js 0 → 100644
This diff could not be displayed because it is too large.
doc/_build/watchmen_20181105/objects.inv 0 → 100644
No preview for this file type
doc/_build/watchmen_doc_20181105/_static/css/theme.css 0 → 100644
This diff could not be displayed because it is too large.
doc/_build/watchmen_doc_20181105/_static/fonts/fontawesome-webfont.svg 0 → 100644
This diff could not be displayed because it is too large.
doc/_build/watchmen_doc_20181105/_static/jquery-3.2.1.js 0 → 100644
This diff could not be displayed because it is too large.
doc/_build/watchmen_doc_20181105/objects.inv 0 → 100644
No preview for this file type
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!