feature: full_text_search. issue #2224

Author: japlee

docs/features/full_text_search.md

@@ -0,0 +1,88 @@
+# 全文检索
+
+## 方案
+基于强大的elasticsearch进行搜索，把cmdb的mongo数据使用mongo-connector同步到
+elasticsearch，封装es的全文搜索的API提供出来。
+mongo-connector，通过读取mongodb的replica oplog，将mongodb产生的操作
+在elasticsearch上replay，来实现单向同步。即mongo里有数据变动，mongo-connector
+就会把相应的数据同步到es中同时进行更新。
+
+[原理图](../resource/img/mongo-connector.png)
+
+## es的使用
+全文检索使用了es的query_string的参数，并且配合使用bool(must, must_not, should)
+进行了定制化的搜索，配合使用aggs进行了数据的分类汇聚上报，配合使用了highlight提供了
+数据的高亮。
+一个完整的es的query请求大致如下：
+\*e\*为搜索条件，
+```
+{
+    "query": {
+        "bool": {
+            "must": [
+                {
+                    "term": { "bk_obj_id": "test_search"}
+                },
+                {
+                    "query_string": {"query": "*e*"}
+                }
+            ],
+            "must_not": [
+                {
+                    "match": {"bk_supplier_account": "*e*"}
+                }
+            ],
+            "should": [
+                {
+                    "bool": {
+                        "must_not": [
+                            {
+                                "regexp": { "metadata.label.bk_biz_id": "[0-9]*" }
+                            }
+                        ]
+                    }
+                },
+                {
+                    "term": { "metadata.label.bk_biz_id": "2" }
+                }
+            ],
+            "minimum_should_match" : 1
+        }
+    },
+    "aggs": {
+        "bk_obj_id_agg": {
+            "terms": {
+                "field": "bk_obj_id.keyword"
+            }
+        },
+        "type_agg": {
+            "terms": {
+                "field": "_type"
+            }
+        }
+    },
+    "highlight": {
+        "fields": {
+            "*" : {}
+        },
+        "require_field_match":  false
+    }
+}
+```
+把返回的结果封装转换成cmdb的api返回值规范：
+[全文检索api](../apidoc/v3.5/full_text_find.md)
+
+## mongo-connector和es的部署
+[部署](../overview/installation.md)
+第6和第7步，以及后面的配置开关full_text_search(值为off或者on)
+
+## 参考github
+[olivere elastic](https://github.com/olivere/elastic)
+
+[mongo-connector](https://github.com/yougov/mongo-connector)
+
+
+## 参考wiki
+[olivere elastic wiki](https://github.com/yougov/mongo-connector/wiki/Usage-with-ElasticSearch)
+
+[mongo-connector wiki](https://github.com/olivere/elastic/wiki)

docs/overview/installation.md

@@ -5,6 +5,8 @@
 * ZooKeeper >= 3.4.11
 * Redis   >= 3.2.11
 * MongoDB >= 2.8.0
+* Elasticsearch >= 5.0.0 & < 7 (用于全文检索功能，推荐使用5.x的版本)
+* Mongo-connector >= 2.5.0 (用于全文检索功能，推荐3.1.1)
 
 ## CMDB 微服务进程清单
 
@@ -51,7 +53,6 @@
 
 推荐版本下载：[MongoDB 2.8.0](http://downloads.mongodb.org/linux/mongodb-linux-x86_64-rhel70-2.8.0-rc5.tgz?_ga=2.109966917.1194957577.1522583108-162706957.1522583108)
 
-
 ### 4. Release包下载
 
 官方发布的 **Linux Release** 包下载地址见[这里](https://github.com/Tencent/bk-cmdb/releases)。如果你想自已编译，具体的编译方法见[这里](source_compile.md)。
@@ -75,7 +76,101 @@
 
 详细手册请参考官方资料 [MongoDB](https://docs.mongodb.com/manual/reference/method/db.createUser/)
 
-### 6. 部署CMDB
+### 6. 部署Elasticsearch (用于全文检索, 可选, 控制开关见第9步的full_text_search)
+
+官方下载 [ElasticSearch](https://www.elastic.co/cn/downloads/past-releases)
+搜索5.x的版本下载，推荐下载5.0.2, 5.6.16
+下载后解压即可，解压后找到配置文件config/elasticsearch.yml，可以配置指定network.host为
+具体的host的地址
+然后到目录的bin目录下运行(注意，不能使用root权限运行，要普通用户)：
+```
+./elasticsearch
+```
+
+如果想部署高可能可扩展的ES，可参考官方文档[ES-guild](https://www.elastic.co/guide/index.html)
+
+### 7. 部署mongo-connector (用于全文检索, 可选, 控制开关见第9步的full_text_search)
+
+官方仓库 [Mongo-connector](https://github.com/yougov/mongo-connector)
+推荐使用pip安装：
+
+```
+pip install elastic2-doc-manager elasticsearch
+pip install 'mongo-connector[elastic5]'
+```
+
+下载后请检查python包版本，尤其python elasticsearch大版本要和下载的elasticsearch一致
+
+配置配置文件config.json(配置说明参见[config](https://github.com/yougov/mongo-connector/wiki/Configuration%20Options)):
+
+主要配置
+key前面添加__代表忽略此配置
+mainAddress指定mongo，如果是mongo集群，可以指向slave节点
+authentication暂时先别配置，认证有问题
+namespaces里面配置要同步的mongo里的table，false代表不同步，true代表同步,
+可以自行配置需要同步哪些table用于全文检索
+
+```
+{
+    "__comment__": "Configuration options starting with '__' are disabled",
+    "__comment__": "To enable them, remove the preceding '__'",
+
+    "mainAddress": "127.0.0.1:27017",
+    "oplogFile": "/var/log/mongo-connector/oplog.timestamp",
+    "noDump": false,
+    "batchSize": -1,
+    "verbosity": 3,
+    "continueOnError": true,
+
+    "logging": {
+        "type": "file",
+        "filename": "/var/log/mongo-connector/mongo-connector.log",
+        "format": "%(asctime)s [%(levelname)s] %(name)s:%(lineno)d - %(message)s",
+        "rotationWhen": "D",
+        "rotationInterval": 1,
+        "rotationBackups": 10,
+
+        "__type": "syslog",
+        "__host": "localhost:514"
+    },
+
+    "__authentication": {
+        "adminUsername": "cc",
+        "password": "cc",
+        "__passwordFile": "mongo-connector.pwd"
+    },
+
+    "__fields": ["field1", "field2", "field3"],
+    
+    "exclude_fields": ["create_time", "last_time"],
+
+    "namespaces": {
+        "cmdb.cc_HostBase": true,
+        "cmdb.cc_ObjectBase": true,
+        "cmdb.cc_ObjDes": true,
+        "cmdb.cc_OperationLog": false
+    },
+
+    "docManagers": [
+        {
+            "docManager": "elastic2_doc_manager",
+            "targetURL": "127.0.0.1:9200",
+            "__bulkSize": 1000,
+            "uniqueKey": "_id",
+            "autoCommitInterval": 0
+        }
+    ]
+}
+```
+
+然后运行命令启动：
+```
+mongo-connector -c config.json
+```
+
+也可以自己写成system服务来运行
+
+### 8. 部署CMDB
 
 编译后下载 **cmdb.tar.gz**
 
@@ -129,7 +224,7 @@ drwxrwxr-x 3 1004 1004 4.0K Mar 29 14:45 cmdb_hostcontroller
 |cmdb_auditcontroller|controller|审计数据维护服务|
 |cmdb_hostcontroller|controller|主机数据维护服务|
 
-### 7. 初始化
+### 9. 初始化
 
 假定安装目录是 **/data/cmdb/**
 
@@ -151,6 +246,8 @@ drwxrwxr-x 3 1004 1004 4.0K Mar 29 14:45 cmdb_hostcontroller
 	--blueking_cmdb_url  <blueking_cmdb_url>    the cmdb site url, eg: http://127.0.0.1:8088 or http://bk.tencent.com
 	--blueking_paas_url  <blueking_paas_url>    the blueking paas url, eg: http://127.0.0.1:8088 or http://bk.tencent.com
 	--listen_port        <listen_port>          the cmdb_webserver listen port, should be the port as same as -c <cc_url> specified, default:8083
+	--full_text_search   <full_text_search>     full text search function, off or on, default off
+	--es_url             <es_url>               the elasticsearch listen url
 
 ```
 
@@ -170,17 +267,20 @@ drwxrwxr-x 3 1004 1004 4.0K Mar 29 14:45 cmdb_hostcontroller
 |--blueking_cmdb_url|该值表示部署完成后,输入到浏览器中访问的cmdb 网址, 格式: http://xx.xxx.com:80, 用户自定义填写;在没有配置 DNS 解析的情况下, 填写服务器的 IP:PORT。端口为当前cmdb_webserver监听的端口。|是|无|
 |--blueking_paas_url|蓝鲸PAAS 平台的地址，对于独立部署的CC版本可以不配置|否|无|
 |--listen_port|cmdb_webserver服务监听的端口，默认是8083|是|8083|
+|--full_text_search|全文检索功能开关(取值：off/on)，默认是off，开启是on|否|off|
+|--es_url|elasticsearch服务监听url，默认是http://127.0.0.1:9200|否|http://127.0.0.1:9200|
 
 **注:init.py 执行成功后会自动生成cmdb各服务进程所需要的配置。**
 
 **示例(示例中的参数需要用真实的值替换)：**
 
+如果部署了用于全文检索的第6和第7步，如要开启全文检索功能把full_text_search的值置为on
 ``` shell
-python init.py --discovery 127.0.0.1:2181 --database cmdb --redis_ip 127.0.0.1 --redis_port 6379 --redis_pass cc --mongo_ip 127.0.0.1 --mongo_port 27017 --mongo_user cc --mongo_pass cc --blueking_cmdb_url http://127.0.0.1:8083 --listen_port 8083
+python init.py --discovery 127.0.0.1:2181 --database cmdb --redis_ip 127.0.0.1 --redis_port 6379 --redis_pass cc --mongo_ip 127.0.0.1 --mongo_port 27017 --mongo_user cc --mongo_pass cc --blueking_cmdb_url http://127.0.0.1:8083 --listen_port 8083 --full_text_search on --es_url http://127.0.0.1:9200
 ```
 
 
-### 8. init.py 生成的配置如下
+### 10. init.py 生成的配置如下
 
 配置文件的存储路径：{安装目录}/cmdb_adminserver/configures/
 

docs/resource/img/mongo-connector.png

@@ -10,3 +10,8 @@ maxIDleConns=1000
 res=conf/errors
 [level]
 businessTopoMax=6
+
+# 全文检索功能开关(off，on)，以及es的url，用于topo中是否启用全文检索api功能以及建立es连接
+[es]
+full_text_search=off
+url=http://127.0.0.1:9200

resources/configures/topo.conf

@@ -19,6 +19,8 @@ bk_account_url=http://bk.tencent.com/login/accounts/get_all_user/?bk_token=%s
 resources_path=/tmp/
 html_root=/data/cmdb/cmdb_webserver/static
 authscheme=internal
+# 全文检索功能开关(off，on), 用于给前端提供是否开启全文检索功能界面展示
+full_text_search=off
 #authscheme=iam
 
 [errors]

resources/configures/webserver.conf

@@ -15,7 +15,7 @@ class FileTemplate(Template):
 def generate_config_file(
         rd_server_v, db_name_v, redis_ip_v, redis_port_v, redis_user_v,
         redis_pass_v, mongo_ip_v, mongo_port_v, mongo_user_v, mongo_pass_v,
-        cc_url_v, paas_url_v, auth_address, auth_app_code,
+        cc_url_v, paas_url_v, full_text_search, es_url_v, auth_address, auth_app_code,
         auth_app_secret, auth_enabled, auth_scheme
 ):
     output = os.getcwd() + "/cmdb_adminserver/configures/"
@@ -31,6 +31,7 @@ def generate_config_file(
         redis_port=redis_port_v,
         cc_url=cc_url_v,
         paas_url=paas_url_v,
+        es_url=es_url_v,
         ui_root="../web",
         agent_url=paas_url_v,
         configures_dir=output,
@@ -39,7 +40,8 @@ def generate_config_file(
         auth_app_code=auth_app_code,
         auth_app_secret=auth_app_secret,
         auth_enabled=auth_enabled,
-        auth_scheme=auth_scheme
+        auth_scheme=auth_scheme,
+        full_text_search=full_text_search
     )
     if not os.path.exists(output):
         os.mkdir(output)
@@ -400,6 +402,10 @@ def generate_config_file(
 appCode = $auth_app_code
 appSecret = $auth_app_secret
 enable = $auth_enabled
+
+[es]
+full_text_search = $full_text_search
+url=$es_url
 '''
 
     template = FileTemplate(topo_file_template_str)
@@ -430,6 +436,7 @@ def generate_config_file(
 resources_path = /tmp/
 html_root = $ui_root
 authscheme = $auth_scheme
+full_text_search = $full_text_search
 
 [app]
 agent_app_url = ${agent_url}/console/?app=bk_agent_setup
@@ -493,6 +500,8 @@ def main(argv):
         "auth_app_code": "bk_cmdb",
         "auth_app_secret": "",
     }
+    full_text_search = 'off'
+    es_url='http://127.0.0.1:9200'
 
     server_ports = {
         "cmdb_adminserver": 60004,
@@ -515,9 +524,9 @@ def main(argv):
         "help", "discovery=", "database=", "redis_ip=", "redis_port=",
         "redis_user=", "redis_pass=", "mongo_ip=", "mongo_port=",
         "mongo_user=", "mongo_pass=", "blueking_cmdb_url=",
-        "blueking_paas_url=", "listen_port=", "auth_address=",
+        "blueking_paas_url=", "listen_port=", "es_url=", "auth_address=",
         "auth_app_code=", "auth_app_secret=", "auth_enabled=",
-        "auth_scheme="
+        "auth_scheme=", "full_text_search="
     ]
     usage = '''
     usage:
@@ -538,9 +547,11 @@ def main(argv):
       --auth_address       <auth_address>         iam address
       --auth_app_code      <auth_app_code>        app code for iam, default bk_cmdb
       --auth_app_secret    <auth_app_secret>      app code for iam
+      --full_text_search   <full_text_search>     full text search on or off
+      --es_url             <es_url>               the es listen url, see in es dir config/elasticsearch.yml, (network.host, http.port), default: http://127.0.0.1:9200
     '''
     try:
-        opts, _ = getopt.getopt(argv, "hd:D:r:p:x:s:m:P:X:S:u:U:a:l:", arr)
+        opts, _ = getopt.getopt(argv, "hd:D:r:p:x:s:m:P:X:S:u:U:a:l:es", arr)
 
     except getopt.GetoptError as e:
         print("\n \t", e.msg)
@@ -553,7 +564,7 @@ def main(argv):
 
     for opt, arg in opts:
         if opt in ('-h', '--help'):
-            print('init.py --discovery <discovery> --database <database>  --redis_ip <redis_ip> --redis_port <redis_port> --redis_pass <redis_pass> --mongo_ip <mongo_ip> --mongo_port <mongo_port> --mongo_user <mongo_user> --mongo_pass <mongo_pass> --blueking_cmdb_url <blueking_cmdb_url> --blueking_paas_url <blueking_paas_url> --listen_port <listen_port>')
+            print('init.py --discovery <discovery> --database <database>  --redis_ip <redis_ip> --redis_port <redis_port> --redis_pass <redis_pass> --mongo_ip <mongo_ip> --mongo_port <mongo_port> --mongo_user <mongo_user> --mongo_pass <mongo_pass> --blueking_cmdb_url <blueking_cmdb_url> --blueking_paas_url <blueking_paas_url> --listen_port <listen_port> --es_url <es_url>')
             sys.exit()
         elif opt in ("-d", "--discovery"):
             rd_server = arg
@@ -606,6 +617,12 @@ def main(argv):
         elif opt in ("--auth_app_secret",):
             auth["auth_app_secret"] = arg
             print("auth_app_secret:", auth["auth_app_secret"])
+        elif opt in ("--full_text_search",):
+            full_text_search = arg
+            print('full_text_search:', full_text_search)
+        elif opt in("-es","--es_url",):
+            es_url = arg
+            print('es_url:', es_url)
 
     if 0 == len(rd_server):
         print('please input the ZooKeeper address, eg:127.0.0.1:2181')
@@ -644,6 +661,14 @@ def main(argv):
         print('blueking cmdb url not start with http://')
         sys.exit()
 
+    if full_text_search not in ["off", "on"]:
+        print('full_text_search can only be off or on')
+        sys.exit()
+    if full_text_search == "on":
+        if not es_url.startswith("http://"):
+            print('es url not start with http://')
+            sys.exit()
+
     if auth["auth_scheme"] not in ["internal", "iam"]:
         print('auth_scheme can only be internal or iam')
         sys.exit()
@@ -667,7 +692,7 @@ def main(argv):
 
     generate_config_file(rd_server, db_name, redis_ip, redis_port, redis_user,
                          redis_pass, mongo_ip, mongo_port, mongo_user,
-                         mongo_pass, cc_url, paas_url, **auth)
+                         mongo_pass, cc_url, paas_url, full_text_search, es_url, **auth)
     update_start_script(rd_server, server_ports)
     print('initial configurations success, configs could be found at cmdb_adminserver/configures')