Skip to content

Commit fa79778

Browse files
alolisalolis
authored
Merge branch 'QHedgeTech:master' into master
2 parents e0fe2ad + 76808e4 commit fa79778

File tree

4 files changed

+136
-100
lines changed

4 files changed

+136
-100
lines changed

‎src/elasticsearch/elasticsearch.cpp

+59-39
Original file line numberDiff line numberDiff line change
@@ -99,45 +99,6 @@ bool ElasticSearch::deleteAll(const char* index, const char* type){
9999
return (msg["_indices"].getObject()[index].getObject()["_shards"].getObject()["failed"].getInt() == 0);
100100
}
101101

102-
int ElasticSearch::fullScan(const std::string& index, const std::string& type, const std::string& query, Json::Array& resultArray, int scrollSize) {
103-
104-
// Get the scroll id
105-
std::stringstream scrollUrl;
106-
scrollUrl << index << "/" << type << "/_search?search_type=scan&scroll=10m&size=" << scrollSize;
107-
108-
Json::Object scrollObject;
109-
_http.post(scrollUrl.str().c_str(),query.c_str(),&scrollObject);
110-
111-
if(!scrollObject.member("hits"))
112-
EXCEPTION("Result corrupted, no member \"hits\".");
113-
114-
if(!scrollObject.getValue("hits").getObject().member("total"))
115-
EXCEPTION("Result corrupted, no member \"total\" nested in \"hits\".");
116-
117-
int total = scrollObject.getValue("hits").getObject().getValue("total").getInt();
118-
119-
std::string scrollId = scrollObject["_scroll_id"].getString();
120-
int count = 0;
121-
while(count < total) {
122-
123-
Json::Object result;
124-
_http.rawpost("_search/scroll?scroll=10m", scrollId.c_str(), &result);
125-
126-
// Kepp the new scroll id we received to inject in the next iteration.
127-
scrollId = result["_scroll_id"].getString();
128-
129-
for(const Json::Value& value : result["hits"].getObject()["hits"].getArray()){
130-
resultArray.addElement(value);
131-
++count;
132-
}
133-
}
134-
135-
if(count != total)
136-
EXCEPTION("Result corrupted, total is different from count.");
137-
138-
return total;
139-
}
140-
141102
// Request the document number of type T in index I.
142103
long unsigned int ElasticSearch::getDocumentCount(const char* index, const char* type){
143104
std::ostringstream oss;
@@ -341,6 +302,65 @@ void ElasticSearch::refresh(const std::string& index){
341302
_http.get(oss.str().c_str(), 0, &msg);
342303
}
343304

305+
bool ElasticSearch::initScroll(std::string& scrollId, const std::string& index, const std::string& type, const std::string& query, int scrollSize) {
306+
std::ostringstream oss;
307+
oss << index << "/" << type << "/_search?scroll=1m&search_type=scan&size=" << scrollSize;
308+
309+
Json::Object msg;
310+
if (200 != _http.post(oss.str().c_str(), query.c_str(), &msg))
311+
return false;
312+
313+
scrollId = msg["_scroll_id"].getString();
314+
return true;
315+
}
316+
317+
bool ElasticSearch::scrollNext(std::string& scrollId, Json::Array& resultArray) {
318+
Json::Object msg;
319+
if (200 != _http.post("/_search/scroll?scroll=1m", scrollId.c_str(), &msg))
320+
return false;
321+
322+
scrollId = msg["_scroll_id"].getString();
323+
324+
appendHitsToArray(msg, resultArray);
325+
return true;
326+
}
327+
328+
void ElasticSearch::clearScroll(const std::string& scrollId) {
329+
_http.remove("/_search/scroll", scrollId.c_str(), 0);
330+
}
331+
332+
int ElasticSearch::fullScan(const std::string& index, const std::string& type, const std::string& query, Json::Array& resultArray, int scrollSize) {
333+
resultArray.clear();
334+
335+
std::string scrollId;
336+
if (!initScroll(scrollId, index, type, query, scrollSize))
337+
return 0;
338+
339+
size_t currentSize=0, newSize;
340+
while (scrollNext(scrollId, resultArray))
341+
{
342+
newSize = resultArray.size();
343+
if (currentSize == newSize)
344+
break;
345+
346+
currentSize = newSize;
347+
}
348+
return currentSize;
349+
}
350+
351+
void ElasticSearch::appendHitsToArray(const Json::Object& msg, Json::Array& resultArray) {
352+
353+
if(!msg.member("hits"))
354+
EXCEPTION("Result corrupted, no member \"hits\".");
355+
356+
if(!msg.getValue("hits").getObject().member("hits"))
357+
EXCEPTION("Result corrupted, no member \"hits\" nested in \"hits\".");
358+
359+
for(const Json::Value& value : msg["hits"].getObject()["hits"].getArray()) {
360+
resultArray.addElement(value);
361+
}
362+
}
363+
344364
// Bulk API of ES.
345365
bool ElasticSearch::bulk(const char* data, Json::Object& jResult) {
346366
if(_readOnly)

‎src/elasticsearch/elasticsearch.h

+17-4
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,6 @@ class ElasticSearch {
5959
/// Search API of ES. Specify the doc type.
6060
long search(const std::string& index, const std::string& type, const std::string& query, Json::Object& result);
6161

62-
/// Perform a scan to get all results from a query.
63-
int fullScan(const std::string& index, const std::string& type, const std::string& query, Json::Array& resultArray, int scrollSize = 1000);
64-
6562
// Bulk API
6663
bool bulk(const char*, Json::Object& jResult);
6764

@@ -81,7 +78,23 @@ class ElasticSearch {
8178

8279
/// Refresh the index.
8380
void refresh(const std::string& index);
84-
81+
82+
public:
83+
/// Initialize a scroll search. Use the returned scroll id when calling scrollNext. Size is based on shardSize. Returns false on error
84+
bool initScroll(std::string& scrollId, const std::string& index, const std::string& type, const std::string& query, int scrollSize = 1000);
85+
86+
/// Scroll to next matches of an initialized scroll search. scroll_id may be updated. End is reached when resultArray.empty() is true (in which scroll is automatically cleared). Returns false on error.
87+
bool scrollNext(std::string& scrollId, Json::Array& resultArray);
88+
89+
/// Clear an initialized scroll search prior to its automatically 1 minute timeout
90+
void clearScroll(const std::string& scrollId);
91+
92+
/// Perform a scan to get all results from a query.
93+
int fullScan(const std::string& index, const std::string& type, const std::string& query, Json::Array& resultArray, int scrollSize = 1000);
94+
95+
private:
96+
void appendHitsToArray(const Json::Object& msg, Json::Array& resultArray);
97+
8598
private:
8699
/// Private constructor.
87100
ElasticSearch();

‎src/http/http.cpp

+4
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,10 @@ unsigned int HTTP::request(const char* method, const char* endUrl, const char* d
275275
EXCEPTION("Unknown exception.");
276276
}
277277

278+
if (jOutput) {
279+
jOutput->addMemberByKey("status", statusCode);
280+
}
281+
278282
result = OK;
279283
return statusCode;
280284
}

‎src/json/json.cpp

+56-57
Original file line numberDiff line numberDiff line change
@@ -160,20 +160,13 @@ bool Json::Value::getBoolean() const {
160160
switch(_type){
161161
case booleanType:
162162
assert(!_data.empty());
163-
if(_data[0] == 't')
164-
return true;
165-
else
166-
return false;
163+
return (_data[0] == 't');
167164

168165
case numberType:
169-
if(getInt() != 0)
170-
return true;
166+
return (getInt() != 0);
171167

172168
case stringType:
173-
if(_data == "true")
174-
return true;
175-
if(_data == "false")
176-
return false;
169+
return (_data == "true");
177170

178171
case nullType:
179172
return false;
@@ -285,41 +278,43 @@ const Json::Array& Json::Value::getArray() const{
285278
return *_array;
286279
}
287280

288-
std::ostream& Json::operator<<(std::ostream& os, const Json::Value& value){
281+
namespace Json {
282+
std::ostream& operator<<(std::ostream& os, const Value& value){
289283

290-
if(value._type == Value::objectType){
291-
assert(value._object != 0);
292-
os << *(value._object);
293-
return os;
294-
}
284+
if(value._type == Value::objectType){
285+
assert(value._object != 0);
286+
os << *(value._object);
287+
return os;
288+
}
295289

296-
if(value._type == Value::arrayType){
297-
assert(value._array != 0);
298-
os << *(value._array);
299-
return os;
300-
}
290+
if(value._type == Value::arrayType){
291+
assert(value._array != 0);
292+
os << *(value._array);
293+
return os;
294+
}
301295

302-
if(value._type == Value::nullType){
303-
os << "null";
304-
return os;
305-
}
296+
if(value._type == Value::nullType){
297+
os << "null";
298+
return os;
299+
}
306300

307-
if(value._type == Value::booleanType){
308-
if(value)
309-
os << "true";
310-
else
311-
os << "false";
312-
return os;
313-
}
301+
if(value._type == Value::booleanType){
302+
if(value)
303+
os << "true";
304+
else
305+
os << "false";
306+
return os;
307+
}
308+
309+
if(value._type == Value::stringType){
310+
os << "\"" << value._data << "\"";
311+
return os;
312+
}
314313

315-
if(value._type == Value::stringType){
316-
os << "\"" << value._data << "\"";
314+
assert(value._type == Value::numberType);
315+
os << value._data;
317316
return os;
318317
}
319-
320-
assert(value._type == Value::numberType);
321-
os << value._data;
322-
return os;
323318
}
324319

325320
// Set this value as a boolean.
@@ -852,17 +847,19 @@ bool Json::Object::operator==(const Object& o) const {
852847
}
853848

854849
// Output in Json format
855-
ostream& Json::operator<<(ostream& os, const Json::Object& obj){
856-
os << "{";
857-
for(std::map< Key, Value >::const_iterator it = obj._memberMap.begin(); it != obj._memberMap.end(); ){
858-
os << "\"" << it->first << "\":" << it->second;
859-
++it;
860-
if(it != obj._memberMap.end())
861-
os << ",";
862-
}
863-
os << "}";
850+
namespace Json {
851+
std::ostream& operator<<(std::ostream& os, const Object& obj){
852+
os << "{";
853+
for(std::map< Key, Value >::const_iterator it = obj._memberMap.begin(); it != obj._memberMap.end(); ){
854+
os << "\"" << it->first << "\":" << it->second;
855+
++it;
856+
if(it != obj._memberMap.end())
857+
os << ",";
858+
}
859+
os << "}";
864860

865-
return os;
861+
return os;
862+
}
866863
}
867864

868865
/*------------------- Json Array ------------------*/
@@ -941,20 +938,22 @@ bool Json::Array::operator==(const Array& a) const {
941938
}
942939

943940
// Output in Json pretty format
944-
ostream& Json::operator<<(ostream& os, const Json::Array& array){
941+
namespace Json {
942+
std::ostream& operator<<(std::ostream& os, const Array& array){
945943

946-
os << "[";
944+
os << "[";
947945

948-
list<Value>::const_iterator it = array._elementList.begin();
949-
if(it != array._elementList.end())
950-
os << *it;
946+
list<Value>::const_iterator it = array._elementList.begin();
947+
if(it != array._elementList.end())
948+
os << *it;
951949

952-
for(++it; it != array._elementList.end(); ++it)
953-
os << "," << *it;
950+
for(++it; it != array._elementList.end(); ++it)
951+
os << "," << *it;
954952

955-
os << ']';
953+
os << ']';
956954

957-
return os;
955+
return os;
956+
}
958957
}
959958

960959
// Returns the data in Json Format.

0 commit comments

Comments
 (0)