Improve performance put/get rows using pandas.DataFrame

- Add function: void Container.put_rows(input: pandas.DataFrame)
- Add function: pandas.DataFrame RowSet.fetch_rows()
- Reduce call function to check NULL field

Author: dangtrungtin

Makefile

@@ -9,7 +9,8 @@ CPPFLAGS = -fPIC -std=c++0x -g -O2
 INCLUDES = -Iinclude -Isrc
 
 INCLUDES_PYTHON = $(INCLUDES)	\
-				-I/usr/include/python3.6
+				-I/usr/include/python3.6		\
+				-I$(HOME)/.pyenv/versions/3.6.4/lib/python3.6/site-packages/numpy/core/include
 
 PROGRAM = _griddb_python.so
 EXTRA = griddb_python.py griddb_python.pyc
@@ -24,6 +25,7 @@ SOURCES = 	  src/TimeSeriesProperties.cpp \
 		  src/Query.cpp				\
 		  src/QueryAnalysisEntry.cpp			\
 		  src/RowKeyPredicate.cpp	\
+		  src/RowList.cpp	\
 		  src/RowSet.cpp			\
 		  src/TimestampUtils.cpp			\
 		  src/Field.cpp \

README.md

@@ -49,6 +49,11 @@ Set CPATH and LIBRARY_PATH.
 
     export LIBRARY_PATH=$LIBRARY_PATH:<C client library file directory path>
 
+Install Pandas and Numpy as below:
+
+    $ python -m pip install numpy
+    $ python -m pip install pandas
+
 ### Build and Run 
 
     1. Execute the command on project directory.

src/Container.cpp

@@ -61,7 +61,7 @@ namespace griddb {
             freeMemoryContainer();
             throw GSException(mContainer, "Memory allocation error");
         }
-        
+
         mContainerInfo->timeSeriesProperties = NULL;
         mContainerInfo->triggerInfoList = NULL;
         mContainerInfo->dataAffinity = NULL;
@@ -74,8 +74,6 @@ namespace griddb {
     }
 
     Container::~Container() {
-        
-
     // allRelated = FALSE, since all row object is managed by Row class
         close(GS_FALSE);
     }
@@ -409,4 +407,11 @@ namespace griddb {
     int Container::getColumnCount(){
         return mContainerInfo->columnCount;
     }
+
+    /**
+     * @brief Put rows with input is numpy data
+     */
+    void Container::put_rows(GSRow** listRow, int rowCount) {
+        this->multi_put(listRow, rowCount);
+    }
 }

src/Container.h

@@ -58,6 +58,7 @@ class Container {
         GSType* getGSTypeList();
         int getColumnCount();
         GSRow* getGSRowPtr();
+        void put_rows(GSRow** listRow, int rowCount);
 
     private:
         Container(GSContainer *container, GSContainerInfo* containerInfo);

src/Query.cpp

@@ -0,0 +1,69 @@
+/*
+    Copyright (c) 2017 TOSHIBA Digital Solutions Corporation.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+        http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+#include "RowList.h"
+
+namespace griddb {
+
+    RowList::RowList(GSRow *gsRow, GSRowSet *gsRowSet, GSType *typelist,
+            int columnCount, bool timetampFloat) :
+            mRowSet(gsRowSet), mRow(gsRow), mTypelist(typelist),
+            mColumnCount(columnCount), mTimetampFloat(timetampFloat) {
+    }
+
+    /**
+     * Support iterator object.
+     */
+    RowList* RowList::__iter__() {
+        return this;
+    }
+
+    /**
+     * Support iterator object: get next row
+     */
+    void RowList::__next__(bool* hasRow) {
+        *hasRow = gsHasNextRow(mRowSet);
+        if (*hasRow) {
+            gsGetNextRow(mRowSet, mRow);
+        }
+    }
+
+    /**
+     * Refer GSRow pointer from RowSet
+     */
+    GSRow* RowList::get_gsrow_ptr() {
+        return this->mRow;
+    }
+
+    /**
+     * Refer GSType pointer from RowSet
+     */
+    GSType* RowList::get_gstype_list() {
+        return mTypelist;
+    }
+
+    /**
+     * Refer number column from RowSet
+     */
+    int RowList::get_column_count() {
+        return mColumnCount;
+    }
+
+    /**
+     * Refer number column from RowSet
+     */
+    bool RowList::get_timestamp_to_float() {
+        return mTimetampFloat;
+    }
+
+}  // namespace griddb

src/Query.h

@@ -0,0 +1,39 @@
+/*
+    Copyright (c) 2017 TOSHIBA Digital Solutions Corporation.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+        http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+#ifndef _ROWLIST_H_
+#define _ROWLIST_H_
+
+#include "gridstore.h"
+
+namespace griddb {
+class RowList {
+ private:
+    GSRowSet *mRowSet;
+    GSRow *mRow;
+    GSType* mTypelist;
+    int mColumnCount;
+    bool mTimetampFloat;
+ public:
+    RowList(GSRow *gsRow, GSRowSet *gsRowSet, GSType* typelist, int columnCount,
+            bool timetampFloat);
+    void __next__(bool* hasRow);
+    RowList* __iter__();
+    GSRow* get_gsrow_ptr();
+    GSType* get_gstype_list();
+    int get_column_count();
+    bool get_timestamp_to_float();
+};
+}  // namespace griddb
+
+#endif  // _ROWLIST_H_

src/RowList.cpp

@@ -239,7 +239,7 @@ namespace griddb {
             } catch (bad_alloc& ba) {
                 throw GSException(mRowSet, "Memory allocation error");
             }
-            
+
             for (int i = 0; i < mContainerInfo->columnCount; i++){
                 typeList[i] = mContainerInfo->columnInfoList[i].type;
             }
@@ -263,4 +263,19 @@ namespace griddb {
         return mRow;
     }
 
+    /**
+     * @brief Get row list data
+     * @param **row List row data
+     * @param **rowSet Rowset data
+     * @return A pointer store row list data in RowList object
+     */
+    griddb::RowList* RowSet::fetch_rows(GSRow **row, GSRowSet **rowSet) {
+        try {
+            return new RowList(this->getGSRowPtr(), this->mRowSet,
+                    this->getGSTypeList(), this->getColumnCount(),
+                    this->timestamp_output_with_float);
+        } catch (bad_alloc &ba) {
+            throw GSException(mRowSet, "Memory allocation error");
+        }
+    }
 }

src/RowList.h

@@ -29,6 +29,7 @@
 #include "QueryAnalysisEntry.h"
 #include "GSException.h"
 #include "Util.h"
+#include "RowList.h"
 
 using namespace std;
 
@@ -67,6 +68,7 @@ class RowSet {
         int getColumnCount();
 
         GSRow* getGSRowPtr();
+        griddb::RowList* fetch_rows(GSRow** row, GSRowSet** rowSet);
 
     private:
         RowSet(GSRowSet *rowSet, GSContainerInfo *containerInfo, GSRow *mRow);