Implement C API for datatable (#1469)

This PR adds external C API to datatable. The user of this API will have to include the file "datatable.h" located in directory "datatable/include/". This file will also be included in `datatable` source and binary distributions.

The following API functions are currently implemented:

- `size_t DtABIVersion()`
  Check the ABI version of the current datatable package. This method allows you to verify that the runtime ABI version of datatable matches your compile-time expectations.

- `int DtFrame_Check(PyObject* ob)`
  Check whether the python object is a datatable Frame.

- `size_t DtFrame_NRows(PyObject* pydt)`, `size_t DtFrame_NColumns(PyObject* pydt)`
  Get the shape of the Frame.

- `int DtFrame_ColumnStype(PyObject* pydt, size_t i)`
  Get the stype of column `i`.

- `PyObject* DtFrame_ColumnRowindex(PyObject* pydt, size_t i)`  
  Get the Rowindex of column `i`.

- `const void* DtFrame_ColumnDataR(PyObject* pydt, size_t i)`, `void* DtFrame_ColumnDataW(PyObject* pydt, size_t i)`, `const char* DtFrame_ColumnStringDataR(PyObject* pydt, size_t i)`
  Get the data pointer(s) of column `i`.

- `int DtRowindex_Check(PyObject* ob)`
  Check whether the python object is a datatable Rowindex.

- `int DtRowindex_Type(PyObject* pyri)`
  Find the type of the Rowindex.

- `size_t DtRowindex_Size(PyObject* pyri)`
  Get the number of elements in the Rowindex.

- `int DtRowindex_UnpackSlice(PyObject* pyri, size_t* start, size_t* length, size_t* step)`
  Get fields of a slice Rowindex.

- `const void* DtRowindex_ArrayData(PyObject* pyri)`
  Get data of an array Rowindex.

Closes #1382

Author: st-pasha

CHANGELOG.md

@@ -39,6 +39,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - function `dt.repeat(frame, n)` that creates a Frame by row-binding `n`
   copies of the `frame`.
 
+- `datatable` now exposes C API, to allow other C/C++ libraries interact
+  with datatable Frames directly (#1469).
+
 
 #### Fixed
 

MANIFEST.in

@@ -4,8 +4,8 @@ include CHANGELOG.md
 include setup.py
 include setup.cfg
 include ci/setup_utils.py
-recursive-include c         *.c *.cc *.h
-recursive-include datatable *.py
+recursive-include c         *.cc *.h
+recursive-include datatable *.py *.h
 recursive-include tests     *.py
 
 exclude datatable/lib/*.*

c/api.cc

@@ -0,0 +1,227 @@
+//------------------------------------------------------------------------------
+// Copyright 2018 H2O.ai
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//------------------------------------------------------------------------------
+#include "../datatable/include/datatable.h"
+#include "datatable.h"
+#include "frame/py_frame.h"
+#include "rowindex.h"
+#include "py_rowindex.h"
+extern "C" {
+
+
+//------------------------------------------------------------------------------
+// Helper functions
+//------------------------------------------------------------------------------
+
+static int _column_index_oob(DataTable* dt, size_t i) {
+  if (i < dt->ncols) return 0;
+  PyErr_Format(PyExc_IndexError, "Column %zu does not exist in the Frame", i);
+  return -1;
+}
+
+static DataTable* _extract_dt(PyObject* pydt) {
+  return static_cast<py::Frame*>(pydt)->get_datatable();
+}
+
+static RowIndex* _extract_ri(PyObject* pyri) {
+  if (pyri == Py_None) return nullptr;
+  return static_cast<pyrowindex::obj*>(pyri)->ref;
+}
+
+
+size_t DtABIVersion() {
+  return 1;
+}
+
+
+
+//------------------------------------------------------------------------------
+// Frame
+//------------------------------------------------------------------------------
+
+int DtFrame_Check(PyObject* ob) {
+  if (ob == nullptr) return 0;
+  auto typeptr = reinterpret_cast<PyObject*>(&py::Frame::Type::type);
+  int ret = PyObject_IsInstance(ob, typeptr);
+  if (ret == -1) {
+    PyErr_Clear();
+    ret = 0;
+  }
+  return ret;
+}
+
+
+size_t DtFrame_NColumns(PyObject* pydt) {
+  auto dt = _extract_dt(pydt);
+  return dt->ncols;
+}
+
+size_t DtFrame_NRows(PyObject* pydt) {
+  auto dt = _extract_dt(pydt);
+  return dt->nrows;
+}
+
+
+int DtFrame_ColumnStype(PyObject* pydt, size_t i) {
+  auto dt = _extract_dt(pydt);
+  if (_column_index_oob(dt, i)) return -1;
+  return static_cast<int>(dt->columns[i]->stype());  // stype() is noexcept
+}
+
+
+PyObject* DtFrame_ColumnRowindex(PyObject* pydt, size_t i) {
+  auto dt = _extract_dt(pydt);
+  if (_column_index_oob(dt, i)) return nullptr;
+  const RowIndex& ri = dt->columns[i]->rowindex();  // rowindex() is noexcept
+  return ri? pyrowindex::wrap(ri) : py::None().release();
+}
+
+
+const void* DtFrame_ColumnDataR(PyObject* pydt, size_t i) {
+  auto dt = _extract_dt(pydt);
+  if (_column_index_oob(dt, i)) return nullptr;
+  try {
+    return dt->columns[i]->data();
+  } catch (const std::exception& e) {
+    exception_to_python(e);
+    return nullptr;
+  }
+}
+
+void* DtFrame_ColumnDataW(PyObject* pydt, size_t i) {
+  auto dt = _extract_dt(pydt);
+  if (_column_index_oob(dt, i)) return nullptr;
+  try {
+    return dt->columns[i]->data_w();
+  } catch (const std::exception& e) {
+    exception_to_python(e);
+    return nullptr;
+  }
+}
+
+
+const char* DtFrame_ColumnStringDataR(PyObject* pydt, size_t i) {
+  auto dt = _extract_dt(pydt);
+  if (_column_index_oob(dt, i)) return nullptr;
+  SType st = dt->columns[i]->stype();
+  try {
+    if (st == SType::STR32) {
+      auto scol = static_cast<StringColumn<uint32_t>*>(dt->columns[i]);
+      return scol->strdata();
+    }
+    if (st == SType::STR64) {
+      auto scol = static_cast<StringColumn<uint64_t>*>(dt->columns[i]);
+      return scol->strdata();
+    }
+  } catch (const std::exception& e) {
+    exception_to_python(e);
+    return nullptr;
+  }
+  PyErr_Format(PyExc_TypeError, "Column %zu is not of string type", i);
+  return nullptr;
+}
+
+
+
+//------------------------------------------------------------------------------
+// Rowindex
+//------------------------------------------------------------------------------
+
+int DtRowindex_Check(PyObject* ob) {
+  if (ob == nullptr) return 0;
+  if (ob == Py_None) return 1;
+  auto typeptr = reinterpret_cast<PyObject*>(&pyrowindex::type);
+  int ret = PyObject_IsInstance(ob, typeptr);
+  if (ret == -1) {
+    PyErr_Clear();
+    ret = 0;
+  }
+  return ret;
+}
+
+
+int DtRowindex_Type(PyObject* pyri) {
+  RowIndex* ri = _extract_ri(pyri);
+  if (!ri) return 0;
+  return static_cast<int>(ri->type());
+}
+
+
+size_t DtRowindex_Size(PyObject* pyri) {
+  RowIndex* ri = _extract_ri(pyri);
+  if (!ri) return 0;
+  return ri->size();
+}
+
+
+int DtRowindex_UnpackSlice(
+    PyObject* pyri, size_t* start, size_t* length, size_t* step)
+{
+  RowIndex* ri = _extract_ri(pyri);
+  if (!ri || ri->type() != RowIndexType::SLICE) {
+    PyErr_Format(PyExc_TypeError, "expected a slice rowindex");
+    return -1;
+  }
+  *start = ri->slice_start();
+  *length = ri->size();
+  *step = ri->slice_step();
+  return 0;
+}
+
+
+const void* DtRowindex_ArrayData(PyObject* pyri) {
+  RowIndex* ri = _extract_ri(pyri);
+  if (ri && ri->type() == RowIndexType::ARR32) {
+    return ri->indices32();
+  }
+  if (ri && ri->type() == RowIndexType::ARR64) {
+    return ri->indices64();
+  }
+  PyErr_Format(PyExc_TypeError, "expected an array rowindex");
+  return nullptr;
+}
+
+
+
+//------------------------------------------------------------------------------
+// Deprecated
+//------------------------------------------------------------------------------
+
+void* datatable_get_column_data(void* dt_, size_t column) {
+  DataTable *dt = static_cast<DataTable*>(dt_);
+  return dt->columns[column]->data_w();
+}
+
+void datatable_unpack_slicerowindex(void* dt_, size_t* start, size_t* step) {
+  DataTable *dt = static_cast<DataTable*>(dt_);
+  RowIndex ri(dt->rowindex);
+  *start = ri.slice_start();
+  *step  = ri.slice_step();
+}
+
+void datatable_unpack_arrayrowindex(void *dt_, void **indices) {
+  DataTable *dt = static_cast<DataTable*>(dt_);
+  RowIndex ri(dt->rowindex);
+  *indices = const_cast<int32_t*>(ri.indices32());
+}
+
+
+} // extern "C"

c/capi.cc

@@ -424,7 +424,7 @@ void Column::verify_integrity(const std::string& name) const {
 
 VoidColumn::VoidColumn() {}
 VoidColumn::VoidColumn(size_t nrows) : Column(nrows) {}
-SType VoidColumn::stype() const { return SType::VOID; }
+SType VoidColumn::stype() const noexcept { return SType::VOID; }
 size_t VoidColumn::elemsize() const { return 0; }
 bool VoidColumn::is_fixedwidth() const { return true; }
 size_t VoidColumn::data_nrows() const { return nrows; }

c/capi.h

@@ -115,11 +115,11 @@ class Column
   virtual void replace_buffer(MemoryRange&&);
   virtual void replace_buffer(MemoryRange&&, MemoryRange&&);
 
-  virtual SType stype() const = 0;
+  virtual SType stype() const noexcept = 0;
   virtual size_t elemsize() const = 0;
   virtual bool is_fixedwidth() const = 0;
 
-  const RowIndex& rowindex() const { return ri; }
+  const RowIndex& rowindex() const noexcept { return ri; }
   RowIndex remove_rowindex();
   void replace_rowindex(const RowIndex& newri);
 
@@ -396,7 +396,7 @@ class BoolColumn : public FwColumn<int8_t>
 {
 public:
   using FwColumn<int8_t>::FwColumn;
-  SType stype() const override;
+  SType stype() const noexcept override;
 
   int8_t min() const;
   int8_t max() const;
@@ -448,7 +448,7 @@ template <typename T> class IntColumn : public FwColumn<T>
 {
 public:
   using FwColumn<T>::FwColumn;
-  virtual SType stype() const override;
+  virtual SType stype() const noexcept override;
 
   T min() const;
   T max() const;
@@ -515,7 +515,7 @@ template <typename T> class RealColumn : public FwColumn<T>
 {
 public:
   using FwColumn<T>::FwColumn;
-  virtual SType stype() const override;
+  virtual SType stype() const noexcept override;
 
   T min() const;
   T max() const;
@@ -594,7 +594,7 @@ class PyObjectColumn : public FwColumn<PyObject*>
 public:
   PyObjectColumn(size_t nrows);
   PyObjectColumn(size_t nrows, MemoryRange&&);
-  virtual SType stype() const override;
+  virtual SType stype() const noexcept override;
   PyObjectStats* get_stats() const override;
 
   py::oobj get_value_at_index(size_t i) const override;
@@ -642,7 +642,7 @@ template <typename T> class StringColumn : public Column
                     WritableBuffer::Strategy strategy) override;
   void replace_buffer(MemoryRange&&, MemoryRange&&) override;
 
-  SType stype() const override;
+  SType stype() const noexcept override;
   size_t elemsize() const override;
   bool is_fixedwidth() const override;
 
@@ -714,7 +714,7 @@ extern template class StringColumn<uint64_t>;
 class VoidColumn : public Column {
   public:
     VoidColumn(size_t nrows);
-    SType stype() const override;
+    SType stype() const noexcept override;
     size_t elemsize() const override;
     bool is_fixedwidth() const override;
     size_t data_nrows() const override;