Read large JSON values from Python (resolves #16) (#42)

* Initial refactor (#16) * Fix exception handling (#16) * Convert raw to string when reading exceptions; * Close `rawConnection` objects after using them; * Remove workaround for R check which was causing problems with build. * Updated NEWS (#16) * Update DESCRIPTION (#16) Ensure CRAN doesn't try to compile this for Windows, since this still doesn't work. * Fix code style (#16)
asieira · Oct 6, 2017 · 880b624 · 880b624
1 parent 5c19b96
commit 880b624
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 31 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SnakeCharmR
 Version: 1.0.7
-Date: 2017-04-02
+Date: 2017-10-05
 Title: R and Python Integration
 Author: Alexandre Sieira, forked off of rPython by Carlos J. Gil Bellosta
 Maintainer: Alexandre Sieira <[email protected]>
@@ -13,7 +13,8 @@ Imports:
     utils
 License: GPL-2
 SystemRequirements: Python (>= 2.7) and Python headers and libraries
-    (See the INSTALL file)
+    (See the README.md file)
+OS_type: unix
 URL: https://github.com/asieira/SnakeCharmR
 BugReports: https://github.com/asieira/SnakeCharmR/issues
 Suggests:

diff --git a/NEWS b/NEWS
@@ -1,5 +1,9 @@
 SnakeCharmR 1.0.7
 =================
+Avoid tripping maximum R string size when reading large data objects from Python. The JSON string
+string read from Python is stored into a raw vector, which is then read with jsonlite's push parser
+through a rawConnection.
+
 Should now compile on Windows using the Rtools Python libraries or the standard Python
 package. Many thanks to Matthew Fidler (https://github.com/mattfidler), his help was invaluable
 in helping implement and test this.

diff --git a/R/RcppExports.R b/R/RcppExports.R
@@ -2,18 +2,18 @@
 # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393
 
 rcpp_Py_Initialize <- function() {
-    invisible(.Call('SnakeCharmR_rcpp_Py_Initialize', PACKAGE = 'SnakeCharmR'))
+    invisible(.Call(`_SnakeCharmR_rcpp_Py_Initialize`))
 }
 
 rcpp_Py_Finalize <- function() {
-    invisible(.Call('SnakeCharmR_rcpp_Py_Finalize', PACKAGE = 'SnakeCharmR'))
+    invisible(.Call(`_SnakeCharmR_rcpp_Py_Finalize`))
 }
 
 rcpp_Py_run_code <- function(code) {
-    .Call('SnakeCharmR_rcpp_Py_run_code', PACKAGE = 'SnakeCharmR', code)
+    .Call(`_SnakeCharmR_rcpp_Py_run_code`, code)
 }
 
 rcpp_Py_get_var <- function(varname) {
-    .Call('SnakeCharmR_rcpp_Py_get_var', PACKAGE = 'SnakeCharmR', varname)
+    .Call(`_SnakeCharmR_rcpp_Py_get_var`, varname)
 }
 
diff --git a/R/misc.R b/R/misc.R
@@ -38,5 +38,9 @@
 }
 
 .py.fromJSON <- function(x, json.options = list()) {
+  if (is.raw(x)) {
+    x <- rawConnection(x)
+    on.exit(close(x))
+  }
   do.call(fromJSON, c(json.options, list(txt = x)))
 }
diff --git a/R/py.call.R b/R/py.call.R
@@ -80,7 +80,7 @@ py.call <- function(fname, ...,
 
   # get return value
   retval <- rcpp_Py_get_var("_SnakeCharmR_return")
-  if (is.na(retval))
+  if (length(retval) == 0)
     stop("Cannot find function call return value")
   else
     py.rm("_SnakeCharmR_return")

diff --git a/R/py.exec.R b/R/py.exec.R
@@ -47,7 +47,8 @@ py.exec <- function(code, stopOnException = TRUE) {
 
   # try to read the stored exception
   exception = rcpp_Py_get_var("_SnakeCharmR_exception")
-  if (!is.na(exception)) {
+  if (length(exception) != 0) {
+    exception <- rawToChar(exception)
     py.rm("_SnakeCharmR_exception")
     if (stopOnException)
       stop(exception)

diff --git a/R/py.get.R b/R/py.get.R
@@ -47,18 +47,18 @@ py.get <- function(var.name, json.opt.ret = getOption("SnakeCharmR.json.opt.ret"
   )
 
   # try to read the return value
-  retval = rcpp_Py_get_var("_SnakeCharmR_return")
-  if (!is.na(retval)) {
+  retval <- rcpp_Py_get_var("_SnakeCharmR_return")
+  if (length(retval) != 0) {
     py.rm("_SnakeCharmR_return")
     return(.py.fromJSON(retval, json.opt.ret))
   }
 
   # value does not exist, stop with the exception value
-  exception = rcpp_Py_get_var("_SnakeCharmR_exception")
-  if (is.na(exception))
+  exception <- rcpp_Py_get_var("_SnakeCharmR_exception")
+  if (length(exception) == 0)
     stop(sprintf("Unexpected error reading %s, JSON encoded return value nor exception exist",
                  var.name))
   py.rm("_SnakeCharmR_exception")
-  stop(exception)
+  stop(rawToChar(exception))
 }
 
diff --git a/R/py.rm.R b/R/py.rm.R
@@ -32,9 +32,10 @@ py.rm <- function(var.name, stopOnException = FALSE) {
   )
 
   # try to read the stored exception
-  exception = rcpp_Py_get_var("_SnakeCharmR_exception")
-  if (!is.na(exception)) {
+  exception <- rcpp_Py_get_var("_SnakeCharmR_exception")
+  if (length(exception) != 0) {
     rcpp_Py_run_code("del _SnakeCharmR_exception")
+    exception <- rawToChar(exception)
     if (stopOnException)
       stop(exception)
     else

diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp
@@ -7,7 +7,7 @@ using namespace Rcpp;
 
 // rcpp_Py_Initialize
 void rcpp_Py_Initialize();
-RcppExport SEXP SnakeCharmR_rcpp_Py_Initialize() {
+RcppExport SEXP _SnakeCharmR_rcpp_Py_Initialize() {
 BEGIN_RCPP
     Rcpp::RNGScope rcpp_rngScope_gen;
     rcpp_Py_Initialize();
@@ -16,7 +16,7 @@ END_RCPP
 }
 // rcpp_Py_Finalize
 void rcpp_Py_Finalize();
-RcppExport SEXP SnakeCharmR_rcpp_Py_Finalize() {
+RcppExport SEXP _SnakeCharmR_rcpp_Py_Finalize() {
 BEGIN_RCPP
     Rcpp::RNGScope rcpp_rngScope_gen;
     rcpp_Py_Finalize();
@@ -25,7 +25,7 @@ END_RCPP
 }
 // rcpp_Py_run_code
 int rcpp_Py_run_code(String code);
-RcppExport SEXP SnakeCharmR_rcpp_Py_run_code(SEXP codeSEXP) {
+RcppExport SEXP _SnakeCharmR_rcpp_Py_run_code(SEXP codeSEXP) {
 BEGIN_RCPP
     Rcpp::RObject rcpp_result_gen;
     Rcpp::RNGScope rcpp_rngScope_gen;
@@ -35,8 +35,8 @@ BEGIN_RCPP
 END_RCPP
 }
 // rcpp_Py_get_var
-String rcpp_Py_get_var(String varname);
-RcppExport SEXP SnakeCharmR_rcpp_Py_get_var(SEXP varnameSEXP) {
+RawVector rcpp_Py_get_var(String varname);
+RcppExport SEXP _SnakeCharmR_rcpp_Py_get_var(SEXP varnameSEXP) {
 BEGIN_RCPP
     Rcpp::RObject rcpp_result_gen;
     Rcpp::RNGScope rcpp_rngScope_gen;
@@ -45,3 +45,16 @@ BEGIN_RCPP
     return rcpp_result_gen;
 END_RCPP
 }
+
+static const R_CallMethodDef CallEntries[] = {
+    {"_SnakeCharmR_rcpp_Py_Initialize", (DL_FUNC) &_SnakeCharmR_rcpp_Py_Initialize, 0},
+    {"_SnakeCharmR_rcpp_Py_Finalize", (DL_FUNC) &_SnakeCharmR_rcpp_Py_Finalize, 0},
+    {"_SnakeCharmR_rcpp_Py_run_code", (DL_FUNC) &_SnakeCharmR_rcpp_Py_run_code, 1},
+    {"_SnakeCharmR_rcpp_Py_get_var", (DL_FUNC) &_SnakeCharmR_rcpp_Py_get_var, 1},
+    {NULL, NULL, 0}
+};
+
+RcppExport void R_init_SnakeCharmR(DllInfo *dll) {
+    R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);
+    R_useDynamicSymbols(dll, FALSE);
+}
diff --git a/src/python.cpp b/src/python.cpp
@@ -10,10 +10,10 @@ using namespace Rcpp;
 #endif
 
 // fix R check warning as per https://github.com/RcppCore/Rcpp/issues/636#issuecomment-280985661
-void R_init_SnakeCharmR(DllInfo* info) {
-  R_registerRoutines(info, NULL, NULL, NULL, NULL);
-  R_useDynamicSymbols(info, TRUE);
-}
+// void R_init_SnakeCharmR(DllInfo* info) {
+//   R_registerRoutines(info, NULL, NULL, NULL, NULL);
+//   R_useDynamicSymbols(info, TRUE);
+// }
 
 // [[Rcpp::export]]
 void rcpp_Py_Initialize() {
@@ -46,22 +46,28 @@ int rcpp_Py_run_code(String code) {
 }
 
 // [[Rcpp::export]]
-String rcpp_Py_get_var(String varname) {
+RawVector rcpp_Py_get_var(String varname) {
   PyObject *value = PyDict_GetItemString(PyModule_GetDict(PyImport_AddModule("__main__")),
                                          varname.get_cstring());
   if (value == NULL)
-    return NA_STRING;
+    return RawVector(0);
 
+  char *str;
   if (PyUnicode_Check(value)) {
 #if RCPP_VERSION > Rcpp_Version(0,12,6)
-    String retval(PyBytes_AS_STRING(PyUnicode_AsUTF8String(value)), CE_UTF8);
+    str = PyBytes_AS_STRING(PyUnicode_AsUTF8String(value));
 #else
-    String retval(PyBytes_AS_STRING(PyUnicode_AsUTF8String(value)));
-    retval.set_encoding("UTF-8");
+    str = PyBytes_AS_STRING(PyUnicode_AsUTF8String(value));
 #endif
-    return retval;
   } else if (PyBytes_Check(value))
-    return String(PyBytes_AS_STRING(value));
+    str = PyBytes_AS_STRING(value);
   else
     throw std::invalid_argument("variable is not a string");
+
+  RawVector retval(0);
+  while (*str != 0) {
+    retval.push_back(*str);
+    str++;
+  }
+  return retval;
 }