Release GIL when doing the compute-intensive CRC computation

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Development
 
+* Release GIL during the computation of the CRC32C hash. A new `gil_release_mode` argument lets users choose between always/never/automatically releasing it (#47).
+* Add keyword support to `crc32c` function (`crc32c(data, value=0, gil_release_mode=-1)`).
 * Adding explicit fallthrough annotations
   in several ``switch`` C statements
   for clarity, and to avoid potential warnings (#46).

README.rst

@@ -29,9 +29,9 @@ by older compiler versions.
 Usage
 -----
 
-The only method exposed by this module is ``crc32c(data, [crc])``.
+The only method exposed by this module is ``crc32c(data, value=0, gil_release_mode=-1)``.
 It computes the CRC32C checksum of ``data``
-starting with an initial ``crc`` checksum,
+starting with an initial ``value`` checksum,
 similarly to how the built-in ``binascii.crc32`` works.
 It can thus be used like this:
 
@@ -40,14 +40,22 @@ It can thus be used like this:
   print(crc32c.crc32c(b'hello world'))
   # 3381945770
   crc = crc32c.crc32c(b'hello')
-  print(crc32c.crc32c(b' world', crc))
+  print(crc32c.crc32c(b' world', value=crc))
   # 3381945770
 
 In older versions,
 the function exposed by this module was called ``crc32``.
 That name is still present but deprecated,
 and will be removed in new versions of the library.
 
+The ``gil_release_mode`` keyword argument
+specifies whether a call of this library shall release or keep the Global Interpreter Lock.
+It can be set to the following values:
+
+* Negative: Only release the GIL when ``data`` >= 32KiB
+* 0: Never release the GIL
+* Positive: Always release the GIL
+
 Additionally one can consult
 the following module-level values:
 

_crc32c.c

@@ -32,15 +32,28 @@
 #include "common.h"
 #include "crc32c.h"
 
+#define MIN_BUFSIZE_FOR_AUTOMATIC_RELEASE 32 * 1024  /* threshold for GIL release is 32KiB */
+
 /* Set at module loading time */
 static crc_function crc_fn;
 int is_big_endian;
 
+static inline int crc32c_inline(uint32_t crc, unsigned char *bin_data, Py_ssize_t len) {
+	int result;
+	crc ^= 0xffffffff;
+	result = crc_fn(crc, bin_data, len);
+	result ^= 0xffffffff;
+	return result;
+}
+
 static
-PyObject* crc32c_crc32c(PyObject *self, PyObject *args) {
+PyObject* crc32c_crc32c(PyObject *self, PyObject *args, PyObject *kwargs) {
 	Py_buffer pbin;
 	unsigned char *bin_data = NULL;
 	uint32_t crc = 0U, result;
+	int gil_release_mode = -1;
+
+	static char *kwlist[] = {"data", "value", "gil_release_mode", NULL};
 
 	/* In python 3 we accept only bytes-like objects */
 	const char *format =
@@ -49,29 +62,33 @@ PyObject* crc32c_crc32c(PyObject *self, PyObject *args) {
 #else
 	"s*"
 #endif
-	"|I:crc32";
+	"|Ii:crc32";
 
-	if (!PyArg_ParseTuple(args, format, &pbin, &crc) )
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, kwlist, &pbin, &crc, &gil_release_mode) )
 		return NULL;
 
 	bin_data = pbin.buf;
-	crc ^= 0xffffffff;
-	result = crc_fn(crc, bin_data, pbin.len);
-	result ^= 0xffffffff;
+	if ((gil_release_mode < 0 && pbin.len >= MIN_BUFSIZE_FOR_AUTOMATIC_RELEASE) || gil_release_mode >= 1) {
+		Py_BEGIN_ALLOW_THREADS
+		result = crc32c_inline(crc, bin_data, pbin.len);
+		Py_END_ALLOW_THREADS
+	} else {
+		result = crc32c_inline(crc, bin_data, pbin.len);
+	}
 
 	PyBuffer_Release(&pbin);
 	return PyLong_FromUnsignedLong(result);
 }
 
 static
-PyObject *crc32c_crc32(PyObject *self, PyObject *args)
+PyObject *crc32c_crc32(PyObject *self, PyObject *args, PyObject *kwargs)
 {
 	if (PyErr_WarnEx(PyExc_DeprecationWarning,
 	                 "crc32c.crc32 will be eventually removed, use crc32c.crc32c instead",
 	                 1) == -1) {
 		return NULL;
 	}
-	return crc32c_crc32c(self, args);
+	return crc32c_crc32c(self, args, kwargs);
 }
 
 /* The different values the SW mode preference can take */
@@ -101,8 +118,8 @@ static enum crc32c_sw_mode get_sw_mode(void)
 }
 
 static PyMethodDef CRC32CMethods[] = {
-	{"crc32",   crc32c_crc32,   METH_VARARGS, "Calculate crc32c incrementally (deprecated)"},
-	{"crc32c",  crc32c_crc32c,  METH_VARARGS, "Calculate crc32c incrementally"},
+	{"crc32",   (PyCFunction)crc32c_crc32,   METH_VARARGS | METH_KEYWORDS, "Calculate crc32c incrementally (deprecated)"},
+	{"crc32c",  (PyCFunction)crc32c_crc32c,  METH_VARARGS | METH_KEYWORDS, "Calculate crc32c incrementally"},
 	{NULL, NULL, 0, NULL}        /* Sentinel */
 };
 

test/test_crc32c.py

@@ -80,6 +80,19 @@ class TestMisc(unittest.TestCase):
     def test_zero(self):
         self.assertEqual(0, crc32c.crc32c(b''))
 
+    def test_keyword(self):
+        self.assertEqual(10, crc32c.crc32c(b'', value=10))
+
+    def test_gil_behaviour(self):
+        def _test(data):
+            expected = crc32c.crc32c(data)
+            self.assertEqual(crc32c.crc32c(data, gil_release_mode=-1), expected)
+            self.assertEqual(crc32c.crc32c(data, gil_release_mode=0), expected)
+            self.assertEqual(crc32c.crc32c(data, gil_release_mode=1), expected)
+
+        _test(b'this_doesnt_release_the_gil_by_default')
+        _test(b'this_releases_the_gil_by_default' * 1024 * 1024)
+
     def test_crc32_deprecated(self):
         with warning_catcher() as warns:
             crc32c.crc32(b'')