From a4ac64370de9990d17ef6c9af3450bb3e28b442f Mon Sep 17 00:00:00 2001 From: Chris Roth Date: Tue, 5 Mar 2019 19:02:48 -0500 Subject: [PATCH 1/2] add support for rocksdb (dramatic memory improvement when using multiple databases) --- .../src/asyncstorage_core.js | 79 +++++++++++++------ 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/packages/pouchdb-adapter-asyncstorage/src/asyncstorage_core.js b/packages/pouchdb-adapter-asyncstorage/src/asyncstorage_core.js index ec6bd33..ed31e83 100644 --- a/packages/pouchdb-adapter-asyncstorage/src/asyncstorage_core.js +++ b/packages/pouchdb-adapter-asyncstorage/src/asyncstorage_core.js @@ -7,18 +7,43 @@ import { AsyncStorage } from 'react-native' import { safeJsonParse, safeJsonStringify } from 'pouchdb-json' + +let AsyncStorageRocksDB +try { + AsyncStorageRocksDB = require('react-native-async-storage-rocks').default +} finally { + // Fall back to regular AsyncStorage +} + + function createPrefix(dbName) { return dbName.replace(/!/g, '!!') + '!' // escape bangs in dbName } function prepareKey(key, core) { - return ( - core._prefix + - key - .replace(/\u0002/g, '\u0002\u0002') - .replace(/\u0001/g, '\u0001\u0002') - .replace(/\u0000/g, '\u0001\u0001') - ) + key = key + .replace(/\u0002/g, '\u0002\u0002') + .replace(/\u0001/g, '\u0001\u0002') + .replace(/\u0000/g, '\u0001\u0001') + + if (AsyncStorageRocksDB) { + key = key.replace(/ÿ/g, '\u0003') // RocksDB can't handle ÿ + } + + return core._prefix + key +} + +function unescapeKey(key) { + key = key + .replace(/\u0001\u0001/g, '\u0000') + .replace(/\u0001\u0002/g, '\u0001') + .replace(/\u0002\u0002/g, '\u0002') + + if (AsyncStorageRocksDB) { + key = key.replace(/\u0003/g, 'ÿ') + } + + return key } function AsyncStorageCore(dbName) { @@ -30,24 +55,32 @@ AsyncStorageCore.prototype.getKeys = function(callback) { const prefix = this._prefix const prefixLen = prefix.length - AsyncStorage.getAllKeys((error, allKeys) => { - if (error) return callback(error) + if (AsyncStorageRocksDB) { + + // getAllKeysWithPrefix is dramatically more memory efficient if there are other items + // stored that are not part of the current database (eg other databases or app data). + // getAllKeys causes memory to increase linearly with the total number of keys. + AsyncStorageRocksDB.getAllKeysWithPrefix(prefix, (error, allKeys) => { + if (error) return callback(error) - allKeys.forEach(fullKey => { - if (fullKey.slice(0, prefixLen) === prefix) { - keys.push( - fullKey - .slice(prefixLen) - .replace(/\u0001\u0001/g, '\u0000') - .replace(/\u0001\u0002/g, '\u0001') - .replace(/\u0002\u0002/g, '\u0002') - ) - } + allKeys.forEach(fullKey => keys.push(unescapeKey(fullKey))) }) - keys.sort() - callback(null, keys) - }) + } else { + + AsyncStorage.getAllKeys((error, allKeys) => { + if (error) return callback(error) + + allKeys.forEach(fullKey => { + if (fullKey.slice(0, prefixLen) === prefix) { + keys.push(unescapeKey(fullKey.slice(prefixLen))) + } + }) + }) + } + + keys.sort() + callback(null, keys) } const stringifyValue = value => { @@ -114,4 +147,4 @@ AsyncStorageCore.destroy = function(dbName, callback) { }) } -module.exports = AsyncStorageCore +module.exports = AsyncStorageCore \ No newline at end of file From ce6a824ffb146279fd238d7e8f2d2301624a4442 Mon Sep 17 00:00:00 2001 From: Chris Roth Date: Tue, 5 Mar 2019 19:11:25 -0500 Subject: [PATCH 2/2] update readme for rocksdb --- packages/pouchdb-adapter-asyncstorage/readme.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/pouchdb-adapter-asyncstorage/readme.md b/packages/pouchdb-adapter-asyncstorage/readme.md index 45c0323..b9d0593 100644 --- a/packages/pouchdb-adapter-asyncstorage/readme.md +++ b/packages/pouchdb-adapter-asyncstorage/readme.md @@ -24,6 +24,11 @@ db.get('4711') ``` +### Memory + +Be careful when using this with other AsyncStorage data or multiple databases as it loads *all* keys in your store into memory. You can use [react-native-async-storage-rocks](https://github.com/tradle/react-native-async-storage-rocks) if this is an issue for you (see below). + + ### Android limit On Android asyncstorage has a limitation of 6 MB per default, you might want to increase it @@ -34,6 +39,11 @@ long size = 50L * 1024L * 1024L; // 50 MB com.facebook.react.modules.storage.ReactDatabaseSupplier.getInstance(getApplicationContext()).setMaximumSize(size); ``` +### RocksDB + +If you are using [react-native-async-storage-rocks](https://github.com/tradle/react-native-async-storage-rocks) then pouchdb-adapter-asyncstorage will detect and use the native `getKeysWithPrefix` method for memory-efficient key lookups. + + For full API documentation and guides on PouchDB, see [PouchDB.com](http://pouchdb.com/). For details on PouchDB sub-packages, see the [Custom Builds documentation](http://pouchdb.com/custom.html). ---