(#2890) - websql: avoid hex() for binaries

Author: nolanlawson

lib/adapters/websql.js

@@ -4,10 +4,31 @@ var utils = require('../utils');
 var merge = require('../merge');
 var errors = require('../deps/errors');
 var vuvuzela = require('vuvuzela');
+var parseHexString = require('../deps/parse-hex');
+
 function quote(str) {
   return "'" + str + "'";
 }
 
+// escapeBlob and unescapeBlob are workarounds for a websql bug:
+// https://code.google.com/p/chromium/issues/detail?id=422690
+// https://bugs.webkit.org/show_bug.cgi?id=137637
+// The goal is to never actually insert the \u0000 character
+// in the database.
+function escapeBlob(str) {
+  return str
+    .replace(/\u0002/g, '\u0002\u0002')
+    .replace(/\u0001/g, '\u0001\u0002')
+    .replace(/\u0000/g, '\u0001\u0001');
+}
+
+function unescapeBlob(str) {
+  return str
+    .replace(/\u0001\u0001/g, '\u0000')
+    .replace(/\u0001\u0002/g, '\u0001')
+    .replace(/\u0002\u0002/g, '\u0002');
+}
+
 var cachedDatabases = {};
 
 var openDB = utils.getArguments(function (args) {
@@ -31,7 +52,7 @@ var openDB = utils.getArguments(function (args) {
 });
 
 var POUCH_VERSION = 1;
-var ADAPTER_VERSION = 5; // used to manage migrations
+var ADAPTER_VERSION = 6; // used to manage migrations
 
 // The object stores created for each database
 // DOC_STORE stores the document meta data, its revision history and state
@@ -92,22 +113,6 @@ function unknownError(callback) {
     callback(errors.error(errors.WSQ_ERROR, errorReason, errorName));
   };
 }
-function decodeUtf8(str) {
-  return decodeURIComponent(window.escape(str));
-}
-function parseHexString(str, encoding) {
-  var result = '';
-  var charWidth = encoding === 'UTF-8' ? 2 : 4;
-  for (var i = 0, len = str.length; i < len; i += charWidth) {
-    var substring = str.substring(i, i + charWidth);
-    if (charWidth === 4) { // UTF-16, twiddle the bits
-      substring = substring.substring(2, 4) + substring.substring(0, 2);
-    }
-    result += String.fromCharCode(parseInt(substring, 16));
-  }
-  result = encoding === 'UTF-8' ? decodeUtf8(result) : result;
-  return result;
-}
 
 function stringifyDoc(doc) {
   // don't bother storing the id/rev. it uses lots of space,
@@ -137,7 +142,7 @@ function getSize(opts) {
   // honest-to-god ceiling for data, so we need to
   // set it to a decently high number.
   var isAndroid = /Android/.test(window.navigator.userAgent);
-  return isAndroid ? 5000000 : 1;
+  return isAndroid ? 5000000 : 1; // in PhantomJS, if you use 0 it will crash
 }
 
 function WebSqlPouch(opts, callback) {
@@ -377,6 +382,15 @@ function WebSqlPouch(opts, callback) {
         });
     });
   }
+
+  // in this migration, we use escapeBlob() and unescapeBlob()
+  // instead of reading out the binary as HEX, which is slow
+  function runMigration6(tx, callback) {
+    var sql = 'ALTER TABLE ' + ATTACH_STORE +
+      ' ADD COLUMN escaped TINYINT(1) DEFAULT 0';
+    tx.executeSql(sql, [], callback);
+  }
+
   function checkEncoding(tx, cb) {
     // UTF-8 on chrome/android, UTF-16 on safari < 7.1
     tx.executeSql('SELECT HEX("a") AS hex', [], function (tx, res) {
@@ -401,7 +415,7 @@ function WebSqlPouch(opts, callback) {
       var meta = 'CREATE TABLE IF NOT EXISTS ' + META_STORE +
         ' (update_seq INTEGER, dbid, db_version INTEGER)';
       var attach = 'CREATE TABLE IF NOT EXISTS ' + ATTACH_STORE +
-        ' (digest UNIQUE, body BLOB)';
+        ' (digest UNIQUE, escaped TINYINT(1), body BLOB)';
       var attachAndRev = 'CREATE TABLE IF NOT EXISTS ' +
         ATTACH_AND_SEQ_STORE + ' (digest, seq INTEGER)';
       var doc = 'CREATE TABLE IF NOT EXISTS ' + DOC_STORE +
@@ -460,6 +474,7 @@ function WebSqlPouch(opts, callback) {
         runMigration3,
         runMigration4,
         runMigration5,
+        runMigration6,
         setupDone
       ];
 
@@ -1001,8 +1016,9 @@ function WebSqlPouch(opts, callback) {
         // we could just insert before selecting and catch the error,
         // but my hunch is that it's cheaper not to serialize the blob
         // from JS to C if we don't have to (TODO: confirm this)
-        sql = 'INSERT INTO ' + ATTACH_STORE + '(digest, body) VALUES (?,?)';
-        tx.executeSql(sql, [digest, data], function () {
+        sql = 'INSERT INTO ' + ATTACH_STORE +
+          ' (digest, body, escaped) VALUES (?,?,1)';
+        tx.executeSql(sql, [digest, escapeBlob(data)], function () {
           callback();
         }, function () {
           // ignore constaint errors, means it already exists
@@ -1304,13 +1320,17 @@ function WebSqlPouch(opts, callback) {
     var tx = opts.ctx;
     var digest = attachment.digest;
     var type = attachment.content_type;
-    var sql = 'SELECT hex(body) as body FROM ' + ATTACH_STORE +
-              ' WHERE digest=?';
+    var sql = 'SELECT escaped, ' +
+      'CASE WHEN escaped = 1 THEN body ELSE HEX(body) END AS body FROM ' +
+      ATTACH_STORE + ' WHERE digest=?';
     tx.executeSql(sql, [digest], function (tx, result) {
-      // sqlite normally stores data as utf8, so even the hex() function
-      // "encodes" the binary data in utf8/16 before returning it. yet hex()
-      // is the only way to get the full data, so we do this.
-      var data = parseHexString(result.rows.item(0).body, encoding);
+      // websql has a bug where \u0000 causes early truncation in strings
+      // and blobs. to work around this, we used to use the hex() function,
+      // but that's not performant. after migration 6, we remove \u0000
+      // and add it back in afterwards
+      var item = result.rows.item(0);
+      var data = item.escaped ? unescapeBlob(item.body) :
+        parseHexString(item.body, encoding);
       if (opts.encode) {
         res = btoa(data);
       } else {

lib/deps/parse-hex.js

@@ -0,0 +1,67 @@
+'use strict';
+
+//
+// Parsing hex strings. Yeah.
+//
+// So basically we need this because of a bug in WebSQL:
+// https://code.google.com/p/chromium/issues/detail?id=422690
+// https://bugs.webkit.org/show_bug.cgi?id=137637
+//
+// UTF-8 and UTF-16 are provided as separate functions
+// for meager performance improvements
+//
+
+function decodeUtf8(str) {
+  return decodeURIComponent(window.escape(str));
+}
+
+function hexToInt(charCode) {
+  // '0'-'9' is 48-57
+  // 'A'-'F' is 65-70
+  // SQLite will only give us uppercase hex
+  return charCode < 65 ? (charCode - 48) : (charCode - 55);
+}
+
+
+// Example:
+// pragma encoding=utf8;
+// select hex('A');
+// returns '41'
+function parseHexUtf8(str, start, end) {
+  var result = '';
+  while (start < end) {
+    result += String.fromCharCode(
+      (hexToInt(str.charCodeAt(start++)) << 4) |
+        hexToInt(str.charCodeAt(start++)));
+  }
+  return result;
+}
+
+// Example:
+// pragma encoding=utf16;
+// select hex('A');
+// returns '4100'
+// notice that the 00 comes after the 41 (i.e. it's swizzled)
+function parseHexUtf16(str, start, end) {
+  var result = '';
+  while (start < end) {
+    // UTF-16, so swizzle the bytes
+    result += String.fromCharCode(
+      (hexToInt(str.charCodeAt(start + 2)) << 12) |
+        (hexToInt(str.charCodeAt(start + 3)) << 8) |
+        (hexToInt(str.charCodeAt(start)) << 4) |
+        hexToInt(str.charCodeAt(start + 1)));
+    start += 4;
+  }
+  return result;
+}
+
+function parseHexString(str, encoding) {
+  if (encoding === 'UTF-8') {
+    return decodeUtf8(parseHexUtf8(str, 0, str.length));
+  } else {
+    return parseHexUtf16(str, 0, str.length);
+  }
+}
+
+module.exports = parseHexString;

tests/browser.migration.js

@@ -831,6 +831,50 @@ describe('migration', function () {
             });
           });
         });
+
+        it('#2890 PNG content after migration', function () {
+          if (skip) { return; }
+
+          var oldPouch = new dbs.first.pouch(
+            dbs.first.local, dbs.first.localOpts);
+
+          var transparent1x1Png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HA' +
+              'wCAAAAC0lEQVR4nGP6zwAAAgcBApocMXEA' +
+              'AAAASUVORK5CYII=';
+          var black1x1Png =
+            'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACkl' +
+              'EQVR4nGNiAAAABgADNjd8qAAA' +
+              'AABJRU5ErkJggg==';
+
+          return oldPouch.put({_id: 'foo'}).then(function (res) {
+            return oldPouch.putAttachment('foo', 'att', res.rev,
+              transparent1x1Png, 'image/png');
+          }).then(function () {
+            return oldPouch.get('foo', {attachments: true});
+          }).then(function (doc) {
+            doc._attachments['att'].content_type.should.equal('image/png');
+            should.exist(doc._attachments['att'].data);
+            return oldPouch.get('foo');
+          }).then(function (doc) {
+            return oldPouch.put(doc);
+          }).then(function () {
+            var newPouch = new dbs.second.pouch(dbs.second.local,
+              {auto_compaction: false});
+            return newPouch.compact().then(function () {
+              return newPouch.get('foo', {attachments: true});
+            }).then(function (doc) {
+              doc._attachments['att'].content_type.should.equal('image/png');
+              doc._attachments['att'].data.should.equal(transparent1x1Png);
+              return newPouch.putAttachment('bar', 'att', null,
+                black1x1Png, 'image/png');
+            }).then(function () {
+              return newPouch.get('bar', {attachments: true});
+            }).then(function (doc) {
+              doc._attachments['att'].content_type.should.equal('image/png');
+              doc._attachments['att'].data.should.equal(black1x1Png);
+            });
+          });
+        });
       }
     });
   });