reduce index size by extracting triple from key

Author: Benjamin Forster

lib/HyperdbReadTransform.js

@@ -1,5 +1,6 @@
 const Transform = require('readable-stream').Transform
 const inherits = require('inherits')
+const utils = require('./utils')
 
 function HyperdbReadTransform (db, options) {
   if (!(this instanceof HyperdbReadTransform)) {
@@ -31,7 +32,9 @@ HyperdbReadTransform.prototype._transform = function transform (nodes, encoding,
     return
   }
   var value = nodes[0].value && JSON.parse(nodes[0].value.toString())
-  if (value !== null && (!this._filter || this._filter(value))) {
+  if (value === null) return done()
+  value = Object.assign(value, utils.decodeKey(nodes[0].key))
+  if (!this._filter || this._filter(value)) {
     if (this._count >= this._offset) {
       this.push(value)
     }

lib/utils.js

@@ -8,6 +8,11 @@ const defs = {
   ops: ['object', 'predicate', 'subject'], // [optional]
   osp: ['object', 'subject', 'predicate']
 }
+const tripleAliasMap = {
+  s: 'subject',
+  p: 'predicate',
+  o: 'object'
+}
 const defKeys = Object.keys(defs)
 
 function collect (stream, cb) {
@@ -20,7 +25,7 @@ function collect (stream, cb) {
 function namespaceBlankVariables (triple) {
   Object.keys(triple).forEach((key) => {
     if (triple[key].startsWith && triple[key].startsWith('_:')) {
-      triple[key] = triple[key].replace(/^_:/, 'hypergraph:')
+      triple[key] = triple[key].replace(/^_:/, 'hg:')
     }
   })
   return triple
@@ -36,21 +41,37 @@ function filterTriple (triple) {
   return filtered
 }
 
-function escape (value) {
+function escapeKeyValue (value) {
   if (typeof value === 'string' || value instanceof String) {
-    return value.replace(/(\\|\/)/g, '\\$&')
+    return value.replace(/(\/)/g, '%2F')
   }
   return value
 }
 
-function genKey (key, triple) {
-  var result = '/' + key
+function unescapeKeyValue (value) {
+  return value.replace(/%2F/g, '/')
+}
+
+function decodeKey (key) {
+  const values = key.split('/')
+  if (values.length < 4) throw new Error('Key is not in triple form')
+  const order = values[0]
+  const triple = {}
+  for (var i = 0; i < 3; i++) {
+    const k = tripleAliasMap[order[i]]
+    triple[k] = unescapeKeyValue(values[i + 1])
+  }
+  return triple
+}
+
+function encodeKey (key, triple) {
+  var result = key
   var def = defs[key]
   var i = 0
   var value = triple[def[i]]
   // need to handle this smarter
   while (value) {
-    result += '/' + escape(value)
+    result += '/' + escapeKeyValue(value)
     i += 1
     value = triple[def[i]]
   }
@@ -60,17 +81,19 @@ function genKey (key, triple) {
   return result
 }
 
-function genKeys (triple) {
-  return defKeys.map(key => genKey(key, triple))
+function encodeKeys (triple) {
+  return defKeys.map(key => encodeKey(key, triple))
 }
 
 function generateBatch (triple, action) {
+  const namespacedTriple = namespaceBlankVariables(triple)
   if (!action) action = 'put'
-  var data = (action === 'put')
-    ? JSON.stringify(namespaceBlankVariables(triple))
-    : null
-  return genKeys(triple).map(key => ({
-    type: 'put', // no delete in hyperdb so just puting nulls
+  var data = null
+  if (action === 'put') {
+    data = JSON.stringify(extraDataMask(triple))
+  }
+  return encodeKeys(namespacedTriple).map(key => ({
+    type: 'put', // no delete in hyperdb so just putting nulls
     key: key,
     value: data
   }))
@@ -122,7 +145,7 @@ function createQuery (pattern, options) {
   var types = typesFromPattern(pattern)
   var preferedIndex = options && options.index
   var index = findIndex(types, preferedIndex)
-  var key = genKey(index, pattern)
+  var key = encodeKey(index, pattern)
   return key
 }
 
@@ -134,10 +157,23 @@ function keyIsNotAObject (tripleKey) {
   return typeof tripleKey !== 'object'
 }
 
+function not (fn) {
+  return (...args) => !fn(...args)
+}
+
 function keyIsAVariable (tripleKey) {
   return tripleKey instanceof Variable
 }
 
+function extraDataMask (obj) {
+  return Object.keys(obj)
+    .filter(not(keyIsAVariable))
+    .reduce((prev, key) => {
+      prev[key] = obj[key]
+      return prev
+    }, {})
+}
+
 function objectMask (criteria, obj) {
   return Object.keys(obj)
     .filter(hasKey)
@@ -157,6 +193,7 @@ function variableNames (obj) {
 function queryMask (object) {
   return objectMask(keyIsNotAObject, object)
 };
+
 function variablesMask (object) {
   return objectMask(keyIsAVariable, object)
 };
@@ -203,11 +240,13 @@ function materializer (pattern, data) {
 module.exports = {
   defs,
   defKeys,
-  genKey,
+  encodeKey,
+  decodeKey,
   filterTriple,
   collect,
   generateBatch,
   createQuery,
+  extraDataMask,
   queryMask,
   variablesMask,
   variableNames,

test/sparql.spec.js

@@ -45,7 +45,7 @@ describe('hypergraph.queryStream', () => {
     it('performs CONSTRUCT query type with UNION operator', (done) => {
       console.time('query')
       var expected = [
-        { subject: 'hypergraph:b0_b',
+        { subject: 'hg:b0_b',
           predicate: 'http://www.w3.org/2001/vcard-rdf/3.0#N',
           object: '_:b0' },
         { subject: '_:b0',
@@ -54,7 +54,7 @@ describe('hypergraph.queryStream', () => {
         { subject: '_:b0',
           predicate: 'http://www.w3.org/2001/vcard-rdf/3.0#familyName',
           object: '"Hacker"' },
-        { subject: 'hypergraph:b0_a',
+        { subject: 'hg:b0_a',
           predicate: 'http://www.w3.org/2001/vcard-rdf/3.0#N',
           object: '_:b1' },
         { subject: '_:b1',

test/utils.spec.js

@@ -0,0 +1,43 @@
+/* eslint-env mocha */
+const expect = require('chai').expect
+const utils = require('../lib/utils')
+
+describe('encodeKey', () => {
+  it('generates a unique index key for a triple (spo)', () => {
+    var key = utils.encodeKey('spo', { subject: 'a-subject', object: 'a-object', predicate: 'a-predicate' })
+    expect(key).to.eql('spo/a-subject/a-predicate/a-object')
+  })
+  it('generates a unique index key for a triple (spo)', () => {
+    var key = utils.encodeKey('sop', { subject: 'a-subject', object: 'a-object', predicate: 'a-predicate' })
+    expect(key).to.eql('sop/a-subject/a-object/a-predicate')
+  })
+  it('generates a unique index key for a triple (osp)', () => {
+    var key = utils.encodeKey('osp', { subject: 'a-subject', object: 'a-object', predicate: 'a-predicate' })
+    expect(key).to.eql('osp/a-object/a-subject/a-predicate')
+  })
+
+  it('escapes forward slashes in the triple', () => {
+    var key = utils.encodeKey('spo', { subject: 'a/subject' })
+    expect(key).to.eql('spo/a%2Fsubject/')
+  })
+})
+
+describe('decodeKey', () => {
+  it('generates a triple from a index key (spo)', () => {
+    var triple = utils.decodeKey('spo/a-subject/a-predicate/a-object')
+    expect(triple).to.eql({ subject: 'a-subject', object: 'a-object', predicate: 'a-predicate' })
+  })
+  it('generates a triple from a index key (spo)', () => {
+    var triple = utils.decodeKey('sop/a-subject/a-object/a-predicate')
+    expect(triple).to.eql({ subject: 'a-subject', object: 'a-object', predicate: 'a-predicate' })
+  })
+  it('generates a triple from a index key (osp)', () => {
+    var triple = utils.decodeKey('osp/a-object/a-subject/a-predicate')
+    expect(triple).to.eql({ subject: 'a-subject', object: 'a-object', predicate: 'a-predicate' })
+  })
+
+  it('unescapes escaped /‘s from index key (spo)', () => {
+    var triple = utils.decodeKey('spo/a%2Fsubject/a%2Fpredicate/a%2Fobject')
+    expect(triple).to.eql({ subject: 'a/subject', object: 'a/object', predicate: 'a/predicate' })
+  })
+})