WIP: all in using js-multiformats for everying

 - One test suite is commented out
 - bignumber.js was added as dependency to work around a type bug
 - some types should work, but don't, hence there are too many `// @ts-ignores`
 - webworker tests are broken, due to a bug in js-multiformats

Author: vmx

packages/ipfs-unixfs-exporter/package.json

@@ -39,9 +39,6 @@
     "detect-node": "^2.0.4",
     "ipfs-core-types": "^0.3.0",
     "ipfs-unixfs-importer": "^6.0.1",
-    "ipld": "^0.28.0",
-    "ipld-dag-pb": "^0.21.0",
-    "ipld-in-memory": "^7.0.0",
     "it-all": "^1.0.5",
     "it-buffer-stream": "^2.0.0",
     "it-first": "^1.0.6",
@@ -54,7 +51,8 @@
     "uint8arrays": "^2.1.2"
   },
   "dependencies": {
-    "cids": "^1.1.5",
+    "@ipld/dag-cbor": "^3.0.0",
+    "bignumber.js": "^9.0.1",
     "err-code": "^3.0.1",
     "hamt-sharding": "^2.0.0",
     "ipfs-unixfs": "^3.0.1",

packages/ipfs-unixfs-exporter/src/index.js

@@ -1,14 +1,14 @@
 'use strict'
 
 const errCode = require('err-code')
-const CID = require('cids')
+const CID = require('multiformats/cid')
 const resolve = require('./resolvers')
 const last = require('it-last')
 
 /**
  * @typedef {import('ipfs-unixfs')} UnixFS
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
- * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
+ * @typedef {import('ipfs-core-types/src/block-service').BlockService} BlockService
  *
  * @typedef {object} UnixFSFile
  * @property {'file'} type
@@ -81,15 +81,19 @@ const toPathComponents = (path = '') => {
  */
 const cidAndRest = (path) => {
   if (path instanceof Uint8Array) {
+    console.log('vmx: index: path:', path)
     return {
-      cid: new CID(path),
+      // @ts-ignore
+      cid: CID.decode(path),
       toResolve: []
     }
   }
 
-  if (CID.isCID(path)) {
+  // @ts-ignore
+  const cid = CID.asCID(path)
+  if (cid) {
     return {
-      cid: path,
+      cid,
       toResolve: []
     }
   }
@@ -102,7 +106,8 @@ const cidAndRest = (path) => {
     const output = toPathComponents(path)
 
     return {
-      cid: new CID(output[0]),
+      // @ts-ignore
+      cid: CID.parse(output[0]),
       toResolve: output.slice(1)
     }
   }
@@ -112,20 +117,20 @@ const cidAndRest = (path) => {
 
 /**
  * @param {string | CID} path
- * @param {IPLD} ipld
+ * @param {BlockService} blockService
  * @param {ExporterOptions} [options]
  */
-const walkPath = async function * (path, ipld, options = {}) {
+const walkPath = async function * (path, blockService, options = {}) {
   let {
     cid,
     toResolve
   } = cidAndRest(path)
-  let name = cid.toBaseEncodedString()
+  let name = cid.toString()
   let entryPath = name
   const startingDepth = toResolve.length
 
   while (true) {
-    const result = await resolve(cid, name, entryPath, toResolve, startingDepth, ipld, options)
+    const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockService, options)
 
     if (!result.entry && !result.next) {
       throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
@@ -149,11 +154,11 @@ const walkPath = async function * (path, ipld, options = {}) {
 
 /**
  * @param {string | CID} path
- * @param {IPLD} ipld
+ * @param {BlockService} blockService
  * @param {ExporterOptions} [options]
  */
-const exporter = async (path, ipld, options = {}) => {
-  const result = await last(walkPath(path, ipld, options))
+const exporter = async (path, blockService, options = {}) => {
+  const result = await last(walkPath(path, blockService, options))
 
   if (!result) {
     throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
@@ -164,11 +169,11 @@ const exporter = async (path, ipld, options = {}) => {
 
 /**
  * @param {string | CID} path
- * @param {IPLD} ipld
+ * @param {BlockService} blockService
  * @param {ExporterOptions} [options]
  */
-const recursive = async function * (path, ipld, options = {}) {
-  const node = await exporter(path, ipld, options)
+const recursive = async function * (path, blockService, options = {}) {
+  const node = await exporter(path, blockService, options)
 
   if (!node) {
     return

packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

@@ -1,14 +1,16 @@
 'use strict'
 
-const CID = require('cids')
+const CID = require('multiformats/cid')
 const errCode = require('err-code')
+// @ts-ignore
+const dagCbor = require('@ipld/dag-cbor')
 
 /**
  * @type {import('./').Resolver}
  */
-const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
-  const object = await ipld.get(cid, options)
-  const block = await ipld.get(new CID(1, 'raw', cid.multihash))
+const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
+  const block = await blockService.get(cid)
+  const object = dagCbor.decode(block.data)
   let subObject = object
   let subPath = path
 
@@ -20,21 +22,25 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       toResolve.shift()
       subPath = `${subPath}/${prop}`
 
-      if (CID.isCID(subObject[prop])) {
+      // @ts-ignore
+      const subObjectCid = CID.asCID(subObject[prop])
+      if (subObjectCid) {
         return {
           entry: {
             type: 'object',
             name,
             path,
+            // @ts-ignore
             cid,
+            // @ts-ignore
             node: block,
             depth,
             content: async function * () {
               yield object
             }
           },
           next: {
-            cid: subObject[prop],
+            cid: subObjectCid,
             name: prop,
             path: subPath,
             toResolve
@@ -45,7 +51,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       subObject = subObject[prop]
     } else {
       // cannot resolve further
-      throw errCode(new Error(`No property named ${prop} found in cbor node ${cid.toBaseEncodedString()}`), 'ERR_NO_PROP')
+      throw errCode(new Error(`No property named ${prop} found in cbor node ${cid.toString()}`), 'ERR_NO_PROP')
     }
   }
 
@@ -54,7 +60,9 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       type: 'object',
       name,
       path,
+      // @ts-ignore
       cid,
+      // @ts-ignore
       node: block,
       depth,
       content: async function * () {

packages/ipfs-unixfs-exporter/src/resolvers/identity.js

@@ -3,7 +3,7 @@
 const errCode = require('err-code')
 const extractDataFromBlock = require('../utils/extract-data-from-block')
 const validateOffsetAndLength = require('../utils/validate-offset-and-length')
-const mh = require('multihashing-async').multihash
+const mh = require('multiformats/hashes/digest')
 
 /**
  * @typedef {import('../').ExporterOptions} ExporterOptions
@@ -31,18 +31,19 @@ const rawContent = (node) => {
 /**
  * @type {import('./').Resolver}
  */
-const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
+const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
   if (toResolve.length) {
-    throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ERR_NOT_FOUND')
+    throw errCode(new Error(`No link named ${path} found in raw node ${cid.toString()}`), 'ERR_NOT_FOUND')
   }
-
-  const buf = await mh.decode(cid.multihash)
+  // @ts-ignore
+  const buf = await mh.decode(cid.multihash.bytes)
 
   return {
     entry: {
       type: 'identity',
       name,
       path,
+      // @ts-ignore
       cid,
       content: rawContent(buf.digest),
       depth,

packages/ipfs-unixfs-exporter/src/resolvers/index.js

@@ -1,9 +1,10 @@
 'use strict'
 
 const errCode = require('err-code')
+const multicodec = require('multicodec')
 
 /**
- * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
+ * @typedef {import('ipfs-core-types/src/block-service').BlockService} BlockService
  * @typedef {import('../').ExporterOptions} ExporterOptions
  * @typedef {import('../').UnixFSEntry} UnixFSEntry
  * @typedef {import('cids')} CID
@@ -23,30 +24,30 @@ const errCode = require('err-code')
 
 /**
  *
- * @typedef {(cid: CID, name: string, path: string, toResolve: string[], depth: number, ipld: IPLD, options: ExporterOptions) => Promise<ResolveResult>} Resolve
+ * @typedef {(cid: CID, name: string, path: string, toResolve: string[], depth: number, blockService: BlockService, options: ExporterOptions) => Promise<ResolveResult>} Resolve
  *
- * @typedef {(cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, ipld: IPLD, options: ExporterOptions) => Promise<ResolveResult>} Resolver
+ * @typedef {(cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockService: BlockService, options: ExporterOptions) => Promise<ResolveResult>} Resolver
  *
  * @type {{ [ key: string ]: Resolver }}
  */
 const resolvers = {
-  'dag-pb': require('./unixfs-v1'),
-  raw: require('./raw'),
-  'dag-cbor': require('./dag-cbor'),
-  identity: require('./identity')
+  [multicodec.DAG_PB]: require('./unixfs-v1'),
+  [multicodec.RAW]: require('./raw'),
+  [multicodec.DAG_CBOR]: require('./dag-cbor'),
+  [multicodec.IDENTITY]: require('./identity')
 }
 
 /**
  * @type {Resolve}
  */
-function resolve (cid, name, path, toResolve, depth, ipld, options) {
-  const resolver = resolvers[cid.codec]
+function resolve (cid, name, path, toResolve, depth, blockService, options) {
+  const resolver = resolvers[cid.code]
 
   if (!resolver) {
-    throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ERR_NO_RESOLVER')
+    throw errCode(new Error(`No resolver for codec ${multicodec.getName(cid.code)}`), 'ERR_NO_RESOLVER')
   }
 
-  return resolver(cid, name, path, toResolve, resolve, depth, ipld, options)
+  return resolver(cid, name, path, toResolve, resolve, depth, blockService, options)
 }
 
 module.exports = resolve

packages/ipfs-unixfs-exporter/src/resolvers/raw.js

@@ -30,22 +30,24 @@ const rawContent = (node) => {
 /**
  * @type {import('./').Resolver}
  */
-const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
+const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
   if (toResolve.length) {
-    throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ERR_NOT_FOUND')
+    throw errCode(new Error(`No link named ${path} found in raw node ${cid.toString()}`), 'ERR_NOT_FOUND')
   }
 
-  const buf = await ipld.get(cid, options)
+  const block = await blockService.get(cid, options)
 
   return {
     entry: {
       type: 'raw',
       name,
       path,
+      // @ts-ignore
       cid,
-      content: rawContent(buf),
+      content: rawContent(block.data),
       depth,
-      node: buf
+      // @ts-ignore
+      node: block.bytes
     }
   }
 }

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js

@@ -6,7 +6,7 @@
  *
  * @type {import('../').UnixfsV1Resolver}
  */
-const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
+const directoryContent = (cid, node, unixfs, path, resolve, depth, blockService) => {
   /**
    * @param {ExporterOptions} [options]
    * @returns {UnixfsV1DirectoryContent}
@@ -17,7 +17,7 @@ const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
     const links = node.Links.slice(offset, length)
 
     for (const link of links) {
-      const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, ipld, options)
+      const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, blockService, options)
 
       if (result.entry) {
         yield result.entry