feat: add glob-source from js-ipfs to be shared (#9)

* feat: add glob-source from js-ipfs

The business of turning a path and pattern into an iterator of files is
duplicated between js-ipfs and js-ipfs-http-client so moving it here to
aid deduplication.

* chore: use env file instead of dep

* test: add test for multiple paths

* feat: make paths (async)iterable or string

Author: achingbrain

package.json

@@ -28,10 +28,12 @@
   "dependencies": {
     "buffer": "^5.2.1",
     "err-code": "^2.0.0",
+    "fs-extra": "^8.1.0",
     "is-buffer": "^2.0.3",
     "is-electron": "^2.2.0",
     "is-pull-stream": "0.0.0",
     "is-stream": "^2.0.0",
+    "it-glob": "0.0.4",
     "kind-of": "^6.0.2",
     "pull-stream-to-async-iterator": "^1.0.2",
     "readable-stream": "^3.4.0"
@@ -40,12 +42,16 @@
     "aegir": "^20.0.0",
     "async-iterator-all": "^1.0.0",
     "chai": "^4.2.0",
+    "chai-as-promised": "^7.1.1",
     "dirty-chai": "^2.0.1",
     "electron": "^6.0.6",
     "electron-mocha": "^8.0.3",
     "pull-stream": "^3.6.13"
   },
   "contributors": [
     "Hugo Dias <[email protected]>"
-  ]
+  ],
+  "browser": {
+    "fs-extra": false
+  }
 }

src/files/glob-source.js

@@ -0,0 +1,93 @@
+'use strict'
+
+const fs = require('fs-extra')
+const glob = require('it-glob')
+const Path = require('path')
+const errCode = require('err-code')
+const kindOf = require('kind-of')
+
+/**
+* Create an async iterator that yields paths that match requested file paths.
+*
+* @param {Iterable|AsyncIterable|String} paths File system path(s) to glob from
+* @param {Object} [options] Optional options
+* @param {Boolean} [options.recursive] Recursively glob all paths in directories
+* @param {Boolean} [options.hidden] Include .dot files in matched paths
+* @param {Array<String>} [options.ignore] Glob paths to ignore
+* @param {Boolean} [options.followSymlinks] follow symlinks
+* @yields {Object} File objects in the form `{ path: String, content: AsyncIterator<Buffer> }`
+*/
+module.exports = async function * globSource (paths, options) {
+  options = options || {}
+
+  if (kindOf(paths) === 'string') {
+    paths = [paths]
+  }
+
+  const globSourceOptions = {
+    recursive: options.recursive,
+    glob: {
+      dot: Boolean(options.hidden),
+      ignore: Array.isArray(options.ignore) ? options.ignore : [],
+      follow: options.followSymlinks != null ? options.followSymlinks : true
+    }
+  }
+
+  // Check the input paths comply with options.recursive and convert to glob sources
+  for await (const path of paths) {
+    if (typeof path !== 'string') {
+      throw errCode(
+        new Error(`Path must be a string`),
+        'ERR_INVALID_PATH',
+        { path }
+      )
+    }
+
+    const absolutePath = Path.resolve(process.cwd(), path)
+    const stat = await fs.stat(absolutePath)
+    const prefix = Path.dirname(absolutePath)
+
+    for await (const entry of toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) {
+      yield entry
+    }
+  }
+}
+
+async function * toGlobSource ({ path, type, prefix }, options) {
+  options = options || {}
+
+  const baseName = Path.basename(path)
+
+  if (type === 'file') {
+    yield {
+      path: baseName.replace(prefix, ''),
+      content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path))
+    }
+
+    return
+  }
+
+  if (type === 'dir' && !options.recursive) {
+    throw errCode(
+      new Error(`'${path}' is a directory and recursive option not set`),
+      'ERR_DIR_NON_RECURSIVE',
+      { path }
+    )
+  }
+
+  const globOptions = Object.assign({}, options.glob, {
+    cwd: path,
+    nodir: true,
+    realpath: false,
+    absolute: true
+  })
+
+  for await (const p of glob(path, '**/*', globOptions)) {
+    yield {
+      path: toPosix(p.replace(prefix, '')),
+      content: fs.createReadStream(p)
+    }
+  }
+}
+
+const toPosix = path => path.replace(/\\/g, '/')

test/files/glob-source.spec.js

@@ -0,0 +1,125 @@
+'use strict'
+
+/* eslint-env mocha */
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const chaiAsPromised = require('chai-as-promised')
+const globSource = require('../../src/files/glob-source')
+const all = require('async-iterator-all')
+const path = require('path')
+const {
+  isNode
+} = require('../../src/env')
+
+chai.use(dirtyChai)
+chai.use(chaiAsPromised)
+const expect = chai.expect
+
+describe('glob-source', () => {
+  it('single file, relative path', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html'))))
+
+    expect(result.length).to.equal(1)
+    expect(result[0].path).to.equal('file-0.html')
+  })
+
+  it('directory, relative path', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+      recursive: true
+    }))
+
+    expect(result.length).to.equal(3)
+    expect(result[0].path).to.equal('/dir/file-1.txt')
+    expect(result[1].path).to.equal('/dir/file-2.js')
+    expect(result[2].path).to.equal('/dir/file-3.css')
+  })
+
+  it('single file, absolute path', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html'))))
+
+    expect(result.length).to.equal(1)
+    expect(result[0].path).to.equal('file-0.html')
+  })
+
+  it('directory, relative path', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+      recursive: true
+    }))
+
+    expect(result.length).to.equal(3)
+    expect(result[0].path).to.equal('/dir/file-1.txt')
+    expect(result[1].path).to.equal('/dir/file-2.js')
+    expect(result[2].path).to.equal('/dir/file-3.css')
+  })
+
+  it('directory, hidden files', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+      recursive: true,
+      hidden: true
+    }))
+
+    expect(result.length).to.equal(4)
+    expect(result[0].path).to.equal('/dir/.hidden.txt')
+    expect(result[1].path).to.equal('/dir/file-1.txt')
+    expect(result[2].path).to.equal('/dir/file-2.js')
+    expect(result[3].path).to.equal('/dir/file-3.css')
+  })
+
+  it('directory, ignore files', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+      recursive: true,
+      ignore: ['**/file-1.txt']
+    }))
+
+    expect(result.length).to.equal(2)
+    expect(result[0].path).to.equal('/dir/file-2.js')
+    expect(result[1].path).to.equal('/dir/file-3.css')
+  })
+
+  it('multiple paths', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource([
+      path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-1.txt')),
+      path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-2.js'))
+    ]))
+
+    expect(result.length).to.equal(2)
+    expect(result[0].path).to.equal('file-1.txt')
+    expect(result[1].path).to.equal('file-2.js')
+  })
+
+  it('requires recursive flag for directory', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    await expect(all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir'))))).to.be.rejectedWith(/recursive option not set/)
+  })
+})