|
|
|
@ -1,6 +1,8 @@ |
|
|
|
/* jshint esversion: 6, node: true */ |
|
|
|
const Fs = require('fs'); |
|
|
|
const Path = require("path"); |
|
|
|
const Semaphore = require('saferphore'); |
|
|
|
const Once = require("../lib/once"); |
|
|
|
const nThen = require('nthen'); |
|
|
|
|
|
|
|
const sema = Semaphore.create(20); |
|
|
|
@ -9,7 +11,7 @@ let dirList; |
|
|
|
const fileList = []; |
|
|
|
const pinned = {}; |
|
|
|
|
|
|
|
const hashesFromPinFile = (pinFile, fileName) => { |
|
|
|
const checkPinStatus = (pinFile, fileName) => { |
|
|
|
var pins = {}; |
|
|
|
pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => { |
|
|
|
switch (l[0]) { |
|
|
|
@ -34,25 +36,34 @@ const hashesFromPinFile = (pinFile, fileName) => { |
|
|
|
}; |
|
|
|
|
|
|
|
module.exports.load = function (cb, config) { |
|
|
|
var pinPath = config.pinPath || './pins'; |
|
|
|
var done = Once(cb); |
|
|
|
|
|
|
|
nThen((waitFor) => { |
|
|
|
Fs.readdir('../pins', waitFor((err, list) => { |
|
|
|
// recurse over the configured pinPath, or the default
|
|
|
|
Fs.readdir(pinPath, waitFor((err, list) => { |
|
|
|
if (err) { |
|
|
|
if (err.code === 'ENOENT') { |
|
|
|
dirList = []; |
|
|
|
return; |
|
|
|
return; // this ends up calling back with an empty object
|
|
|
|
} |
|
|
|
throw err; |
|
|
|
waitFor.abort(); |
|
|
|
return void done(err); |
|
|
|
} |
|
|
|
dirList = list; |
|
|
|
})); |
|
|
|
}).nThen((waitFor) => { |
|
|
|
dirList.forEach((f) => { |
|
|
|
sema.take((returnAfter) => { |
|
|
|
Fs.readdir('../pins/' + f, waitFor(returnAfter((err, list2) => { |
|
|
|
if (err) { throw err; } |
|
|
|
// iterate over all the subdirectories in the pin store
|
|
|
|
Fs.readdir(Path.join(pinPath, f), waitFor(returnAfter((err, list2) => { |
|
|
|
if (err) { |
|
|
|
waitFor.abort(); |
|
|
|
return void done(err); |
|
|
|
} |
|
|
|
list2.forEach((ff) => { |
|
|
|
if (config && config.exclude && config.exclude.indexOf(ff) > -1) { return; } |
|
|
|
fileList.push('../pins/' + f + '/' + ff); |
|
|
|
fileList.push(Path.join(pinPath, f, ff)); |
|
|
|
}); |
|
|
|
}))); |
|
|
|
}); |
|
|
|
@ -61,8 +72,11 @@ module.exports.load = function (cb, config) { |
|
|
|
fileList.forEach((f) => { |
|
|
|
sema.take((returnAfter) => { |
|
|
|
Fs.readFile(f, waitFor(returnAfter((err, content) => { |
|
|
|
if (err) { throw err; } |
|
|
|
const hashes = hashesFromPinFile(content.toString('utf8'), f); |
|
|
|
if (err) { |
|
|
|
waitFor.abort(); |
|
|
|
return void done(err); |
|
|
|
} |
|
|
|
const hashes = checkPinStatus(content.toString('utf8'), f); |
|
|
|
hashes.forEach((x) => { |
|
|
|
(pinned[x] = pinned[x] || {})[f.replace(/.*\/([^/]*).ndjson$/, (x, y)=>y)] = 1; |
|
|
|
}); |
|
|
|
@ -70,14 +84,20 @@ module.exports.load = function (cb, config) { |
|
|
|
}); |
|
|
|
}); |
|
|
|
}).nThen(() => { |
|
|
|
cb(pinned); |
|
|
|
done(void 0, pinned); |
|
|
|
}); |
|
|
|
}; |
|
|
|
|
|
|
|
if (!module.parent) { |
|
|
|
module.exports.load(function (data) { |
|
|
|
module.exports.load(function (err, data) { |
|
|
|
if (err) { |
|
|
|
return void console.error(err); |
|
|
|
} |
|
|
|
|
|
|
|
Object.keys(data).forEach(function (x) { |
|
|
|
console.log(x + ' ' + JSON.stringify(data[x])); |
|
|
|
}); |
|
|
|
}, { |
|
|
|
pinPath: require("../config/config").pinPath |
|
|
|
}); |
|
|
|
} |