You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

80 lines
2.6 KiB

  1. /* jshint esversion: 6, node: true */
  2. const Fs = require('fs');
  3. const Path = require("path");
  4. const Semaphore = require('saferphore');
  5. const Once = require("../lib/once");
  6. const nThen = require('nthen');
  7. const Pins = require("../lib/pins");
  8. const sema = Semaphore.create(20);
  9. let dirList;
  10. const fileList = [];
  11. const pinned = {};
  12. module.exports.load = function (cb, config) {
  13. var pinPath = config.pinPath || './pins';
  14. var done = Once(cb);
  15. nThen((waitFor) => {
  16. // recurse over the configured pinPath, or the default
  17. Fs.readdir(pinPath, waitFor((err, list) => {
  18. if (err) {
  19. if (err.code === 'ENOENT') {
  20. dirList = [];
  21. return; // this ends up calling back with an empty object
  22. }
  23. waitFor.abort();
  24. return void done(err);
  25. }
  26. dirList = list;
  27. }));
  28. }).nThen((waitFor) => {
  29. dirList.forEach((f) => {
  30. sema.take((returnAfter) => {
  31. // iterate over all the subdirectories in the pin store
  32. Fs.readdir(Path.join(pinPath, f), waitFor(returnAfter((err, list2) => {
  33. if (err) {
  34. waitFor.abort();
  35. return void done(err);
  36. }
  37. list2.forEach((ff) => {
  38. if (config && config.exclude && config.exclude.indexOf(ff) > -1) { return; }
  39. fileList.push(Path.join(pinPath, f, ff));
  40. });
  41. })));
  42. });
  43. });
  44. }).nThen((waitFor) => {
  45. fileList.forEach((f) => {
  46. sema.take((returnAfter) => {
  47. Fs.readFile(f, waitFor(returnAfter((err, content) => {
  48. if (err) {
  49. waitFor.abort();
  50. return void done(err);
  51. }
  52. const hashes = Pins.calculateFromLog(content.toString('utf8'), f);
  53. hashes.forEach((x) => {
  54. (pinned[x] = pinned[x] || {})[f.replace(/.*\/([^/]*).ndjson$/, (x, y)=>y)] = 1;
  55. });
  56. })));
  57. });
  58. });
  59. }).nThen(() => {
  60. done(void 0, pinned);
  61. });
  62. };
  63. if (!module.parent) {
  64. module.exports.load(function (err, data) {
  65. if (err) {
  66. return void console.error(err);
  67. }
  68. Object.keys(data).forEach(function (x) {
  69. console.log(x + ' ' + JSON.stringify(data[x]));
  70. });
  71. }, {
  72. pinPath: require("../lib/load-config").pinPath
  73. });
  74. }