You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

292 lines
9.8 KiB

  1. var nThen = require("nthen");
  2. var Store = require("../lib/storage/file");
  3. var BlobStore = require("../lib/storage/blob");
  4. var Pins = require("../lib/pins");
  5. var config = require("../lib/load-config");
  6. // the administrator should have set an 'inactiveTime' in their config
  7. // if they didn't, just exit.
  8. if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") { return; }
  9. // files which have not been changed since before this date can be considered inactive
  10. var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
  11. // files which were archived before this date can be considered safe to remove
  12. var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
  13. var getNewestTime = function (stats) {
  14. return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
  15. return stats[b] > stats[a]? b: a;
  16. })];
  17. };
  18. var store;
  19. var pins;
  20. var Log;
  21. var blobs;
  22. var startTime = +new Date();
  23. var msSinceStart = function () {
  24. return (+new Date()) - startTime;
  25. };
  26. nThen(function (w) {
  27. // load the store which will be used for iterating over channels
  28. // and performing operations like archival and deletion
  29. Store.create(config, w(function (err, _) {
  30. if (err) {
  31. w.abort();
  32. throw err;
  33. }
  34. store = _;
  35. })); // load the list of pinned files so you know which files
  36. // should not be archived or deleted
  37. Pins.load(w(function (err, _) {
  38. if (err) {
  39. w.abort();
  40. return void console.error(err);
  41. }
  42. pins = _;
  43. }), {
  44. pinPath: config.pinPath,
  45. });
  46. // load the logging module so that you have a record of which
  47. // files were archived or deleted at what time
  48. var Logger = require("../lib/log");
  49. Logger.create(config, w(function (_) {
  50. Log = _;
  51. }));
  52. config.getSession = function () {};
  53. BlobStore.create(config, w(function (err, _) {
  54. if (err) {
  55. w.abort();
  56. return console.error(err);
  57. }
  58. blobs = _;
  59. }));
  60. }).nThen(function () {
  61. Log.info("EVICT_TIME_TO_LOAD_PINS", msSinceStart());
  62. }).nThen(function (w) {
  63. // this block will iterate over archived channels and removes them
  64. // if they've been in cold storage for longer than your configured archive time
  65. // if the admin has not set an 'archiveRetentionTime', this block makes no sense
  66. // so just skip it
  67. if (typeof(config.archiveRetentionTime) !== "number") { return; }
  68. // count the number of files which have been removed in this run
  69. var removed = 0;
  70. var handler = function (err, item, cb) {
  71. if (err) {
  72. Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err);
  73. return void cb();
  74. }
  75. // don't mess with files that are freshly stored in cold storage
  76. // based on ctime because that's changed when the file is moved...
  77. if (+new Date(item.ctime) > retentionTime) {
  78. return void cb();
  79. }
  80. // but if it's been stored for the configured time...
  81. // expire it
  82. store.removeArchivedChannel(item.channel, w(function (err) {
  83. if (err) {
  84. Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', {
  85. error: err,
  86. channel: item.channel,
  87. });
  88. return void cb();
  89. }
  90. Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel);
  91. removed++;
  92. cb();
  93. }));
  94. };
  95. // if you hit an error, log it
  96. // otherwise, when there are no more channels to process
  97. // log some stats about how many were removed
  98. var done = function (err) {
  99. if (err) {
  100. return Log.error('EVICT_ARCHIVED_FINAL_ERROR', err);
  101. }
  102. Log.info('EVICT_ARCHIVED_CHANNELS_REMOVED', removed);
  103. };
  104. store.listArchivedChannels(handler, w(done));
  105. }).nThen(function (w) {
  106. if (typeof(config.archiveRetentionTime) !== "number") { return; }
  107. // Iterate over archive blob ownership proofs and remove them
  108. // if they are older than the specified retention time
  109. var removed = 0;
  110. blobs.list.archived.proofs(function (err, item, next) {
  111. if (err) {
  112. Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
  113. return void next();
  114. }
  115. if (pins[item.blobId]) { return void next(); }
  116. if (item && getNewestTime(item) > retentionTime) { return void next(); }
  117. blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
  118. if (err) {
  119. Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
  120. return void next();
  121. }
  122. Log.info("EVICT_ARCHIVED_BLOB_PROOF", item);
  123. removed++;
  124. next();
  125. }));
  126. }, w(function () {
  127. Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed);
  128. }));
  129. }).nThen(function (w) {
  130. if (typeof(config.archiveRetentionTime) !== "number") { return; }
  131. // Iterate over archived blobs and remove them
  132. // if they are older than the specified retention time
  133. var removed = 0;
  134. blobs.list.archived.blobs(function (err, item, next) {
  135. if (err) {
  136. Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
  137. return void next();
  138. }
  139. if (pins[item.blobId]) { return void next(); }
  140. if (item && getNewestTime(item) > retentionTime) { return void next(); }
  141. blobs.remove.archived.blob(item.blobId, function (err) {
  142. if (err) {
  143. Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
  144. return void next();
  145. }
  146. Log.info("EVICT_ARCHIVED_BLOB", item);
  147. removed++;
  148. next();
  149. });
  150. }, w(function () {
  151. Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed);
  152. }));
  153. }).nThen(function (w) {
  154. // iterate over blobs and remove them
  155. // if they have not been accessed within the specified retention time
  156. var removed = 0;
  157. blobs.list.blobs(function (err, item, next) {
  158. if (err) {
  159. Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err);
  160. return void next();
  161. }
  162. if (pins[item.blobId]) { return void next(); }
  163. if (item && getNewestTime(item) > retentionTime) { return void next(); }
  164. blobs.archive.blob(item.blobId, function (err) {
  165. if (err) {
  166. Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
  167. error: err,
  168. item: item,
  169. });
  170. return void next();
  171. }
  172. Log.info("EVICT_ARCHIVE_BLOB", {
  173. item: item,
  174. });
  175. removed++;
  176. next();
  177. });
  178. }, w(function () {
  179. Log.info('EVICT_BLOBS_REMOVED', removed);
  180. }));
  181. }).nThen(function (w) {
  182. // iterate over blob proofs and remove them
  183. // if they don't correspond to a pinned or active file
  184. var removed = 0;
  185. blobs.list.proofs(function (err, item, next) {
  186. if (err) {
  187. next();
  188. return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
  189. }
  190. if (pins[item.blobId]) { return void next(); }
  191. if (item && getNewestTime(item) > retentionTime) { return void next(); }
  192. nThen(function (w) {
  193. blobs.size(item.blobId, w(function (err, size) {
  194. if (err) {
  195. w.abort();
  196. next();
  197. return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
  198. }
  199. if (size !== 0) {
  200. w.abort();
  201. next();
  202. }
  203. }));
  204. }).nThen(function () {
  205. blobs.remove.proof(item.safeKey, item.blobId, function (err) {
  206. next();
  207. if (err) {
  208. return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item);
  209. }
  210. removed++;
  211. return Log.info("EVICT_BLOB_PROOF_LONELY", item);
  212. });
  213. });
  214. }, w(function () {
  215. Log.info("EVICT_BLOB_PROOFS_REMOVED", removed);
  216. }));
  217. }).nThen(function (w) {
  218. var channels = 0;
  219. var archived = 0;
  220. var handler = function (err, item, cb) {
  221. channels++;
  222. if (err) {
  223. Log.error('EVICT_CHANNEL_ITERATION', err);
  224. return void cb();
  225. }
  226. // check if the database has any ephemeral channels
  227. // if it does it's because of a bug, and they should be removed
  228. if (item.channel.length === 34) {
  229. return void store.removeChannel(item.channel, w(function (err) {
  230. if (err) {
  231. Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', {
  232. error: err,
  233. channel: item.channel,
  234. });
  235. return void cb();
  236. }
  237. Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel);
  238. cb();
  239. }));
  240. }
  241. // bail out if the channel was modified recently
  242. if (+new Date(item.mtime) > inactiveTime) { return void cb(); }
  243. // ignore the channel if it's pinned
  244. if (pins[item.channel]) { return void cb(); }
  245. return void store.archiveChannel(item.channel, w(function (err) {
  246. if (err) {
  247. Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
  248. error: err,
  249. channel: item.channel,
  250. });
  251. return void cb();
  252. }
  253. Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel);
  254. archived++;
  255. cb();
  256. }));
  257. };
  258. var done = function () {
  259. return void Log.info('EVICT_CHANNELS_ARCHIVED', archived);
  260. };
  261. store.listChannels(handler, w(done));
  262. }).nThen(function () {
  263. Log.info("EVICT_TIME_TO_RUN_SCRIPT", msSinceStart());
  264. }).nThen(function () {
  265. // the store will keep this script running if you don't shut it down
  266. store.shutdown();
  267. Log.shutdown();
  268. });