Browse Source

add support for an optional handler for each pin log

great for analyzing which files are most in need of optimization
master
ansuz 11 months ago
parent
commit
65f88617cf
2 changed files with 43 additions and 22 deletions
  1. 18
      lib/pins.js
  2. 47
      scripts/tests/test-pins.js

18
lib/pins.js

@ -28,7 +28,8 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
// it's a weird API but it's faster than unpinning manually
var pins = ref.pins = {};
ref.index = 0;
ref.latest = 0;
ref.latest = 0; // the latest message (timestamp in ms)
ref.surplus = 0; // how many lines exist behind a reset
return function (line) {
ref.index++;
if (!Boolean(line)) { return; }
@ -52,6 +53,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
case 'RESET': {
pins = ref.pins = {};
if (l[1] && l[1].length) { l[1].forEach((x) => { ref.pins[x] = 1; }); }
ref.surplus = ref.index;
//jshint -W086
// fallthrough
}
@ -90,9 +92,14 @@ Pins.calculateFromLog = function (pinFile, fileName) {
pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson
*/
const getSafeKeyFromPath = function (path) {
return path.replace(/^.*\//, '').replace(/\.ndjson/, '');
}
Pins.list = function (done, config) {
const pinPath = config.pinPath || './data/pins';
const plan = Plan(5);
const plan = Plan(config.workers || 5);
const handler = config.handler || function () {};
// TODO externalize this via optional handlers?
const stats = {
@ -106,8 +113,12 @@ Pins.list = function (done, config) {
console.log(label, info);
};
const pinned = {};
// TODO replace this with lib-readline?
const streamFile = function (path, cb) {
const id = getSafeKeyFromPath(path);
return void Fs.readFile(path, 'utf8', function (err, body) {
if (err) { return void cb(err); }
const ref = {};
@ -115,6 +126,7 @@ Pins.list = function (done, config) {
var lines = body.split('\n');
stats.lines += lines.length;
lines.forEach(pinHandler);
handler(ref, id, pinned);
cb(void 0, ref);
});
};
@ -130,8 +142,6 @@ Pins.list = function (done, config) {
});
};
const pinned = {};
scanDirectory(pinPath, function (err, paths) {
if (err) { return; } // XXX
paths.forEach(function (path) {

47
scripts/tests/test-pins.js

@ -3,32 +3,43 @@ const Pins = require("../../lib/pins");
var stats = {
users: 0,
lines: 0,
pinned: 0,
events: 0,
lines: 0, // how many lines did you iterate over
surplus: 0, // how many of those lines were not needed?
pinned: 0, // how many files are pinned?
duplicated: 0,
};
var handler = function (ref, id /* safeKey */, pinned) {
if (ref.surplus) {
//console.log("%s has %s trimmable lines", id, ref.surplus);
stats.surplus += ref.surplus;
}
for (var item in ref.pins) {
if (!pinned.hasOwnProperty(item)) {
//console.log("> %s is pinned", item);
stats.pinned++;
} else {
//console.log("> %s was already pinned", item);
stats.duplicated++;
}
}
stats.users++;
stats.lines += ref.index;
//console.log(ref, id);
};
Pins.list(function (err, pinned) {
/*
for (var id in pinned) {
console.log(id);
stats.pinned++;
}
*/
console.log(stats);
}, {
pinPath: require("../../lib/load-config").pinPath
pinPath: require("../../lib/load-config").pinPath,
handler: handler,
});
/*
function (ref, safeKey, pinned) {
stats.users++;
stats.lines += ref.index;
Object.keys(ref.pins).forEach(function (id) {
if (!pinned[id]) {
pinned[id] = true;
stats.pinned++;
}
});
//console.log("pin", stats.events++);
//console.log(ref, safeKey);
}*/
Loading…
Cancel
Save