track high-level app events

app.js

@@ -63,6 +63,7 @@ debug.enable(process.env.DEBUG || debugDefaultCategories);
 
 
 global.cacheStats = {};
+global.appEventStats = {};
 
 
 
@@ -256,6 +257,14 @@ if (rateLimitWindowMinutes == -1) {
 		standardHeaders: 'draft-7', // draft-6: `RateLimit-*` headers; draft-7: combined `RateLimit` header
 		legacyHeaders: false, // Disable the `X-RateLimit-*` headers.
 		skip: function (req, res) {
+			// tor traffic all comes in via tor proxy showing 127.0.0.1
+			// for now, until we identify it as a serious problem, let it pass
+			if (req.hostname.includes(".onion")) {
+				utils.trackAppEvent("torRequest");
+
+				return true;
+			}
+
 			if (req.originalUrl.includes("/snippet/")) {
 				return true;
 			}
@@ -268,7 +277,9 @@ if (rateLimitWindowMinutes == -1) {
 		},
 		handler: function (req, res, next) {
 			debugErrorLog(`Rate-limiting request: req=${JSON.stringify(utils.expressRequestToJson(req))}`);
-
+
+			utils.trackAppEvent("rateLimitedRequest");
+
 			res.status(429).json({
 				message: "Too many requests, please try again later.",
 			});
@@ -600,6 +611,8 @@ async function monitorNewTransactions() {
 	sock.subscribe();
 
 	for await (const [topic, message] of sock) {
+		utils.trackAppEvent("newTransaction");
+
 		console.log(
 			topic.toString("ascii") +
 			" - " +
@@ -1038,6 +1051,8 @@ expressApp.continueStartup = function() {
 };
 
 expressApp.use(function(req, res, next) {
+	utils.trackAppEvent("request");
+
 	req.startTime = Date.now();
 
 	next();
@@ -1187,6 +1202,8 @@ expressApp.use(function(req, res, next) {
 
 /// catch 404 and forwarding to error handler
 expressApp.use(function(req, res, next) {
+	utils.trackAppEvent("error404");
+
 	var err = new Error(`Not Found: ${req ? req.url : 'unknown url'}`);
 	err.status = 404;
 
@@ -1206,6 +1223,8 @@ const sharedErrorHandler = (req, err) => {
 
 		if (crawler) {
 			attributes.crawler = crawler;
+
+			utils.trackAppEvent("crawlRequest", 1, {"crawler": crawler});
 		}
 
 		debugErrorLog(`404 NotFound: path=${path}, ip=${ip}, userAgent=${userAgent} (crawler=${(crawler != null)}${crawler ? crawler : ""})`);

app/utils.js

@@ -1630,6 +1630,35 @@ const perfLogNewItem = (tags) => {
 	};
 };
 
+function trackAppEvent(name, count=1, params=null) {
+	if (!global.appEventStats[name]) {
+		global.appEventStats[name] = {count:0};
+	}
+
+	global.appEventStats[name].count += count;
+	global.appEventStats[name].last = new Date();
+
+	if (params != null) {
+		if (global.appEventStats[name].params == null) {
+			global.appEventStats[name].params = {};
+		}
+
+		let props = objectProperties(params);
+
+		console.log("props=" + JSON.stringify(props));
+
+		props.forEach(prop => {
+			console.log("prop=" + prop);
+
+			if (global.appEventStats[name].params[prop] == null) {
+				global.appEventStats[name].params[prop] = {count: 0};
+			}
+
+			global.appEventStats[name].params[prop].count += count;
+		});
+	}
+}
+
 module.exports = {
 	reflectPromise: reflectPromise,
 	redirectToConnectPageIfNeeded: redirectToConnectPageIfNeeded,
@@ -1693,5 +1722,6 @@ module.exports = {
 	perfLogNewItem: perfLogNewItem,
 	perfLog: perfLog,
 	fileCache: fileCache,
-	expressRequestToJson: expressRequestToJson
+	expressRequestToJson: expressRequestToJson,
+	trackAppEvent: trackAppEvent
 };

routes/adminRouter.js

@@ -37,6 +37,7 @@ router.get("/dashboard", function(req, res, next) {
 	res.locals.electrumStats = global.electrumStats;
 	res.locals.cacheStats = global.cacheStats;
 	res.locals.errorStats = global.errorStats;
+	res.locals.appEventStats = global.appEventStats;
 
 	res.locals.cacheSizes = {
 		misc: {

views/admin/dashboard.pug

@@ -32,6 +32,36 @@ block content
 				pre
 					code.json #{JSON.stringify(appConfig, null, 4)}
 
+			if (JSON.stringify(appEventStats) != "{}")
+				pre #{JSON.stringify(appEventStats)}
+				+contentSection("App Events")
+					.table-responsive
+						table.table.table-borderless.table-hover.table-striped
+							thead
+								tr
+									th.text-start Event
+									th.text-end Count
+									th.text-end Last
+									th.text-end Params
+
+							tbody
+								each item, itemName in appEventStats
+									tr
+										td #{itemName}
+
+										td.text-end #{item.count}
+
+										td.text-end
+											+timestamp(item.last.getTime() / 1000)
+
+										td.text-end
+											if (item.params)
+												each paramItem, paramKey in item.params
+													div
+														| #{paramKey}: #{paramItem.count}
+
+
+
 			+contentSection("Memory Stats")
 				.table-responsive
 					table.table.table-borderless.table-hover.table-striped