Updated the stats endpoint (#200)

* Performance improvements on the investment task * updated query * Moved the match into the lookup for better performance * Moved calls to create investments into paralel calls * Added information to the stats endoint * Removed commented code and set parameters names * Renamed profile * Added comments * Performance optimization
block-core · Mar 21, 2024 · 134ed27 · 134ed27
1 parent a8278fd
commit 134ed27
Show file tree

Hide file tree

Showing 11 changed files with 308 additions and 90 deletions.
diff --git a/src/Blockcore.Indexer.Angor/AngorStartup.cs b/src/Blockcore.Indexer.Angor/AngorStartup.cs
@@ -32,7 +32,7 @@ public void ConfigureServices(IServiceCollection services)
       services.AddSingleton<IAngorMongoDb, AngorMongoDb>();
 
       services.AddScoped<TaskRunner,ProjectsSyncRunner>();
-      services.AddScoped<TaskRunner,ProjectTransactionsSyncRunner>();
+      services.AddScoped<TaskRunner,ProjectInvestmentsSyncRunner>();
    }
 
    public void Configure(IApplicationBuilder app, IWebHostEnvironment env)

diff --git a/src/Blockcore.Indexer.Angor/Operations/Types/ProjectStats.cs b/src/Blockcore.Indexer.Angor/Operations/Types/ProjectStats.cs
@@ -4,6 +4,7 @@ public class ProjectStats
 {
    public long InvestorCount { get; set; }
    public long AmountInvested { get; set; }
+   public long AmountSpentSoFarByFounder { get; set; }
    public long AmountInPenalties { get; set; }
-   public long CountInPenalties { get; set; }
+   public int CountInPenalties { get; set; }
 }
diff --git a/src/Blockcore.Indexer.Angor/Properties/launchSettings.json b/src/Blockcore.Indexer.Angor/Properties/launchSettings.json
@@ -1,6 +1,6 @@
 {
    "profiles": {
-      "TBTC-signet": {
+      "Angor TBTC-signet": {
          "commandName": "Project",
          "commandLineArgs": "--chain=TBTC --Network:RPCPort=38332",
          "dotnetRunMessages": true,

diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoBuilder.cs
@@ -52,7 +52,7 @@ public override Task OnExecute()
       //TODO move this to the block indexer task runner, but we'll need to move the indexes in there to a different class for each project/blockchain
       AngorMongoDb.InvestmentTable.Indexes
          .CreateOne(new CreateIndexModel<Investment>(Builders<Investment>
-            .IndexKeys.Hashed(_ => _.TransactionIndex)));
+            .IndexKeys.Hashed(_ => _.TransactionId)));
 
       return Task.CompletedTask;
    }

diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/AngorMongoData.cs
@@ -6,9 +6,11 @@
 using Blockcore.Indexer.Core.Settings;
 using Blockcore.Indexer.Core.Storage;
 using Blockcore.Indexer.Core.Storage.Mongo;
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
 using Blockcore.Indexer.Core.Storage.Types;
 using Blockcore.Indexer.Core.Sync;
 using Microsoft.Extensions.Options;
+using MongoDB.Bson;
 using MongoDB.Driver;
 
 namespace Blockcore.Indexer.Angor.Storage.Mongo;
@@ -54,28 +56,170 @@ public AngorMongoData(ILogger<AngorMongoDb> dbLogger, SyncConnection connection,
 
    public async Task<ProjectStats?> GetProjectStatsAsync(string projectId)
    {
-      var project = mongoDb.ProjectTable
+      var projectExists = mongoDb.ProjectTable
          .AsQueryable()
-         .FirstOrDefault(_ => _.AngorKey == projectId);
+         .Any(_ => _.AngorKey == projectId);
 
-      if (project != null)
+      if (projectExists)
       {
-         var total = await mongoDb.InvestmentTable.CountDocumentsAsync(Builders<Investment>.Filter.Eq(_ => _.AngorKey, project.AngorKey));
+         var total = await mongoDb.InvestmentTable.CountDocumentsAsync(Builders<Investment>.Filter.Eq(_ => _.AngorKey, projectId));
 
          var sum = mongoDb.InvestmentTable.AsQueryable()
             .Where(_ => _.AngorKey == projectId)
             .Sum(s => s.AmountSats);
 
+         var spendingSummery = await GetSpentProjectFundsSplitToFounderAndPenalty(projectId);
+
          return new ProjectStats
          {
             InvestorCount = total,
             AmountInvested = sum,
+            AmountSpentSoFarByFounder = spendingSummery.founderSpent,
+            AmountInPenalties = spendingSummery.investorWithdrawn,
+            CountInPenalties = spendingSummery.invetorTrxCount
          };
       }
 
       return null;
    }
 
+
+   private async Task<(long founderSpent, long investorWithdrawn, int invetorTrxCount)> GetSpentProjectFundsSplitToFounderAndPenalty(string projectId)
+   {
+      var founderSummery = "founder"; var investorSummery = "investor"; var total = "total"; var trxCount = "trxCount";
+
+      var outputsSpentSummery = await mongoDb.InvestmentTable.Aggregate(PipelineDefinition<Investment, BsonDocument>.Create(
+            //Filter by project id
+            new BsonDocument("$match",
+               new BsonDocument(nameof(Investment.AngorKey), projectId)),
+            //Break down to object per stage outpoint for the inner join
+            new BsonDocument("$unwind",
+               new BsonDocument("path", "$" + nameof(Investment.StageOutpoint))),
+            // Inner join to input table on outpoint for each stage
+            new BsonDocument("$lookup",
+               new BsonDocument
+               {
+                  { "from", "Input" },
+                  { "localField", nameof(Investment.StageOutpoint) },
+                  { "foreignField", nameof(InputTable.Outpoint) },
+                  { "as", "inputs" }
+               }),
+            new BsonDocument("$unwind", "$inputs"),
+            //Aggregate the value by investment transaction and spending transaction and counting the number of trx in both
+            new BsonDocument("$group",
+               new BsonDocument
+               {
+                  { "_id",
+                     new BsonDocument
+                     {
+                        { "TransactionId", "$" + nameof(Investment.TransactionId) },
+                        { "TrxHash", "$inputs." + nameof(InputTable.TrxHash)}
+                     } },
+                  { "spent",
+                     new BsonDocument("$sum", "$inputs." + nameof(InputTable.Value)) },
+                  { "numTrx",
+                     new BsonDocument("$sum", 1) }
+               }),
+            //Aggregate the value on 1 trx in both transactions or more than 1 in both transactions to identify founder and investor patterns
+            new BsonDocument("$group",
+               new BsonDocument
+               {
+                  { "_id",
+                     new BsonDocument("$cond",
+                        new BsonDocument
+                        {
+                           { "if",
+                              new BsonDocument("$eq",
+                                 new BsonArray
+                                 {
+                                    "$numTrx",
+                                    1
+                                 }) },
+                           { "then", founderSummery },
+                           { "else", investorSummery }
+                        }) },
+                  { total,
+                     new BsonDocument("$sum", "$spent") },
+                  { trxCount,
+                     new BsonDocument("$sum", 1) }
+               })))
+         .ToListAsync();
+
+      var founder = outputsSpentSummery.SingleOrDefault(x => x["_id"] == founderSummery);
+
+      var investor = outputsSpentSummery.SingleOrDefault(x => x["_id"] == investorSummery);
+
+      return (founder?[total].AsInt64 ?? 0, investor?[total].AsInt64 ?? 0, investor?[trxCount].AsInt32 ?? 0);
+   }
+
+   //Not used but kept for now for reference
+   private Task<BsonDocument> GetTotalInvestmentWithdrawn(string projectId)
+   {
+      return mongoDb.InvestmentTable.Aggregate(PipelineDefinition<Investment, BsonDocument>.Create(new[]
+         {
+            new BsonDocument("$match",
+               new BsonDocument("AngorKey", projectId)),
+            new BsonDocument("$lookup",
+               new BsonDocument
+               {
+                  { "from", "Input" },
+                  { "localField", "TransactionId" },
+                  { "foreignField", "Outpoint.TransactionId" },
+                  { "as", "joinedData" }
+               }),
+            new BsonDocument("$unwind", "$joinedData"),
+            new BsonDocument("$group",
+               new BsonDocument
+               {
+                  {
+                     "_id", new BsonDocument { { "AngorKey", "$AngorKey" }, { "TransactionId", "$joinedData.TrxHash" } }
+                  },
+                  { "count", new BsonDocument("$sum", 1) },
+                  { "totalValue", new BsonDocument("$sum", "$joinedData.Value") }
+               }),
+            new BsonDocument("$match",
+               new BsonDocument("count",
+                  new BsonDocument("$gt", 1))),
+            new BsonDocument("$group",
+               new BsonDocument
+               {
+                  { "_id", "$_id.AngorKey" }, { "totalValueSum", new BsonDocument("$sum", "$totalValue") }
+               }),
+            new BsonDocument("$project",
+               new BsonDocument { { "_id", 0 }, { "AngorKey", "$_id" }, { "totalValueSum", 1 } })
+         }))
+         .FirstOrDefaultAsync();
+   }
+
+   /// <summary>
+   /// Sum of all inputs spending outputs from the investment transaction
+   /// </summary>
+   private Task<BsonDocument> GetSumOfOutputsSpentOnProject(string projectId)
+   {
+      return mongoDb.InvestmentTable.Aggregate(PipelineDefinition<Investment, BsonDocument>.Create(new[]
+      {
+         new BsonDocument("$match",
+            new BsonDocument("AngorKey", projectId)),
+         new BsonDocument("$lookup",
+               new BsonDocument
+               {
+                  { "from", "Input" },
+                  { "localField", "TransactionId" },
+                  { "foreignField", "Outpoint.TransactionId" },
+                  { "as", "inputs" }
+               }),
+            new BsonDocument("$unwind",
+               new BsonDocument("path", "$inputs")),
+            new BsonDocument("$group",
+               new BsonDocument
+               {
+                  { "_id", "$AngorKey" },
+                  { "Spent",
+                     new BsonDocument("$sum", "$inputs.Value") }
+               })
+         })).FirstOrDefaultAsync();
+   }
+
    public async Task<QueryResult<ProjectIndexerData>> GetProjectsAsync(int? offset, int limit)
    {
       long total = await mongoDb.ProjectTable.CountDocumentsAsync(FilterDefinition<Project>.Empty);
@@ -115,7 +259,7 @@ public async Task<QueryResult<ProjectInvestment>> GetProjectInvestmentsAsync(str
          .Take(limit)
          .Select(_ => new ProjectInvestment
          {
-            TransactionId = _.TransactionIndex,
+            TransactionId = _.TransactionId,
             TotalAmount = _.AmountSats,
             InvestorPublicKey = _.InvestorPubKey,
             HashOfSecret = _.SecretHash
@@ -138,7 +282,7 @@ public Task<ProjectInvestment> GetInvestmentsByInvestorPubKeyAsync(string invest
          .Project(_ => new ProjectInvestment
          {
             TotalAmount = _.AmountSats,
-            TransactionId = _.TransactionIndex,
+            TransactionId = _.TransactionId,
             InvestorPublicKey = _.InvestorPubKey,
             HashOfSecret = _.SecretHash
          })

diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs
@@ -1,3 +1,5 @@
+using Blockcore.Indexer.Core.Storage.Mongo.Types;
+
 namespace Blockcore.Indexer.Angor.Storage.Mongo.Types;
 
 public class Investment
@@ -10,9 +12,9 @@ public class Investment
 
    public long BlockIndex { get; set; }
 
-   public string TransactionIndex { get; set; }
+   public string TransactionId { get; set; }
 
    public long AmountSats { get; set; }
 
-
+   public List<Outpoint> StageOutpoint { get; set; }
 }