feat: alterations

docs/dev_account_token.md

@@ -15,15 +15,17 @@ See: [Broker JWT](/operations_jwt.md)
 
 Teams are encouraged to document the client_id used by a service. This documentation should clearly state the locations the account is used.
 
-## Update the Token into the Github Repo
+Generated tokens are saved in vault 'tools' space for all associated service's by default.
 
-When generate a token, by default the new token will be saved in vault 'tools' space along with project and service folders that this broker account associated with and sync to all Github Repos of services.
+## Update Github Secrets
+
+### Prerequisites
 
 * Have a [Github App](https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/about-creating-github-apps) ready
 
-* Install and Grant the [Github App](https://docs.github.com/en/apps/using-github-apps/installing-your-own-github-app) to have the read/write access to the individual service Github Repo secrets.
+* Install the [Github App](https://docs.github.com/en/apps/using-github-apps/installing-your-own-github-app) in all repositories associated with services. Grant the app read/write access to repository secrets.
 
-* After a token is generated, all secrets under tools/project/service in vault will be synced to this service Github repo secrets.
+* After a token is generated, all secrets in the tools namespace for a service (tools/project/service) in Vault will be synced to the associated service's Github repository as secrets.
 
 ### Renewing a token
 

docs/development.md

@@ -95,6 +95,10 @@ Once started, you must run the vault setup script to bootstrap it. MongoDB must
 $ ./scripts/vault-setup.sh
 ```
 
+#### Github secret sync
+
+To setup a Github App to test secret syncing, set the values GITHUB_CLIENT_ID and GITHUB_PRIVATE_KEY at the Vault path `apps/prod/vault/vsync`.
+
 ## Running Locally
 
 The following assumes the setup steps have occurred and the databases have been successfully bootstrapped.
@@ -110,8 +114,6 @@ The UI should be built before starting the backend server.
 
 ### Running the backend server
 
-After updating GITHUB_CLIENT_ID and GITHUB_PRIVATE_KEY into local vault path apps/prod/vault/vsync
-
 ```bash
 # Run server in watch mode
 # Will source ./scripts/setenv-backend-dev.sh for environment vars

package.json

@@ -74,6 +74,7 @@
     "@types/express": "^4.17.21",
     "@types/express-session": "^1.18.0",
     "@types/jest": "^29.5.12",
+    "@types/libsodium-wrappers": "^0.7.14",
     "@types/lodash.merge": "^4.6.9",
     "@types/node": "^22.5.2",
     "@types/passport": "^1.0.16",

src/collection/account.service.ts

@@ -1,5 +1,10 @@
 import { createHmac, randomUUID } from 'node:crypto';
-import { Injectable, BadRequestException } from '@nestjs/common';
+import {
+  Injectable,
+  BadRequestException,
+  NotFoundException,
+  ServiceUnavailableException,
+} from '@nestjs/common';
 import { Request } from 'express';
 import { Cron, CronExpression } from '@nestjs/schedule';
 import { plainToInstance } from 'class-transformer';
@@ -27,6 +32,7 @@ import { CollectionNameEnum } from '../persistence/dto/collection-dto-union.type
 import { ProjectDto } from '../persistence/dto/project.dto';
 import { RedisService } from '../redis/redis.service';
 import { VaultService } from '../vault/vault.service';
+import { GithubService } from '../github/github.service';
 
 export class TokenCreateDTO {
   token: string;
@@ -37,6 +43,7 @@ export class AccountService {
   constructor(
     private readonly auditService: AuditService,
     private readonly opensearchService: OpensearchService,
+    private readonly githubService: GithubService,
     private readonly vaultService: VaultService,
     private readonly redisService: RedisService,
     private readonly graphRepository: GraphRepository,
@@ -187,6 +194,9 @@ export class AccountService {
     if (patchVault) {
       await this.addTokenToAccountServices(token, account);
     }
+    if (!this.githubService.isEnabled()) {
+      await this.refresh(account.id.toString());
+    }
     this.auditService.recordAccountTokenLifecycle(
       req,
       payload,
@@ -258,7 +268,7 @@ export class AccountService {
       const projectName = projectDtoArr[0].collection.name;
       try {
         await this.addTokenToServiceTools(projectName, serviceName, {
-          [`broker_jwt_${account.clientId.replace(/-/g, '_')}`]: token,
+          [`broker-jwt:${account.clientId}`]: token,
         });
         this.redisService.publish(REDIS_PUBSUB.VAULT_SERVICE_TOKEN, {
           data: {
@@ -301,44 +311,41 @@ export class AccountService {
   }
 
   async refresh(id: string): Promise<void> {
-    try {
-      const account = await this.collectionRepository.getCollectionById(
-        'brokerAccount',
-        id,
-      );
+    const account = await this.collectionRepository.getCollectionById(
+      'brokerAccount',
+      id,
+    );
 
-      if (!account) {
-        throw new Error(`Account with ID ${id} not found`);
-      }
-      const downstreamServices =
-        await this.graphRepository.getDownstreamVertex<ServiceDto>(
-          account.vertex.toString(),
-          CollectionNameEnum.service,
-          3,
+    if (!account) {
+      throw new NotFoundException(`Account with ID ${id} not found`);
+    }
+    if (!this.githubService.isEnabled()) {
+      throw new ServiceUnavailableException();
+    }
+    const downstreamServices =
+      await this.graphRepository.getDownstreamVertex<ServiceDto>(
+        account.vertex.toString(),
+        CollectionNameEnum.service,
+        3,
+      );
+    if (downstreamServices) {
+      for (const service of downstreamServices) {
+        const serviceName = service.collection.name;
+        const projectDtoArr =
+          await this.graphRepository.getUpstreamVertex<ProjectDto>(
+            service.collection.vertex.toString(),
+            CollectionNameEnum.project,
+            null,
+          );
+        const projectName = projectDtoArr[0].collection.name;
+        await this.githubService.refresh(
+          projectName,
+          serviceName,
+          service.collection.scmUrl,
         );
-      if (downstreamServices) {
-        for (const service of downstreamServices) {
-          const serviceName = service.collection.name;
-          const projectDtoArr =
-            await this.graphRepository.getUpstreamVertex<ProjectDto>(
-              service.collection.vertex.toString(),
-              CollectionNameEnum.project,
-              null,
-            );
-          const projectName = projectDtoArr[0].collection.name;
-          this.redisService.publish(REDIS_PUBSUB.VAULT_SERVICE_TOKEN, {
-            data: {
-              clientId: account.clientId,
-              environment: 'tools',
-              project: projectName,
-              service: serviceName,
-              scmUrl: service.collection.scmUrl,
-            },
-          });
-        }
-      } else console.log('No services associated with this broker account');
-    } catch (error) {
-      console.error(error);
+      }
+    } else {
+      // console.log('No services associated with this broker account');
     }
   }
 

src/collection/collection.controller.ts

@@ -101,7 +101,7 @@ export class CollectionController {
     graphIdFromParamKey: 'id',
     permission: 'sudo',
   })
-  @UseGuards(BrokerCombinedAuthGuard)
+  @UseGuards(BrokerOidcAuthGuard)
   async refresh(@Param('id') id: string): Promise<void> {
     return await this.accountService.refresh(id);
   }

src/collection/collection.module.ts

@@ -6,6 +6,7 @@ import { UserCollectionService } from './user-collection.service';
 import { AuditModule } from '../audit/audit.module';
 import { AuthModule } from '../auth/auth.module';
 import { AwsModule } from '../aws/aws.module';
+import { GithubModule } from '../github/github.module';
 import { GraphModule } from '../graph/graph.module';
 import { IntentionModule } from '../intention/intention.module';
 import { PersistenceModule } from '../persistence/persistence.module';
@@ -24,6 +25,7 @@ import { VaultModule } from '../vault/vault.module';
     AuditModule,
     AuthModule,
     PersistenceModule,
+    GithubModule,
     GraphModule,
     forwardRef(() => IntentionModule),
     RedisModule,

src/github/github.health.ts

@@ -0,0 +1,17 @@
+import { Injectable } from '@nestjs/common';
+import { HealthIndicator, HealthIndicatorResult } from '@nestjs/terminus';
+import { GithubService } from './github.service';
+
+@Injectable()
+export class GithubHealthIndicator extends HealthIndicator {
+  constructor(private readonly githubService: GithubService) {
+    super();
+  }
+  async isHealthy(key: string): Promise<HealthIndicatorResult> {
+    const result = this.getStatus(key, this.githubService.isEnabled(), {
+      enabled: this.githubService.isEnabled(),
+    });
+
+    return result;
+  }
+}

src/github/github.module.ts

@@ -2,9 +2,11 @@ import { Module } from '@nestjs/common';
 import { GithubService } from './github.service';
 import { VaultModule } from '../vault/vault.module';
 import { RedisModule } from '../redis/redis.module';
+import { GithubHealthIndicator } from './github.health';
 
 @Module({
   imports: [VaultModule, RedisModule],
-  providers: [GithubService],
+  providers: [GithubService, GithubHealthIndicator],
+  exports: [GithubService, GithubHealthIndicator],
 })
 export class GithubModule {}

src/github/github.service.ts

@@ -1,26 +1,20 @@
-import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
+import axios, { AxiosInstance } from 'axios';
+import { lastValueFrom } from 'rxjs';
+import sodium from 'libsodium-wrappers';
+import * as jwt from 'jsonwebtoken';
 import {
   GITHUB_CLIENT_ID,
   GITHUB_PRIVATE_KEY,
-  REDIS_PUBSUB,
   VAULT_KV_APPS_MOUNT,
 } from '../constants';
-import axios, { AxiosInstance } from 'axios';
-import * as jwt from 'jsonwebtoken';
-import sodium from 'libsodium-wrappers';
-import { RedisService } from '../redis/redis.service';
 import { VaultService } from '../vault/vault.service';
 
 @Injectable()
-export class GithubService implements OnModuleInit {
+export class GithubService {
   private readonly axiosInstance: AxiosInstance;
-  private readonly clientId = GITHUB_CLIENT_ID;
-  private readonly privateKey = GITHUB_PRIVATE_KEY;
 
-  constructor(
-    private readonly vaultService: VaultService,
-    private readonly redisService: RedisService,
-  ) {
+  constructor(private readonly vaultService: VaultService) {
     this.axiosInstance = axios.create({
       baseURL: 'https://api.github.com',
       headers: {
@@ -29,55 +23,40 @@ export class GithubService implements OnModuleInit {
     });
   }
 
-  onModuleInit() {
-    // Subscribe to the Redis channel
-    this.redisService.subscribeAndProcess(
-      REDIS_PUBSUB.VAULT_SERVICE_TOKEN,
-      async (event) => {
-        try {
-          const { project, service, scmUrl } = event.data as {
-            project: string;
-            service: string;
-            scmUrl: string;
-          };
-          const path = `tools/${project}/${service}`;
-          const kvData = await this.vaultService.getKv(
-            VAULT_KV_APPS_MOUNT,
-            path,
-          );
-          if (kvData) {
-            for (const [secretName, secretValue] of Object.entries(kvData)) {
-              if (scmUrl) {
-                await this.updateSecret(
-                  scmUrl,
-                  secretName,
-                  secretValue.toString(),
-                );
-              } else
-                console.log(
-                  'Service does not have Github repo URL to update:',
-                  service,
-                );
-            }
-          }
-        } catch (error) {
-          console.error(
-            'Failed to retrieve KV data or update GitHub secret:',
-            error,
+  public isEnabled() {
+    return GITHUB_CLIENT_ID !== '' && GITHUB_PRIVATE_KEY !== '';
+  }
+
+  public async refresh(project: string, service: string, scmUrl: string) {
+    if (!this.isEnabled()) {
+      throw new Error();
+    }
+    const path = `tools/${project}/${service}`;
+    const kvData = await lastValueFrom(
+      this.vaultService.getKv(VAULT_KV_APPS_MOUNT, path),
+    );
+    if (kvData) {
+      for (const [secretName, secretValue] of Object.entries(kvData)) {
+        if (scmUrl) {
+          await this.updateSecret(scmUrl, secretName, secretValue.toString());
+        } else {
+          console.log(
+            'Service does not have Github repo URL to update:',
+            service,
           );
         }
-      },
-    );
+      }
+    }
   }
 
   // Generate JWT
   private generateJWT(): string {
     const payload = {
       iat: Math.floor(Date.now() / 1000) - 60,
       exp: Math.floor(Date.now() / 1000) + 2 * 60, // JWT expires in 2 minutes
-      iss: this.clientId,
+      iss: GITHUB_CLIENT_ID,
     };
-    return jwt.sign(payload, this.privateKey, { algorithm: 'RS256' });
+    return jwt.sign(payload, GITHUB_PRIVATE_KEY, { algorithm: 'RS256' });
   }
 
   private async getInstallationId(
@@ -198,6 +177,7 @@ export class GithubService implements OnModuleInit {
   ): Promise<void> {
     const { owner, repo } = this.getOwnerAndRepoFromUrl(repoUrl);
     const token = await this.getInstallationAccessToken(owner, repo);
+    const validatedSecretName = secretName.replace(/[^a-zA-Z0-9_]/g, '_');
 
     try {
       if (token) {
@@ -213,7 +193,7 @@ export class GithubService implements OnModuleInit {
         );
         // Update secret
         await this.axiosInstance.put(
-          `/repos/${owner}/${repo}/actions/secrets/${secretName}`,
+          `/repos/${owner}/${repo}/actions/secrets/${validatedSecretName}`,
           {
             encrypted_value: encryptedSecret,
             key_id: keyId,
@@ -225,7 +205,7 @@ export class GithubService implements OnModuleInit {
           },
         );
         console.log(
-          `Secret ${secretName} updated successfully on ${owner}/${repo}!`,
+          `Secret ${validatedSecretName} updated successfully on ${owner}/${repo}!`,
         );
       } else {
         console.log(