-
Notifications
You must be signed in to change notification settings - Fork 629
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
# Description The runner module uses SSM to provide the JIT config or token to the runner. In case the runner does not start healthy the SSM parameter is not deleted. This PR adds a Lambda to remove by default SSM paramaters in the token path that are older then a day. The lambda will be deployed by default as part of the control plane and manage the tokens in the path used by the scale-up runner function. --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Scott Guymer <scott.guymer@philips.com>
- Loading branch information
3 people
authored
Oct 30, 2023
1 parent
f38f20a
commit 340deea
Showing
20 changed files
with
460 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
15 changes: 15 additions & 0 deletions
15
lambdas/functions/control-plane/src/local-ssm-housekeeper.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| import { cleanSSMTokens } from './scale-runners/ssm-housekeeper'; | ||
|
|
||
| export function run(): void { | ||
| cleanSSMTokens({ | ||
| dryRun: true, | ||
| minimumDaysOld: 3, | ||
| tokenPath: '/ghr/my-env/runners/tokens', | ||
| }) | ||
| .then() | ||
| .catch((e) => { | ||
| console.log(e); | ||
| }); | ||
| } | ||
|
|
||
| run(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
117 changes: 117 additions & 0 deletions
117
lambdas/functions/control-plane/src/scale-runners/ssm-housekeeper.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| import { DeleteParameterCommand, GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; | ||
| import { mockClient } from 'aws-sdk-client-mock'; | ||
| import 'aws-sdk-client-mock-jest'; | ||
| import { cleanSSMTokens } from './ssm-housekeeper'; | ||
|
|
||
| process.env.AWS_REGION = 'eu-east-1'; | ||
|
|
||
| const mockSSMClient = mockClient(SSMClient); | ||
|
|
||
| const deleteAmisOlderThenDays = 1; | ||
| const now = new Date(); | ||
| const dateOld = new Date(); | ||
| dateOld.setDate(dateOld.getDate() - deleteAmisOlderThenDays - 1); | ||
|
|
||
| const tokenPath = '/path/to/tokens/'; | ||
|
|
||
| describe('clean SSM tokens / JIT config', () => { | ||
| beforeEach(() => { | ||
| mockSSMClient.reset(); | ||
| mockSSMClient.on(GetParametersByPathCommand).resolves({ | ||
| Parameters: undefined, | ||
| }); | ||
| mockSSMClient.on(GetParametersByPathCommand, { Path: tokenPath }).resolves({ | ||
| Parameters: [ | ||
| { | ||
| Name: tokenPath + 'i-old-01', | ||
| LastModifiedDate: dateOld, | ||
| }, | ||
| ], | ||
| NextToken: 'next', | ||
| }); | ||
| mockSSMClient.on(GetParametersByPathCommand, { Path: tokenPath, NextToken: 'next' }).resolves({ | ||
| Parameters: [ | ||
| { | ||
| Name: tokenPath + 'i-new-01', | ||
| LastModifiedDate: now, | ||
| }, | ||
| ], | ||
| NextToken: undefined, | ||
| }); | ||
| }); | ||
|
|
||
| it('should delete parameters older then minimumDaysOld', async () => { | ||
| await cleanSSMTokens({ | ||
| dryRun: false, | ||
| minimumDaysOld: deleteAmisOlderThenDays, | ||
| tokenPath: tokenPath, | ||
| }); | ||
|
|
||
| expect(mockSSMClient).toHaveReceivedCommandWith(GetParametersByPathCommand, { Path: tokenPath }); | ||
| expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-old-01' }); | ||
| expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-new-01' }); | ||
| }); | ||
|
|
||
| it('should not delete when dry run is activated', async () => { | ||
| await cleanSSMTokens({ | ||
| dryRun: true, | ||
| minimumDaysOld: deleteAmisOlderThenDays, | ||
| tokenPath: tokenPath, | ||
| }); | ||
|
|
||
| expect(mockSSMClient).toHaveReceivedCommandWith(GetParametersByPathCommand, { Path: tokenPath }); | ||
| expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-old-01' }); | ||
| expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-new-01' }); | ||
| }); | ||
|
|
||
| it('should not call delete when no parameters are found.', async () => { | ||
| await expect( | ||
| cleanSSMTokens({ | ||
| dryRun: false, | ||
| minimumDaysOld: deleteAmisOlderThenDays, | ||
| tokenPath: 'no-exist', | ||
| }), | ||
| ).resolves.not.toThrow(); | ||
|
|
||
| expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-old-01' }); | ||
| expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-new-01' }); | ||
| }); | ||
|
|
||
| it('should not error on delete failure.', async () => { | ||
| mockSSMClient.on(DeleteParameterCommand).rejects(new Error('ParameterNotFound')); | ||
|
|
||
| await expect( | ||
| cleanSSMTokens({ | ||
| dryRun: false, | ||
| minimumDaysOld: deleteAmisOlderThenDays, | ||
| tokenPath: tokenPath, | ||
| }), | ||
| ).resolves.not.toThrow(); | ||
| }); | ||
|
|
||
| it('should only accept valid options.', async () => { | ||
| await expect( | ||
| cleanSSMTokens({ | ||
| dryRun: false, | ||
| minimumDaysOld: undefined as unknown as number, | ||
| tokenPath: tokenPath, | ||
| }), | ||
| ).rejects.toBeInstanceOf(Error); | ||
|
|
||
| await expect( | ||
| cleanSSMTokens({ | ||
| dryRun: false, | ||
| minimumDaysOld: 0, | ||
| tokenPath: tokenPath, | ||
| }), | ||
| ).rejects.toBeInstanceOf(Error); | ||
|
|
||
| await expect( | ||
| cleanSSMTokens({ | ||
| dryRun: false, | ||
| minimumDaysOld: 1, | ||
| tokenPath: undefined as unknown as string, | ||
| }), | ||
| ).rejects.toBeInstanceOf(Error); | ||
| }); | ||
| }); |
61 changes: 61 additions & 0 deletions
61
lambdas/functions/control-plane/src/scale-runners/ssm-housekeeper.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| import { DeleteParameterCommand, GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; | ||
| import { logger } from '@terraform-aws-github-runner/aws-powertools-util'; | ||
|
|
||
| export interface SSMCleanupOptions { | ||
| dryRun: boolean; | ||
| minimumDaysOld: number; | ||
| tokenPath: string; | ||
| } | ||
|
|
||
| function validateOptions(options: SSMCleanupOptions): void { | ||
| const errorMessages: string[] = []; | ||
| if (!options.minimumDaysOld || options.minimumDaysOld < 1) { | ||
| errorMessages.push(`minimumDaysOld must be greater then 0, value is set to "${options.minimumDaysOld}"`); | ||
| } | ||
| if (!options.tokenPath) { | ||
| errorMessages.push('tokenPath must be defined'); | ||
| } | ||
| if (errorMessages.length > 0) { | ||
| throw new Error(errorMessages.join(', ')); | ||
| } | ||
| } | ||
|
|
||
| export async function cleanSSMTokens(options: SSMCleanupOptions): Promise<void> { | ||
| logger.info(`Cleaning tokens / JIT config older then ${options.minimumDaysOld} days, dryRun: ${options.dryRun}`); | ||
| logger.debug('Cleaning with options', { options }); | ||
| validateOptions(options); | ||
|
|
||
| const client = new SSMClient({ region: process.env.AWS_REGION }); | ||
| const parameters = await client.send(new GetParametersByPathCommand({ Path: options.tokenPath })); | ||
| while (parameters.NextToken) { | ||
| const nextParameters = await client.send( | ||
| new GetParametersByPathCommand({ Path: options.tokenPath, NextToken: parameters.NextToken }), | ||
| ); | ||
| parameters.Parameters?.push(...(nextParameters.Parameters ?? [])); | ||
| parameters.NextToken = nextParameters.NextToken; | ||
| } | ||
| logger.info(`Found #${parameters.Parameters?.length} parameters in path ${options.tokenPath}`); | ||
| logger.debug('Found parameters', { parameters }); | ||
|
|
||
| // minimumDate = today - minimumDaysOld | ||
| const minimumDate = new Date(); | ||
| minimumDate.setDate(minimumDate.getDate() - options.minimumDaysOld); | ||
|
|
||
| for (const parameter of parameters.Parameters ?? []) { | ||
| if (parameter.LastModifiedDate && new Date(parameter.LastModifiedDate) < minimumDate) { | ||
| logger.info(`Deleting parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); | ||
| try { | ||
| if (!options.dryRun) { | ||
| // sleep 50ms to avoid rait limit | ||
| await new Promise((resolve) => setTimeout(resolve, 50)); | ||
| await client.send(new DeleteParameterCommand({ Name: parameter.Name })); | ||
| } | ||
| } catch (e) { | ||
| logger.warn(`Failed to delete parameter ${parameter.Name} with error ${(e as Error).message}`); | ||
| logger.debug('Failed to delete parameter', { e }); | ||
| } | ||
| } else { | ||
| logger.debug(`Skipping parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.