diff --git a/src/CostApi/AzureCostApiRetriever.cs b/src/CostApi/AzureCostApiRetriever.cs index f889577..ec42a85 100644 --- a/src/CostApi/AzureCostApiRetriever.cs +++ b/src/CostApi/AzureCostApiRetriever.cs @@ -116,7 +116,7 @@ private Uri DeterminePath(Scope scope, string path) } - private async Task ExecuteCallToCostApi(bool includeDebugOutput, object? payload, Uri uri) + private async Task ExecutePagedCallToCostApi(bool includeDebugOutput, object? payload, Uri uri) { await RetrieveToken(includeDebugOutput); @@ -138,22 +138,91 @@ private async Task ExecuteCallToCostApi(bool includeDebugOu PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - var response = payload == null - ? await _client.GetAsync(uri) - : await _client.PostAsJsonAsync(uri, payload, options); + QueryResponse? combinedResponse = new QueryResponse { properties = new Properties { rows = new List() } }; - if (includeDebugOutput) + Uri? nextUri = uri; + + while (nextUri != null) { - AnsiConsole.WriteLine( - $"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}"); - if (!response.IsSuccessStatusCode) + var response = payload == null + ? await _client.GetAsync(nextUri) + : await _client.PostAsJsonAsync(nextUri, payload, options); + + if (includeDebugOutput) + { + AnsiConsole.WriteLine( + $"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}"); + if (!response.IsSuccessStatusCode) + { + AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}"); + } + } + + response.EnsureSuccessStatusCode(); + + QueryResponse? content = await response.Content.ReadFromJsonAsync(); + + if (content == null) { - AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}"); + throw new Exception("Failed to deserialize the response from the API."); } + + combinedResponse.Combine(content); + + nextUri = string.IsNullOrEmpty(content.properties.nextLink) ? null : new Uri(content.properties.nextLink); } - response.EnsureSuccessStatusCode(); - return response; + return combinedResponse; + } + + private async Task ExecuteTypedCallToCostApi(bool includeDebugOutput, object? payload, Uri uri) + where T : class + { + var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); + + var content = await response.Content.ReadFromJsonAsync(); + return content; + } + + private async Task ExecuteCallToCostApi(bool includeDebugOutput, object? payload, Uri uri) + { + await RetrieveToken(includeDebugOutput); + + if (includeDebugOutput) + { + AnsiConsole.WriteLine($"Retrieving data from {uri} using the following payload:"); + AnsiConsole.Write(new JsonText(JsonSerializer.Serialize(payload))); + AnsiConsole.WriteLine(); + } + + if (!string.Equals(_client.BaseAddress?.ToString(), CostApiAddress)) + { + _client.BaseAddress = new Uri(CostApiAddress); + } + + var options = new JsonSerializerOptions + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + + var response = payload == null + ? await _client.GetAsync(uri) + : await _client.PostAsJsonAsync(uri, payload, options); + + if (includeDebugOutput) + { + AnsiConsole.WriteLine( + $"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}"); + if (!response.IsSuccessStatusCode) + { + AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}"); + } + } + + response.EnsureSuccessStatusCode(); + return response; } public async Task> RetrieveCosts(bool includeDebugOutput, Scope scope, @@ -202,10 +271,8 @@ public async Task> RetrieveCosts(bool includeDebugOutput, } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + var items = new List(); foreach (var row in content.properties.rows) { @@ -275,10 +342,8 @@ public async Task> RetrieveCostByServiceName(bool inc filter = GenerateFilters(filter) } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + var items = new List(); foreach (var row in content.properties.rows) { @@ -347,10 +412,8 @@ public async Task> RetrieveCostByLocation(bool includ filter = GenerateFilters(filter) } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + var items = new List(); foreach (var row in content.properties.rows) { @@ -424,10 +487,8 @@ public async Task> RetrieveCostByResourceGroup(bool i filter = GenerateFilters(filter) } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + var items = new List(); foreach (var row in content.properties.rows) { @@ -501,10 +562,8 @@ public async Task> RetrieveCostBySubscription(bool in filter = GenerateFilters(filter) } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + var items = new List(); foreach (var row in content.properties.rows) { @@ -579,9 +638,7 @@ public async Task> RetrieveDailyCost(bool includeDebu filter = GenerateFilters(filter) } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); var items = new List(); foreach (var row in content.properties.rows) @@ -632,10 +689,8 @@ public async Task RetrieveSubscription(bool includeDebugOutput, Gu $"/subscriptions/{subscriptionId}/?api-version=2019-11-01", UriKind.Relative); - var response = await ExecuteCallToCostApi(includeDebugOutput, null, uri); - - var content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecuteTypedCallToCostApi(includeDebugOutput, null, uri); + if (includeDebugOutput) { var json = JsonSerializer.Serialize(content, new JsonSerializerOptions { WriteIndented = true }); @@ -692,11 +747,9 @@ public async Task> RetrieveForecastedCosts(bool includeDeb try { // Allow this one to fail, as it is not supported for all subscriptions - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + foreach (var row in content.properties.rows) { var date = DateOnly.ParseExact(row[1].ToString(), "yyyyMMdd", CultureInfo.InvariantCulture); @@ -847,10 +900,8 @@ public async Task> RetrieveCostForResources(bool i grouping = grouping, } }; - var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri); - - CostQueryResponse? content = await response.Content.ReadFromJsonAsync(); - + var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri); + var items = new List(); foreach (JsonElement row in content.properties.rows) { @@ -933,11 +984,8 @@ public async Task> RetrieveUsageDetails(bool includeDe while (uri != null) { - var response = await ExecuteCallToCostApi(includeDebugOutput, null, uri); - - UsageDetailsResponse payload = await response.Content.ReadFromJsonAsync() ?? - new UsageDetailsResponse(); - + var payload = await ExecuteTypedCallToCostApi(includeDebugOutput, null, uri); + items.AddRange(payload.value); uri = payload.nextLink != null ? new Uri(payload.nextLink, UriKind.Relative) : null; } diff --git a/src/CostApi/CostQueryResponse.cs b/src/CostApi/QueryResponse.cs similarity index 57% rename from src/CostApi/CostQueryResponse.cs rename to src/CostApi/QueryResponse.cs index c1bfeb6..0b9b51c 100644 --- a/src/CostApi/CostQueryResponse.cs +++ b/src/CostApi/QueryResponse.cs @@ -2,7 +2,7 @@ namespace AzureCostCli.CostApi; -public class CostQueryResponse +public class QueryResponse { public object eTag { get; set; } public string id { get; set; } @@ -10,6 +10,15 @@ public class CostQueryResponse public string name { get; set; } public Properties properties { get; set; } public string type { get; set; } + + // Combine method to merge results + public void Combine(QueryResponse other) + { + if (other?.properties?.rows != null) + { + this.properties.rows.AddRange(other.properties.rows); + } + } } public class Columns @@ -21,6 +30,6 @@ public class Columns public class Properties { public Columns[] columns { get; set; } - public object nextLink { get; set; } - public JsonElement[] rows { get; set; } + public string nextLink { get; set; } + public List rows { get; set; } } \ No newline at end of file