Skip to content

Commit

Permalink
Merge pull request #147 from mivano/feat/nextlink
Browse files Browse the repository at this point in the history 
An attempt to add paging support in a generic way.
  • Loading branch information
@mivano
mivano authored May 28, 2024
2 parents c3a301c + 500cffd commit 1eb9d89
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 54 deletions.
150 changes: 99 additions & 51 deletions src/CostApi/AzureCostApiRetriever.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ private Uri DeterminePath(Scope scope, string path)

}

private async Task<HttpResponseMessage> ExecuteCallToCostApi(bool includeDebugOutput, object? payload, Uri uri)
private async Task<QueryResponse> ExecutePagedCallToCostApi(bool includeDebugOutput, object? payload, Uri uri)
{
await RetrieveToken(includeDebugOutput);

Expand All @@ -138,22 +138,91 @@ private async Task<HttpResponseMessage> ExecuteCallToCostApi(bool includeDebugOu
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};

var response = payload == null
? await _client.GetAsync(uri)
: await _client.PostAsJsonAsync(uri, payload, options);
QueryResponse? combinedResponse = new QueryResponse { properties = new Properties { rows = new List<JsonElement>() } };

if (includeDebugOutput)
Uri? nextUri = uri;

while (nextUri != null)
{
AnsiConsole.WriteLine(
$"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}");
if (!response.IsSuccessStatusCode)
var response = payload == null
? await _client.GetAsync(nextUri)
: await _client.PostAsJsonAsync(nextUri, payload, options);

if (includeDebugOutput)
{
AnsiConsole.WriteLine(
$"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}");
if (!response.IsSuccessStatusCode)
{
AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}");
}
}

response.EnsureSuccessStatusCode();

QueryResponse? content = await response.Content.ReadFromJsonAsync<QueryResponse>();

if (content == null)
{
AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}");
throw new Exception("Failed to deserialize the response from the API.");
}

combinedResponse.Combine(content);

nextUri = string.IsNullOrEmpty(content.properties.nextLink) ? null : new Uri(content.properties.nextLink);
}

response.EnsureSuccessStatusCode();
return response;
return combinedResponse;
}

private async Task<T?> ExecuteTypedCallToCostApi<T>(bool includeDebugOutput, object? payload, Uri uri)
where T : class
{
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

var content = await response.Content.ReadFromJsonAsync<T>();
return content;
}

private async Task<HttpResponseMessage> ExecuteCallToCostApi(bool includeDebugOutput, object? payload, Uri uri)
{
await RetrieveToken(includeDebugOutput);

if (includeDebugOutput)
{
AnsiConsole.WriteLine($"Retrieving data from {uri} using the following payload:");
AnsiConsole.Write(new JsonText(JsonSerializer.Serialize(payload)));
AnsiConsole.WriteLine();
}

if (!string.Equals(_client.BaseAddress?.ToString(), CostApiAddress))
{
_client.BaseAddress = new Uri(CostApiAddress);
}

var options = new JsonSerializerOptions
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};


var response = payload == null
? await _client.GetAsync(uri)
: await _client.PostAsJsonAsync(uri, payload, options);

if (includeDebugOutput)
{
AnsiConsole.WriteLine(
$"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}");
if (!response.IsSuccessStatusCode)
{
AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}");
}
}

response.EnsureSuccessStatusCode();
return response;
}

public async Task<IEnumerable<CostItem>> RetrieveCosts(bool includeDebugOutput, Scope scope,
Expand Down Expand Up @@ -202,10 +271,8 @@ public async Task<IEnumerable<CostItem>> RetrieveCosts(bool includeDebugOutput,
}
};

var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -275,10 +342,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostByServiceName(bool inc
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -347,10 +412,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostByLocation(bool includ
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -424,10 +487,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostByResourceGroup(bool i
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -501,10 +562,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostBySubscription(bool in
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -579,9 +638,7 @@ public async Task<IEnumerable<CostDailyItem>> RetrieveDailyCost(bool includeDebu
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();
var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostDailyItem>();
foreach (var row in content.properties.rows)
Expand Down Expand Up @@ -632,10 +689,8 @@ public async Task<Subscription> RetrieveSubscription(bool includeDebugOutput, Gu
$"/subscriptions/{subscriptionId}/?api-version=2019-11-01",
UriKind.Relative);

var response = await ExecuteCallToCostApi(includeDebugOutput, null, uri);

var content = await response.Content.ReadFromJsonAsync<Subscription>();

var content = await ExecuteTypedCallToCostApi<Subscription>(includeDebugOutput, null, uri);

if (includeDebugOutput)
{
var json = JsonSerializer.Serialize(content, new JsonSerializerOptions { WriteIndented = true });
Expand Down Expand Up @@ -692,11 +747,9 @@ public async Task<IEnumerable<CostItem>> RetrieveForecastedCosts(bool includeDeb
try
{
// Allow this one to fail, as it is not supported for all subscriptions
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);


foreach (var row in content.properties.rows)
{
var date = DateOnly.ParseExact(row[1].ToString(), "yyyyMMdd", CultureInfo.InvariantCulture);
Expand Down Expand Up @@ -847,10 +900,8 @@ public async Task<IEnumerable<CostResourceItem>> RetrieveCostForResources(bool i
grouping = grouping,
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostResourceItem>();
foreach (JsonElement row in content.properties.rows)
{
Expand Down Expand Up @@ -933,11 +984,8 @@ public async Task<IEnumerable<UsageDetails>> RetrieveUsageDetails(bool includeDe

while (uri != null)
{
var response = await ExecuteCallToCostApi(includeDebugOutput, null, uri);

UsageDetailsResponse payload = await response.Content.ReadFromJsonAsync<UsageDetailsResponse>() ??
new UsageDetailsResponse();

var payload = await ExecuteTypedCallToCostApi<UsageDetailsResponse>(includeDebugOutput, null, uri);

items.AddRange(payload.value);
uri = payload.nextLink != null ? new Uri(payload.nextLink, UriKind.Relative) : null;
}
Expand Down
15 changes: 12 additions & 3 deletions src/CostApi/CostQueryResponse.cs → src/CostApi/QueryResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,23 @@

namespace AzureCostCli.CostApi;

public class CostQueryResponse
public class QueryResponse
{
public object eTag { get; set; }

Check warning on line 7 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'eTag' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public string id { get; set; }

Check warning on line 8 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'id' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public object location { get; set; }

Check warning on line 9 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'location' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public string name { get; set; }
public Properties properties { get; set; }
public string type { get; set; }

// Combine method to merge results
public void Combine(QueryResponse other)
{
if (other?.properties?.rows != null)
{
this.properties.rows.AddRange(other.properties.rows);
}
}
}

public class Columns
Expand All @@ -21,6 +30,6 @@ public class Columns
public class Properties
{
public Columns[] columns { get; set; }

Check warning on line 32 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'columns' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public object nextLink { get; set; }
public JsonElement[] rows { get; set; }
public string nextLink { get; set; }

Check warning on line 33 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'nextLink' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public List<JsonElement> rows { get; set; }

Check warning on line 34 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'rows' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
}

0 comments on commit 1eb9d89

Please sign in to comment.