Add/google prices (#38)

* wip: add google prices

we can add them from the official price api,
albeit it is a bit convoluted. Currently the
prices are close but I think I need to add in
basics for network or another variable because
sometimes they are "spot on" (see what I did there)
and other times not.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/converged-computing/cloud-select/tree/main) (0.0.x)
+ - support for parsing Google prices and preemptible (spot) instances (0.0.25)
  - add support for since to aws spot prices (0.0.24)
  - support for aws spot prices (function to request) (0.0.23)
  - aws prices no longer works to receive an empty NextToken (0.0.22)

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022-2023 LLNS, LLC and other HPCIC DevTools Developers.
+Copyright (c) 2022-2024 LLNS, LLC and other HPCIC DevTools Developers.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

cloudselect/client/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/client/cache.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/client/config.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/client/dbshell.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/client/instance.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/client/shell.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/cloud/aws/client.py

@@ -1,11 +1,12 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
 
 import json
 import random
 import re
+import statistics
 import time
 from datetime import datetime
 
@@ -93,6 +94,36 @@ def prices(self):
                 prices.append(json.loads(pricestr))
             print(f"{len(prices)} total aws prices matching {regex}...", end="\r")
         print()
+
+        # Try to add current spot
+        try:
+            updated = []
+            instances = self.instances()
+            spot_prices = self.spot_prices(instances)
+            for p in prices:
+                instance_type = p["product"]["attributes"].get("instanceType")
+                if not instance_type or instance_type not in spot_prices:
+                    continue
+                region = p["product"]["attributes"]["regionCode"]
+
+                # Save the availability zones, and the mean across
+                sps = {
+                    v["AvailabilityZone"]: float(v["SpotPrice"])
+                    for k, v in spot_prices[instance_type].items()
+                    if k.startswith(region)
+                }
+                if not sps:
+                    continue
+
+                p["terms"]["SpotPrices"] = sps
+                p["terms"]["SpotPrice"] = statistics.mean(list(sps.values()))
+                updated.append(p)
+            prices = updated
+
+        except Exception as e:
+            print(f"Issue with parsing spot prices: {e}")
+            pass
+
         return self.load_prices(prices)
 
     def spot_prices(self, instances, since=None, latest=True):

cloudselect/cloud/aws/instance.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -34,6 +34,12 @@ def attr_price(self):
         """
         return self.data.get("Price")
 
+    def attr_spot_price(self):
+        """
+        Spot price of an instance, USD per hour.
+        """
+        return self.data.get("SpotPrice")
+
     def attr_region(self):
         """
         Return the single region
@@ -90,12 +96,6 @@ def attr_instance_storage(self):
         """
         return self.data.get("InstanceStorageSupported")
 
-    def attr_price_per_hour(self):
-        """
-        Return price per hour, if known.
-        """
-        return self.data.get("Price")
-
     def attr_gpu_model(self):
         """
         Return GPU model, if there are gpus
@@ -213,7 +213,8 @@ def iter_instances(self):
             # Provide finally as single region and price
             for region in regions:
                 item["Region"] = region
-                item["Price"] = item.get("Prices", {}).get(region)
+                item["Price"] = item.get("Prices", {}).get(region, {}).get("OnDemand")
+                item["SpotPrice"] = item.get("Prices", {}).get(region, {}).get("Spot")
                 yield self.Instance(item)
 
     def add_instance_prices(self, prices):
@@ -234,7 +235,7 @@ def add_instance_prices(self, prices):
                 for region in instance["Regions"]:
                     if region in lookup[instance["InstanceType"]]:
                         region_prices[region] = lookup[instance["InstanceType"]][region]
-                    instance["Prices"] = region_prices
+                instance["Prices"] = region_prices
 
 
 def build_instance_price_lookup(prices):
@@ -270,6 +271,9 @@ def build_instance_price_lookup(prices):
         location = price["product"]["attributes"]["regionCode"]
         instance_type = price["product"]["attributes"]["instanceType"]
 
+        # If we have a spot price
+        spot_price = price["terms"].get("SpotPrice")
+
         assert len(price["terms"]["OnDemand"]) == 1
         priceid = list(price["terms"]["OnDemand"].keys())[0]
         for _, rate in price["terms"]["OnDemand"][priceid]["priceDimensions"].items():
@@ -278,8 +282,12 @@ def build_instance_price_lookup(prices):
                     lookup[instance_type] = {}
                 if location in lookup[instance_type]:
                     logger.warning(
-                        f"Found two rates for {instance_type} in {location} - we will choose one but this likely should not happen."
+                        f"Found more than one entry for {instance_type} in {location} - we will choose one but this likely should not happen."
                     )
+
                 # These are provided as strings, since the original data is completely string
-                lookup[instance_type][location] = float(rate["pricePerUnit"]["USD"])
+                lookup[instance_type][location] = {
+                    "OnDemand": float(rate["pricePerUnit"]["USD"]),
+                    "Spot": spot_price,
+                }
     return lookup

cloudselect/cloud/aws/prices.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

cloudselect/cloud/base.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -175,6 +175,7 @@ def generate_row(self):
             "name": self.name,
             "memory": self.attr_memory(),
             "price": self.attr_price(),
+            "spot_price": self.attr_spot_price(),
             "cpus": self.attr_cpus(),
             "gpus": self.attr_gpus(),
             "region": self.attr_region(),

cloudselect/cloud/google/client.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2022-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -28,6 +28,8 @@ def __init__(self, **kwargs):
         self.regions = kwargs.get("regions") or ["us-east1", "us-west1", "us-central1"]
         self.project = None
         self.compute_cli = None
+        # This is the pricing api - billing is for our usage
+        # https://cloud.google.com/billing/docs/how-to/get-pricing-information-api
         self.billing_cli = None
         try:
             self._set_services(kwargs.get("cache_only"))
@@ -36,6 +38,15 @@ def __init__(self, **kwargs):
             self.has_auth = False
         super(GoogleCloud, self).__init__()
 
+    def spot_prices(self, instances, since=None, latest=True):
+        """
+        Get spot prices for a set of instances and availability zones
+
+        This is currently not implemented because we add spot prices to
+        the normal pricing data, but could be added if needed.
+        """
+        pass
+
     def prices(self):
         """
         Use the API to retrieve and return prices to cache.
@@ -116,7 +127,9 @@ def instances(self):
         # Get accelerator types (for GPU) to add to them?
         # TODO - I don't see where we can get memory for GPUs :(
         # https://cloud.google.com/compute/docs/reference/rest/v1/acceleratorTypes
-        # accels = self._retry_request(self.compute_cli.acceleratorTypes().aggregatedList(project=self.project))
+        # accels = self._retry_request(
+        #    self.compute_cli.acceleratorTypes().aggregatedList(project=self.project)
+        # )
 
         # Return a wrapped set of instances
         return self.load_instances(machine_types)