Merge branch 'master' into configack

faucetsdn · Oct 27, 2023 · 8dcae46 · 8dcae46
2 parents 2a0e2ec + ec66d0d
commit 8dcae46
Show file tree

Hide file tree

Showing 44 changed files with 914 additions and 159 deletions.
diff --git a/.gencode_hash.txt b/.gencode_hash.txt
@@ -3,8 +3,8 @@ d3758eba2529d4a5f1dfd5ed3355a536936b02285ddde7cc75b1f41f4916203a  gencode/docs/c
 2067b2eab579b6f6667d6890dfeb72cb3ec6fbd779668c8ff7dc8f42e3f9a60b  gencode/docs/config.html
 d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673  gencode/docs/config_mapping.html
 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79  gencode/docs/configuration_endpoint.html
-46682449ddb07a7c98e925af7854f93db9789768c479707bfb5af316e5adb403  gencode/docs/configuration_execution.html
-f353c6f23baae7940ecf7a68203924d1217ffafb2802705e5f3e0498bc700e88  gencode/docs/configuration_pod.html
+4f854453e98d79f6dffc7e9f966f115939f679a160e5e8aa93eba0c1245571fa  gencode/docs/configuration_execution.html
+d0f6ea11943f46c955891b3b49a25a9bffcf558fcabb9ed2b0f7667d8413953b  gencode/docs/configuration_pod.html
 1f00351201985451aaa1a3e3247fc138b8bea7943b801c18cd5618f7600c872d  gencode/docs/configuration_pubber.html
 96186777da06f95eae1d16d73555445d23608a9301636ea1ccd17922b3fe4019  gencode/docs/event.html
 587e048c161273b927de67b899204bf0e183db64e59ae513f833e5eff406b1ab  gencode/docs/event_discovery.html
@@ -67,7 +67,7 @@ d3fdb2d8b485c1e61786dfde45cc8be08294339bb31a1b8f02180485789a3ab2  gencode/java/u
 60a8115ae1acae7c199b63180823198d38ec50d57b48dd85aca1ccc865058f85  gencode/java/udmi/schema/GatewayConfig.java
 56b46f4914ef1f4baa59bf597186ff7901b7c8b607720ec798f4e4e6ad59aa08  gencode/java/udmi/schema/GatewayModel.java
 f84fd54baf7529fae4347ac9a8c847f4b48ae594e24b917e47158b632861bf90  gencode/java/udmi/schema/GatewayState.java
-08078f6a1dc3ced76be45a7719d277f57ceba4cdddca7d879224dda36eb60f49  gencode/java/udmi/schema/IotAccess.java
+58881d02bf7047ec9b29d7c80459b0db331e878af8eb9ea978361d855144acd4  gencode/java/udmi/schema/IotAccess.java
 b0d4bff14a65ebddc2dd253c996708a4cce99592e82978f057f32a6d9c7768a1  gencode/java/udmi/schema/Jwt.java
 a5e5adfc187709e8646a11c92e804acfb67743f9d72149008aaca954df3177f6  gencode/java/udmi/schema/Level.java
 07fd4911363437b274c19b024759b04b116152176702da8d4203c4ff4cb55b7f  gencode/java/udmi/schema/LocalnetConfig.java

diff --git a/bin/container b/bin/container
@@ -20,7 +20,6 @@ cmd=$2
 shift 2 || usage missing arguments
 
 cd ${ROOT}/${target}
-DROOT=$PWD
 
 git_branch=$(git rev-parse --abbrev-ref HEAD)
 git_remote=$(git config --get branch.$git_branch.remote) || fail missing git remote
@@ -82,6 +81,7 @@ if [[ $cmd == prep || $cmd == build ]]; then
     rm -rf var tmp && mkdir -p var tmp
 
     cp -f etc/*.json var/ || true
+    cp -a ../schema tmp/
 
     bin/build
 
@@ -99,7 +99,7 @@ fi
 
 if [[ $cmd == build ]]; then
     echo Building Dockerfile.$target
-    docker build -f Dockerfile.$target -t $target $DROOT
+    docker build -f Dockerfile.$target -t $target .
 fi
 
 TEMPLATES=$(cd etc; ls k8s_*.yaml)

diff --git a/bin/support b/bin/support
@@ -2,7 +2,7 @@
 
 ROOT_DIR=$(dirname $0)/..
 
-TIME=`date --utc +%Y%M%d-%H%m%S`
+TIME=`date --utc +%Y%m%d-%H%M%S`
 OUTFILE=/tmp/udmi-support_$TIME.tgz
 OUT_DIR=$(realpath $ROOT_DIR/out --relative-to=$PWD)
 

diff --git a/common/src/main/java/com/google/udmi/util/Common.java b/common/src/main/java/com/google/udmi/util/Common.java
@@ -1,6 +1,7 @@
 package com.google.udmi.util;
 
 import static com.google.udmi.util.GeneralUtils.stackTraceString;
+import static java.util.Optional.ofNullable;
 
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
@@ -19,7 +20,7 @@
  */
 public abstract class Common {
 
-  public static final String STATE_QUERY_TOPIC = "update/query";
+  public static final String UPDATE_QUERY_TOPIC = "update/query";
   public static final String EXCEPTION_KEY = "exception";
   public static final String ERROR_KEY = "error";
   public static final String DETAIL_KEY = "detail";
@@ -34,16 +35,21 @@ public abstract class Common {
   public static final String UDMI_VERSION_KEY = "udmi_version";
   public static final String SUBTYPE_PROPERTY_KEY = "subType";
   public static final String SUBFOLDER_PROPERTY_KEY = "subFolder";
-  public static final String DEVICE_ID_PROPERTY_KEY = "deviceId";
+  public static final String PROJECT_ID_PROPERTY_KEY = "projectId";
   public static final String REGISTRY_ID_PROPERTY_KEY = "deviceRegistryId";
+  public static final String DEVICE_ID_PROPERTY_KEY = "deviceId";
+  public static final String MESSAGE_SOURCE_PROPERTY_KEY = "source";
   public static final String NO_SITE = "--";
   public static final String GCP_REFLECT_KEY_PKCS8 = "reflector/rsa_private.pkcs8";
-  public static final String PREFIX_SEPARATOR = "~";
   public static final String CONDENSER_STRING = "^^";
-  private static final String UDMI_VERSION_ENV = "UDMI_VERSION";
   public static final char DETAIL_SEPARATOR_CHAR = ';';
   public static final String DETAIL_SEPARATOR = DETAIL_SEPARATOR_CHAR + " ";
   public static final Joiner DETAIL_JOINER = Joiner.on(DETAIL_SEPARATOR);
+  public static final String CONFIG_CATEGORY = "config";
+  public static final String COMMANDS_CATEGORY = "commands";
+  public static final String CATEGORY_PROPERTY_KEY = "category";
+  private static final String PREFIX_SEPARATOR = "~";
+  private static final String UDMI_VERSION_ENV = "UDMI_VERSION";
 
   /**
    * Remove the next item from the list in an exception-safe way.
@@ -111,8 +117,8 @@ public static Set<String> allClassesInPackage(Class<?> clazz) {
   }
 
   /**
-   * Load a java class given the name. Converts ClassNotFoundException to
-   * RuntimeException for convenience.
+   * Load a java class given the name. Converts ClassNotFoundException to RuntimeException for
+   * convenience.
    *
    * @param className class to load
    * @return loaded class
@@ -125,7 +131,8 @@ public static Class<?> classForName(String className) {
     }
   }
 
-  public static String getExceptionDetail(Throwable exception, Class<?> container, Function<Throwable, String> customFilter) {
+  public static String getExceptionDetail(Throwable exception, Class<?> container,
+      Function<Throwable, String> customFilter) {
     List<String> messages = new ArrayList<>();
     String previousMessage = null;
     while (exception != null) {
@@ -143,13 +150,17 @@ public static String getExceptionDetail(Throwable exception, Class<?> container,
   private static String getExceptionMessage(Throwable exception, Class<?> container,
       Function<Throwable, String> customFilter) {
     if (customFilter != null) {
-       String customMessage = customFilter.apply(exception);
-       if (customMessage != null) {
-         return customMessage;
-       }
+      String customMessage = customFilter.apply(exception);
+      if (customMessage != null) {
+        return customMessage;
+      }
     }
     String message = getExceptionMessage(exception);
     String line = getExceptionLine(exception, container);
     return message + (line == null ? "" : " @" + line);
   }
+
+  public static String getNamespacePrefix(String udmiNamespace) {
+    return ofNullable(udmiNamespace).map(x -> x + PREFIX_SEPARATOR).orElse("");
+  }
 }
diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java
@@ -256,6 +256,13 @@ public static <T> void ifNotTrueThen(Object conditional, Runnable action) {
     }
   }
 
+  public static <T> T ifTrueGet(Object conditional, Supplier<T> action) {
+    if (isTrue(conditional)) {
+      return action.get();
+    }
+    return null;
+  }
+
   public static <T> void ifTrueThen(Object conditional, Runnable action) {
     if (isTrue(conditional)) {
       action.run();

diff --git a/common/src/main/java/com/google/udmi/util/JsonUtil.java b/common/src/main/java/com/google/udmi/util/JsonUtil.java
@@ -341,6 +341,18 @@ public static Map<String, String> toStringMap(Object message) {
     return map;
   }
 
+  /**
+   * Convert the pojo to a mapped representation of strings only.
+   *
+   * @param message input object to convert
+   * @return object-as-map
+   */
+  public static Map<String, String> toStringMap(String message) {
+    @SuppressWarnings("unchecked")
+    Map<String, String> map = fromString(TreeMap.class, message);
+    return map;
+  }
+
   /**
    * Convert the pojo to a mapped representation.
    *

diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java
@@ -1,7 +1,7 @@
 package com.google.udmi.util;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.udmi.util.Common.PREFIX_SEPARATOR;
+import static com.google.udmi.util.Common.getNamespacePrefix;
 import static com.google.udmi.util.GeneralUtils.ifNullThen;
 import static com.google.udmi.util.JsonUtil.loadFileStrict;
 import static java.util.Objects.requireNonNull;
@@ -39,16 +39,16 @@
 
 public class SiteModel {
 
+  public static final String DEFAULT_GCP_HOSTNAME = "mqtt.googleapis.com";
+  public static final String DEFAULT_CLEARBLADE_HOSTNAME = "us-central1-mqtt.clearblade.com";
+  public static final String DEFAULT_GBOS_HOSTNAME = "mqtt.bos.goog";
   public static final String MOCK_PROJECT = "mock-project";
   private static final String ID_FORMAT = "projects/%s/locations/%s/registries/%s/devices/%s";
   private static final String KEY_SITE_PATH_FORMAT = "%s/devices/%s/%s_private.pkcs8";
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
       .enable(SerializationFeature.INDENT_OUTPUT)
       .setDateFormat(new ISO8601DateFormat())
       .setSerializationInclusion(JsonInclude.Include.NON_NULL);
-  private static final String DEFAULT_GCP_HOSTNAME = "mqtt.googleapis.com";
-  private static final String DEFAULT_CLEARBLADE_HOSTNAME = "mqtt.googleapis.com";
-  private static final String DEFAULT_GBOS_HOSTNAME = "mqtt.googleapis.com";
   private static final Pattern ID_PATTERN = Pattern.compile(
       "projects/(.*)/locations/(.*)/registries/(.*)/devices/(.*)");
 
@@ -185,8 +185,7 @@ public static String getRegistryActual(String namespace,
     if (registry_id == null) {
       return null;
     }
-    String prefix = ofNullable(namespace).map(name -> name + PREFIX_SEPARATOR).orElse("");
-    return prefix + registry_id + ofNullable(registry_suffix).orElse("");
+    return getNamespacePrefix(namespace) + registry_id + ofNullable(registry_suffix).orElse("");
   }
 
   public EndpointConfiguration makeEndpointConfig(String iotProject, String deviceId) {

diff --git a/gencode/docs/configuration_execution.html b/gencode/docs/configuration_execution.html
diff --git a/gencode/docs/configuration_pod.html b/gencode/docs/configuration_pod.html
diff --git a/gencode/java/udmi/schema/IotAccess.java b/gencode/java/udmi/schema/IotAccess.java
diff --git a/pubber/.idea/gradle.xml b/pubber/.idea/gradle.xml
diff --git a/schema/common.json b/schema/common.json
@@ -20,6 +20,7 @@
         "local",
         "dynamic",
         "implicit",
+        "pubsub",
         "jwt",
         "gcp_native",
         "gcp",

diff --git a/selfie/bin/selfie b/selfie/bin/selfie
@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-GCP_SERVICE_ACCOUNT=$(curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email)
+GCP_SERVICE_ACCOUNT=$(curl -s -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email)
 echo Configured with GCP service account $GCP_SERVICE_ACCOUNT
 
 # Cluster info comes from a k8s secret that needs to be externally added.

diff --git a/udmis/.idea/gradle.xml b/udmis/.idea/gradle.xml
diff --git a/udmis/bin/container b/udmis/bin/container
@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+echo
+echo This script has moved, what used to be:
+echo "  udmis/bin/container build"
+echo is now:
+echo "  bin/container udmis build"
+echo
+
+false
diff --git a/udmis/bin/run b/udmis/bin/run
@@ -22,8 +22,10 @@ java -version
 
 POD_JAR=build/libs/udmis-1.0-SNAPSHOT-all.jar
 
-gce_project=$(curl "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google" || true)
 gcloud_project=$(gcloud config get project || true)
+if [[ -z $GCP_PROJECT && -z $gcloud_project ]]; then
+    gce_project=$(curl "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google" || true)
+fi
 
 if [[ -n $GCP_PROJECT ]]; then
     echo Using GCP_PROJECT $GCP_PROJECT from env.
@@ -55,8 +57,10 @@ if [[ -n $CLEARBLADE_CONFIGURATION ]]; then
     echo Extracted ClearBlade project $CLEARBLADE_PROJECT from $CLEARBLADE_CONFIGURATION
 fi
 
-export GCP_SERVICE_ACCOUNT=$(curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/email)
-echo Configured with GCP service account $GCP_SERVICE_ACCOUNT
+if nslookup metadata.google.internal; then
+    export GCP_SERVICE_ACCOUNT=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email)
+    echo Configured with GCP service account $GCP_SERVICE_ACCOUNT
+fi
 
 echo
 echo Final java env:

diff --git a/udmis/etc/prod_pod.json b/udmis/etc/prod_pod.json
@@ -23,7 +23,7 @@
   "iot_access": {
     "iot-access": {
       "provider": "dynamic",
-      "project_id": "gcp-iot-core,clearblade-iot-core"
+      "project_id": "gcp-iot-core, clearblade-iot-core, UDMI-REFLECT"
     },
     "clearblade-iot-core": {
       "provider": "clearblade",
@@ -34,6 +34,11 @@
       "provider": "gcp",
       "project_id": "${GCP_PROJECT}",
       "options": "${GCP_IOT_OPTIONS},distributor=stately"
+    },
+    "UDMI-REFLECT": {
+      "provider": "pubsub",
+      "project_id": "${GCP_PROJECT}",
+      "options": "topic=${UDMI_PREFIX}udmi_reply"
     }
   },
   "distributors": {