From 6d9c87748154a981431b4582a812cf04b2c4f283 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 17 Mar 2020 19:14:35 +0000 Subject: [PATCH] twitter-service: add metric_metadata endpoint to admin server Problem Finagle stats are currently storing but not exposing Metric Schemas containing useful metadata about metrics. Solution Create an endpoint (/admin/metric_metadata.json) which exposes this metadata in a JSON format, and which allows for metrics to be filtered by name with a query param "name" ala "m" in /admin/metrics.json. Result Users and tools can now query the new /admin/metric_metadata.json endpoint to obtain the stored metadata about a metric. JIRA Issues: CSL-8921 Differential Revision: https://phabricator.twitter.biz/D449149 --- .../main/scala/com/twitter/server/Admin.scala | 7 + .../handler/MetricMetadataQueryHandler.scala | 101 ++++++ .../twitter/server/util/JsonConverter.scala | 1 + .../server/util/MetricSchemaJsonModule.scala | 60 ++++ .../server/util/MetricSchemaSource.scala | 52 +++ .../MetricMetadataQueryHandlerTest.scala | 337 ++++++++++++++++++ .../util/MetricSchemaJsonModuleTest.scala | 91 +++++ .../server/util/MetricSchemaSourceTest.scala | 98 +++++ 8 files changed, 747 insertions(+) create mode 100644 server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala create mode 100644 server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala create mode 100644 server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala create mode 100644 server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala create mode 100644 server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala create mode 100644 server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala diff --git a/server/src/main/scala/com/twitter/server/Admin.scala b/server/src/main/scala/com/twitter/server/Admin.scala index a81eaec0..39b3d328 100644 --- a/server/src/main/scala/com/twitter/server/Admin.scala +++ b/server/src/main/scala/com/twitter/server/Admin.scala @@ -161,6 +161,13 @@ object Admin { group = Some(Grouping.Metrics), includeInIndex = true ), + Route( + path = "/admin/metric_metadata.json", + handler = new MetricMetadataQueryHandler(), + alias = "Metric Metadata", + group = Some(Grouping.Metrics), + includeInIndex = true + ), Route( path = Path.Clients, handler = new ClientRegistryHandler(Path.Clients), diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala new file mode 100644 index 00000000..0f358332 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala @@ -0,0 +1,101 @@ +package com.twitter.server.handler + +import com.twitter.finagle.Service +import com.twitter.finagle.http.{MediaType, Request, Response, Uri} +import com.twitter.io.Buf +import com.twitter.server.util.HttpUtils.newResponse +import com.twitter.server.util.{JsonConverter, MetricSchemaSource} +import com.twitter.util.Future + +/** + * A handler which accepts metrics metadata queries via http query strings and returns + * json encoded metrics with the metadata contained in their MetricSchemas, as well as an indicator + * of whether or not counters are latched. + * + * @note When passing an explicit histogram metric via ?name=, users must provide the raw histogram + * name, no percentile (eg, .p99) appended. + * + * Example Request: + * http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=your/fine/gauge&name=my/only/histo + * + * Response: + * { + * "@version" : 1.0, + * "counters_latched" : false, + * "metrics" : [ + * { + * "name" : "my/cool/counter", + * "kind" : "counter", + * "source" : { + * "class": "finagle.stats.cool", + * "category": "Server", + * "process_path": "dc/role/zone/service" + * }, + * "description" : "Counts how many cools are seen", + * "unit" : "Requests", + * "verbosity": "Verbosity(default)", + * "key_indicator" : true + * }, + * { + * "name" : "your/fine/gauge", + * "kind" : "gauge", + * "source" : { + * "class": "finagle.stats.your", + * "category": "Client", + * "process_path": "dc/your_role/zone/your_service" + * }, + * "description" : "Measures how fine the downstream system is", + * "unit" : "Percentage", + * "verbosity": "Verbosity(debug)", + * "key_indicator" : false + * }, + * { + * "name" : "my/only/histo", + * "kind" : "histogram", + * "source" : { + * "class": "Unspecified", + * "category": "NoRoleSpecified", + * "process_path": "Unspecified" + * }, + * "description" : "No description provided", + * "unit" : "Unspecified", + * "verbosity": "Verbosity(default)", + * "key_indicator" : false, + * "buckets" : [ + * 0.5, + * 0.9, + * 0.99, + * 0.999, + * 0.9999 + * ] + * } + * ]} + */ +class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSource) + extends Service[Request, Response] { + + private[this] def query(keys: Iterable[String]) = + for (k <- keys; e <- source.getSchema(k)) yield e + + def apply(req: Request): Future[Response] = { + val uri = Uri.fromRequest(req) + + val latched = source.hasLatchedCounters + val keysParam = uri.params.getAll("name") + + val metrics = + if (keysParam.isEmpty) source.schemaList + else query(keysParam) + + newResponse( + contentType = MediaType.JsonUtf8, + content = Buf.Utf8( + JsonConverter.writeToString( + Map( + "@version" -> 1.0, + "counters_latched" -> latched, + "metrics" -> metrics + ))) + ) + } +} diff --git a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala index 71dfca9b..d53b26bb 100644 --- a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala +++ b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala @@ -13,6 +13,7 @@ object JsonConverter { factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) val mapper = new ObjectMapper(factory) .registerModule(DefaultScalaModule) + .registerModule(MetricSchemaJsonModule) val printer = new DefaultPrettyPrinter printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala new file mode 100644 index 00000000..50ce5cf0 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaJsonModule.scala @@ -0,0 +1,60 @@ +package com.twitter.server.util + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.finagle.stats.{CounterSchema, GaugeSchema, HistogramSchema, MetricSchema} + +object SchemaSerializer extends StdSerializer[MetricSchema](classOf[MetricSchema]) { + + /** + * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata + * endpoint. + */ +// The impetus for a customer serializer over case class serde is: +// 1) The nested nature of MetricBuilder with MetricSchema means we do not have the ability to +// make decisions about which MetricBuilder fields to include based on MetricSchema type +// (ie, buckets should only be present for HistogramSchema). +// 2) Reshaping the MetricBuilder to be the top level JSON object removes the ability to inject +// the "kind" field based on the MetricSchema type. +// 3) The MetricBuilder class would need to be reworked to have a Source case class +// nested in it which would contain a few of the currently MetricBuilder level values. + def serialize( + metricSchema: MetricSchema, + jsonGenerator: JsonGenerator, + serializerProvider: SerializerProvider + ): Unit = { + jsonGenerator.writeStartObject() + jsonGenerator.writeStringField("name", metricSchema.metricBuilder.name.mkString("/")) + val dataType = metricSchema match { + case _: CounterSchema => "counter" + case _: GaugeSchema => "gauge" + case _: HistogramSchema => "histogram" + } + jsonGenerator.writeStringField("kind", dataType) + jsonGenerator.writeObjectFieldStart("source") + jsonGenerator.writeStringField( + "class", + metricSchema.metricBuilder.sourceClass.getOrElse("Unspecified")) + jsonGenerator.writeStringField("category", metricSchema.metricBuilder.role.toString) + jsonGenerator.writeStringField( + "process_path", + metricSchema.metricBuilder.processPath.getOrElse("Unspecified")) + jsonGenerator.writeEndObject() + jsonGenerator.writeStringField("description", metricSchema.metricBuilder.description) + jsonGenerator.writeStringField("unit", metricSchema.metricBuilder.units.toString) + jsonGenerator.writeStringField("verbosity", metricSchema.metricBuilder.verbosity.toString) + jsonGenerator.writeBooleanField("key_indicator", metricSchema.metricBuilder.keyIndicator) + if (metricSchema.isInstanceOf[HistogramSchema]) { + jsonGenerator.writeArrayFieldStart("buckets") + metricSchema.metricBuilder.percentiles.foreach(bucket => jsonGenerator.writeNumber(bucket)) + jsonGenerator.writeEndArray() + } + jsonGenerator.writeEndObject() + } +} + +object MetricSchemaJsonModule extends SimpleModule { + addSerializer(SchemaSerializer) +} diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala new file mode 100644 index 00000000..864ddb40 --- /dev/null +++ b/server/src/main/scala/com/twitter/server/util/MetricSchemaSource.scala @@ -0,0 +1,52 @@ +package com.twitter.server.util + +import com.twitter.finagle.stats.{MetricSchema, SchemaRegistry} +import com.twitter.finagle.util.LoadService + +private[server] object MetricSchemaSource { + lazy val registry: Seq[SchemaRegistry] = LoadService[SchemaRegistry]() +} + +/** + * A map from stats names to [[com.twitter.finagle.stats.StatEntry StatsEntries]] + * which allows for stale StatEntries up to `refreshInterval`. + */ +private[server] class MetricSchemaSource( + registry: Seq[SchemaRegistry] = MetricSchemaSource.registry) { + + /** + * Indicates whether or not the MetricSource is using latched Counters. + * @note this relies on the fact that there is only one StatsRegistry and that it is + * the finagle implementation. + */ + lazy val hasLatchedCounters: Boolean = { + assert(registry.length > 0) + registry.head.hasLatchedCounters + } + + /** Returns the entry for `key` if it exists */ + def getSchema(key: String): Option[MetricSchema] = synchronized { + registry.map(_.schemas()).find(_.contains(key)).flatMap(_.get(key)) + } + + /** Returns all schemas */ + def schemaList(): Iterable[MetricSchema] = synchronized { + registry + .foldLeft(IndexedSeq[MetricSchema]()) { (seq, r) => + seq ++ r.schemas().values + } + } + + /** Returns true if the map contains `key` and false otherwise. */ + def contains(key: String): Boolean = synchronized { + registry.exists(_.schemas().contains(key)) + } + + /** Returns the set of stat keys. */ + def keySet: Set[String] = synchronized { + registry + .foldLeft(Set[String]()) { (set, r) => + set ++ r.schemas().keySet + } + } +} diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala new file mode 100644 index 00000000..944e4fd1 --- /dev/null +++ b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala @@ -0,0 +1,337 @@ +package com.twitter.server.handler + +import com.twitter.finagle.http.Request +import com.twitter.finagle.stats._ +import com.twitter.server.util.MetricSchemaSource +import com.twitter.util.Await +import org.scalatest.FunSuite + +class MetricMetadataQueryHandlerTest extends FunSuite { + + val schemaMap: Map[String, MetricSchema] = Map( + "my/cool/counter" -> CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "Counts how many cools are seen", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.cool"), + name = Seq("my", "cool", "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "your/fine/gauge" -> GaugeSchema( + new MetricBuilder( + keyIndicator = false, + description = "Measures how fine the downstream system is", + units = Percentage, + role = Client, + verbosity = Verbosity.Debug, + sourceClass = Some("finagle.stats.your"), + name = Seq("your", "fine", "gauge"), + processPath = Some("dc/your_role/zone/your_service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "my/only/histo" -> HistogramSchema( + new MetricBuilder( + name = Seq("my", "only", "histo"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + ) + + trait UnlatchedRegistry extends SchemaRegistry { + val hasLatchedCounters = false + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + trait LatchedRegistry extends SchemaRegistry { + val hasLatchedCounters = true + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + val latchedSchemaRegistry = new LatchedRegistry() {} + + val unlatchedSchemaRegistry = new UnlatchedRegistry() {} + + val latchedMetricSchemaSource = new MetricSchemaSource( + Seq(latchedSchemaRegistry) + ) + + val unlatchedMetricSchemaSource = new MetricSchemaSource( + Seq(unlatchedSchemaRegistry) + ) + + private[this] val latchedHandler = + new MetricMetadataQueryHandler(latchedMetricSchemaSource) + private[this] val unlatchedHandler = + new MetricMetadataQueryHandler(unlatchedMetricSchemaSource) + + val typeRequestNoArg = Request("http://$HOST:$PORT/admin/metric_metadata") + + val typeRequestWithAnArg = Request( + "http://$HOST:$PORT/admin/metric_metadata?name=your/fine/gauge") + + val typeRequestWithManyArgs = Request( + "http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=your/fine/gauge") + + val typeRequestWithHisto = Request("http://$HOST:$PORT/admin/metric_metadata?name=my/only/histo") + + val typeRequestWithHistoAndNon = Request( + "http://$HOST:$PORT/admin/metric_metadata?name=my/cool/counter&name=my/only/histo") + + val responseToNoArg = + """ + | "metrics" : [ + | { + | "name" : "my/cool/counter", + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.cool", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "Counts how many cools are seen", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true + | }, + | { + | "name" : "your/fine/gauge", + | "kind" : "gauge", + | "source" : { + | "class": "finagle.stats.your", + | "category": "Client", + | "process_path": "dc/your_role/zone/your_service" + | }, + | "description" : "Measures how fine the downstream system is", + | "unit" : "Percentage", + | "verbosity": "Verbosity(debug)", + | "key_indicator" : false + | }, + | { + | "name" : "my/only/histo", + | "kind" : "histogram", + | "source" : { + | "class": "Unspecified", + | "category": "NoRoleSpecified", + | "process_path": "Unspecified" + | }, + | "description" : "No description provided", + | "unit" : "Unspecified", + | "verbosity": "Verbosity(default)", + | "key_indicator" : false, + | "buckets" : [ + | 0.5, + | 0.9, + | 0.95, + | 0.99, + | 0.999, + | 0.9999 + | ] + | } + | ] + | } + """.stripMargin + + val responseToAnArg = + """ + | "metrics" : [ + | { + | "name" : "your/fine/gauge", + | "kind" : "gauge", + | "source" : { + | "class": "finagle.stats.your", + | "category": "Client", + | "process_path": "dc/your_role/zone/your_service" + | }, + | "description" : "Measures how fine the downstream system is", + | "unit" : "Percentage", + | "verbosity": "Verbosity(debug)", + | "key_indicator" : false + | } + | ] + | } """.stripMargin + + val responseToManyArgs = + """ + | "metrics" : [ + | { + | "name" : "my/cool/counter", + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.cool", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "Counts how many cools are seen", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true + | }, + | { + | "name" : "your/fine/gauge", + | "kind" : "gauge", + | "source" : { + | "class": "finagle.stats.your", + | "category": "Client", + | "process_path": "dc/your_role/zone/your_service" + | }, + | "description" : "Measures how fine the downstream system is", + | "unit" : "Percentage", + | "verbosity": "Verbosity(debug)", + | "key_indicator" : false + | } + | ] + | } + """.stripMargin + + val responseToHisto = + """ + | "metrics" : [ + | { + | "name" : "my/only/histo", + | "kind" : "histogram", + | "source" : { + | "class": "Unspecified", + | "category": "NoRoleSpecified", + | "process_path": "Unspecified" + | }, + | "description" : "No description provided", + | "unit" : "Unspecified", + | "verbosity": "Verbosity(default)", + | "key_indicator" : false, + | "buckets" : [ + | 0.5, + | 0.9, + | 0.95, + | 0.99, + | 0.999, + | 0.9999 + | ] + | } + | ] + | } """.stripMargin + + val responseToHistoAndNon = + """ + | "metrics" : [ + | { + | "name" : "my/cool/counter", + | "kind" : "counter", + | "source" : { + | "class": "finagle.stats.cool", + | "category": "Server", + | "process_path": "dc/role/zone/service" + | }, + | "description" : "Counts how many cools are seen", + | "unit" : "Requests", + | "verbosity": "Verbosity(default)", + | "key_indicator" : true + | }, + | { + | "name" : "my/only/histo", + | "kind" : "histogram", + | "source" : { + | "class": "Unspecified", + | "category": "NoRoleSpecified", + | "process_path": "Unspecified" + | }, + | "description" : "No description provided", + | "unit" : "Unspecified", + | "verbosity": "Verbosity(default)", + | "key_indicator" : false, + | "buckets" : [ + | 0.5, + | 0.9, + | 0.95, + | 0.99, + | 0.999, + | 0.9999 + | ] + | } + | ] + | } + """.stripMargin + + val testNameStart = "MetricTypeQueryHandler generates reasonable json for " + + // NOTE: these tests assume a specific iteration order over the registries + // and HashMaps which IS NOT a guarantee. should these tests begin to fail + // due to that, we will need a more robust approach to validation. + private[this] def assertJsonResponse(actual: String, expected: String) = { + assert(stripWhitespace(actual) == stripWhitespace(expected)) + } + + private[this] def stripWhitespace(string: String): String = + string.filter { case c => !c.isWhitespace } + + def testCase(latched: Boolean, request: Request): Unit = { + if (request == typeRequestNoArg) { + testCase(latched, request, testNameStart + "full set of metrics", responseToNoArg) + } else if (request == typeRequestWithAnArg) { + testCase(latched, request, testNameStart + "a single requested metric", responseToAnArg) + } else if (request == typeRequestWithManyArgs) { + testCase(latched, request, testNameStart + "requested subset of metrics", responseToManyArgs) + } else if (request == typeRequestWithHisto) { + testCase(latched, request, testNameStart + "a single requested histogram", responseToHisto) + } else if (request == typeRequestWithHistoAndNon) { + testCase( + latched, + request, + testNameStart + "requested subset of metrics with a histogram", + responseToHistoAndNon) + } + } + + def testCase( + latched: Boolean, + request: Request, + testName: String, + responseMetrics: String + ): Unit = { + if (latched) { + val responseStart = + """ + | { + | "@version" : 1.0, + | "counters_latched" : true, + """.stripMargin + test(testName + " when using latched counters") { + assertJsonResponse( + responseStart + responseMetrics, + Await.result(latchedHandler(request)).contentString) + } + + } else { + val responseStart = + """ + | { + | "@version" : 1.0, + | "counters_latched" : false, + """.stripMargin + test(testName + " when using unlatched counters") { + assertJsonResponse( + responseStart + responseMetrics, + Await.result(unlatchedHandler(request)).contentString) + } + + } + } + + Seq( + testCase(true, typeRequestNoArg), + testCase(true, typeRequestWithManyArgs), + testCase(true, typeRequestWithAnArg), + testCase(true, typeRequestWithHisto), + testCase(true, typeRequestWithHistoAndNon), + testCase(false, typeRequestNoArg), + testCase(false, typeRequestWithManyArgs), + testCase(false, typeRequestWithAnArg), + testCase(false, typeRequestWithHisto), + testCase(false, typeRequestWithHistoAndNon) + ) +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala new file mode 100644 index 00000000..aaad8f0e --- /dev/null +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala @@ -0,0 +1,91 @@ +package com.twitter.server.util + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.twitter.finagle.stats._ +import org.scalatest.FunSuite + +class MetricSchemaJsonModuleTest extends FunSuite { + + private val counterSchema = CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "Counts how many cools are seen", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.cool"), + name = Seq("my", "cool", "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + private val gaugeSchema = GaugeSchema( + new MetricBuilder( + keyIndicator = false, + description = "Measures how fine the downstream system is", + units = Percentage, + role = Client, + verbosity = Verbosity.Debug, + sourceClass = Some("finagle.stats.your"), + name = Seq("your", "fine", "gauge"), + processPath = Some("dc/your_role/zone/your_service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + private val histogramSchema = HistogramSchema( + new MetricBuilder( + name = Seq("my", "only", "histo"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + + private val topLevelFieldNameSet = + Set("name", "kind", "source", "description", "unit", "verbosity", "key_indicator") + + def jsonStrToMap(jsonStr: String): Map[String, Any] = { + val mapper = new ObjectMapper() with ScalaObjectMapper + mapper.registerModule(DefaultScalaModule) + mapper.readValue[Map[String, Any]](jsonStr) + } + + test("CounterGauge serializes with kind counter and the correct set of fields") { + val serializedString = JsonConverter.writeToString(counterSchema) + val jsonMap = jsonStrToMap(serializedString) + assert(jsonMap.keys == topLevelFieldNameSet) + assert( + jsonMap.get("source").get == Map( + "class" -> "finagle.stats.cool", + "category" -> "Server", + "process_path" -> "dc/role/zone/service")) + assert(jsonMap.get("kind").get == "counter") + } + + test("GaugeSchema serializes with kind gauge and the correct set of fields") { + val serializedString = JsonConverter.writeToString(gaugeSchema) + val jsonMap = jsonStrToMap(serializedString) + assert(jsonMap.keys == topLevelFieldNameSet) + assert( + jsonMap.get("source").get == Map( + "class" -> "finagle.stats.your", + "category" -> "Client", + "process_path" -> "dc/your_role/zone/your_service")) + assert(jsonMap.get("kind").get == "gauge") + } + + test( + "HistogramSchema serializes with kind histogram and the correct set of fields (includeing buckets)") { + val serializedString = JsonConverter.writeToString(histogramSchema) + val jsonMap = jsonStrToMap(serializedString) + assert(jsonMap.keys == topLevelFieldNameSet ++ Seq("buckets")) + assert( + jsonMap.get("source").get == Map( + "class" -> "Unspecified", + "category" -> "NoRoleSpecified", + "process_path" -> "Unspecified")) + assert(jsonMap.get("kind").get == "histogram") + assert(jsonMap.get("buckets").get == IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999)) + } + +} diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala new file mode 100644 index 00000000..b4b856ce --- /dev/null +++ b/server/src/test/scala/com/twitter/server/util/MetricSchemaSourceTest.scala @@ -0,0 +1,98 @@ +package com.twitter.server.util + +import com.twitter.finagle.stats._ +import org.scalatest.FunSuite + +class MetricSchemaSourceTest extends FunSuite { + + private val schemaMap: Map[String, MetricSchema] = Map( + "my/cool/counter" -> CounterSchema( + new MetricBuilder( + keyIndicator = true, + description = "Counts how many cools are seen", + units = Requests, + role = Server, + verbosity = Verbosity.Default, + sourceClass = Some("finagle.stats.cool"), + name = Seq("my", "cool", "counter"), + processPath = Some("dc/role/zone/service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "your/fine/gauge" -> GaugeSchema( + new MetricBuilder( + keyIndicator = false, + description = "Measures how fine the downstream system is", + units = Percentage, + role = Client, + verbosity = Verbosity.Debug, + sourceClass = Some("finagle.stats.your"), + name = Seq("your", "fine", "gauge"), + processPath = Some("dc/your_role/zone/your_service"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )), + "my/only/histo" -> HistogramSchema( + new MetricBuilder( + name = Seq("my", "only", "histo"), + percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999), + statsReceiver = null + )) + ) + + private val latchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { + override def hasLatchedCounters: Boolean = true + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + private val unlatchedPopulatedRegistry: SchemaRegistry = new SchemaRegistry { + override def hasLatchedCounters: Boolean = false + override def schemas(): Map[String, MetricSchema] = schemaMap + } + + private val metricSchemaSource = new MetricSchemaSource(Seq(latchedPopulatedRegistry)) + private val unlatchedMetricSchemaSource = new MetricSchemaSource(Seq(unlatchedPopulatedRegistry)) + private val emptyMetricSchemaSource = new MetricSchemaSource(Seq()) + + test("hasLatchedCounters asserts if there is no SchemaRegistry") { + assertThrows[AssertionError](emptyMetricSchemaSource.hasLatchedCounters) + } + + test("hasLatchedCounters returns the underlying SchemaRegistry's hasLatchedCounters value") { + assert(metricSchemaSource.hasLatchedCounters) + assert(!unlatchedMetricSchemaSource.hasLatchedCounters) + } + + test("getSchema returns the appropriate MetricSchema when there is one") { + assert(metricSchemaSource.getSchema("my/cool/counter") == schemaMap.get("my/cool/counter")) + } + + test("getSchema returns the None when absent") { + assert(metricSchemaSource.getSchema("my/dull/counter") == None) + } + + test("schemaList returns the full list of MetricSchemas") { + assert(metricSchemaSource.schemaList() == schemaMap.values.toVector) + } + + test("schemaList returns empty list if there is no registry") { + assert(emptyMetricSchemaSource.schemaList() == Seq()) + } + + test("contains accurately reflect the presence or absence of a Metric from the MetricSchema map") { + assert(metricSchemaSource.contains("my/cool/counter")) + assert(!metricSchemaSource.contains("my/dull/counter")) + assert(metricSchemaSource.contains("my/only/histo")) + assert(!emptyMetricSchemaSource.contains("my/cool/counter")) + assert(!emptyMetricSchemaSource.contains("my/dull/counter")) + assert(!emptyMetricSchemaSource.contains("my/only/histo")) + } + + test("keySet returns Set of Metric names (key portion of the schema map)") { + assert(metricSchemaSource.keySet == schemaMap.keySet) + } + + test("keySet returns empty Set if there is no registry") { + assert(emptyMetricSchemaSource.keySet == Set()) + } +}