[mdatagen] copy files from contrib to core (#9172)

This is to allow us to use it in core as well

---------

Signed-off-by: Alex Boten <aboten@lightstep.com>

.chloggen/codeboten_mdatagen.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: move component from contrib to core
+
+# One or more tracking issues or pull requests related to the change
+issues: [9172]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.golangci.yml

@@ -17,6 +17,7 @@ run:
   # default value is empty list, but default dirs are skipped independently
   # from this option's value (see skip-dirs-use-default).
   skip-dirs:
+    - third_party
 
   # default is true. Enables skipping of directories:
   #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$

Makefile

@@ -3,6 +3,7 @@ include ./Makefile.Common
 # This is the code that we want to run lint, etc.
 ALL_SRC := $(shell find . -name '*.go' \
 							-not -path './internal/tools/*' \
+							-not -path '*/third_party/*' \
 							-not -path './pdata/internal/data/protogen/*' \
 							-not -path './service/internal/zpages/tmplgen/*' \
 							-type f | sort)
@@ -57,7 +58,7 @@ gotest-with-cover:
 
 .PHONY: goporto
 goporto: $(PORTO)
-	$(PORTO) -w --include-internal ./
+	$(PORTO) -w --include-internal --skip-dirs "^cmd/mdatagen/third_party$$" ./
 
 .PHONY: golint
 golint:
@@ -77,7 +78,9 @@ gotidy:
 
 .PHONY: gogenerate
 gogenerate:
+	cd cmd/mdatagen && $(GOCMD) install .
 	@$(MAKE) for-all-target TARGET="generate"
+	$(MAKE) fmt
 
 .PHONY: addlicense
 addlicense: $(ADDLICENSE)

cmd/mdatagen/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

cmd/mdatagen/README.md

@@ -0,0 +1,56 @@
+# Metadata Generator
+
+Every component's documentation should include a brief description of the component and guidance on how to use it.
+There is also some information about the component (or metadata) that should be included to help end-users understand the current state of the component and whether it is right for their use case.
+Examples of this metadata about a component are:
+
+* its stability level
+* the distributions containing it
+* the types of pipelines it supports
+* metrics emitted in the case of a scraping receiver
+
+The metadata generator defines a schema for specifying this information to ensure it is complete and well-formed.
+The metadata generator is then able to ingest the metadata, validate it against the schema and produce documentation in a standardized format.
+An example of how this generated documentation looks can be found in [documentation.md](./documentation.md).
+
+## Using the Metadata Generator
+
+In order for a component to benefit from the metadata generator (`mdatagen`) these requirements need to be met:
+1. A `metadata.yaml` file containing the metadata needs to be included in the component
+2. The component should declare a `go:generate mdatagen` directive which tells `mdatagen` what to generate
+
+As an example, here is a minimal `metadata.yaml` for the [OTLP receiver](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver):
+```yaml
+type: otlp
+status:
+  class: receiver
+  stability:
+    beta: [logs]
+    stable: [metrics, traces]
+```
+
+Detailed information about the schema of `metadata.yaml` can be found in [metadata-schema.yaml](./metadata-schema.yaml).
+
+The `go:generate mdatagen` directive is usually defined in a `doc.go` file in the same package as the component, for example:
+```go
+//go:generate mdatagen metadata.yaml
+
+package main
+```
+
+Below are some more examples that can be used for reference:
+
+* The ElasticSearch receiver has an extensive [metadata.yaml](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/elasticsearchreceiver/metadata.yaml)
+* The host metrics receiver has internal subcomponents, each with their own `metadata.yaml` and `doc.go`. See [cpuscraper](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver/internal/scraper/cpuscraper) for example.
+
+You can run `cd cmd/mdatagen && $(GOCMD) install .` to install the `mdatagen` tool in `GOBIN` and then run `mdatagen metadata.yaml` to generate documentation for a specific component or you can run `make generate` to generate documentation for all components.
+
+## Contributing to the Metadata Generator
+
+The code for generating the documentation can be found in [loader.go](./loader.go) and the templates for rendering the documentation can be found in [templates](./templates).
+When making updates to the metadata generator or introducing support for new functionality:
+
+1. Ensure the [metadata-schema.yaml](./metadata-schema.yaml) and [./metadata.yaml](metadata.yaml) files reflect the changes.
+2. Run `make mdatagen-test`.
+3. Make sure all tests are passing including [generated tests](./internal/metadata/generated_metrics_test.go).
+4. Run `make generate`.

cmd/mdatagen/doc.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Generate a test metrics builder from a sample metrics set covering all configuration options.
+//go:generate mdatagen metadata-sample.yaml
+
+package main

cmd/mdatagen/documentation.md

@@ -0,0 +1,96 @@
+[comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
+
+# file
+
+## Default Metrics
+
+The following metrics are emitted by default. Each of them can be disabled by applying the following configuration:
+
+```yaml
+metrics:
+  <metric_name>:
+    enabled: false
+```
+
+### default.metric
+
+Monotonic cumulative sum int metric enabled by default.
+
+The metric will be become optional soon.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic |
+| ---- | ----------- | ---------- | ----------------------- | --------- |
+| s | Sum | Int | Cumulative | true |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| string_attr | Attribute with any string value. | Any Str |
+| state | Integer attribute with overridden name. | Any Int |
+| enum_attr | Attribute with a known set of string values. | Str: ``red``, ``green``, ``blue`` |
+| slice_attr | Attribute with a slice value. | Any Slice |
+| map_attr | Attribute with a map value. | Any Map |
+
+### default.metric.to_be_removed
+
+[DEPRECATED] Non-monotonic delta sum double metric enabled by default.
+
+The metric will be will be removed soon.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic |
+| ---- | ----------- | ---------- | ----------------------- | --------- |
+| s | Sum | Double | Delta | false |
+
+## Optional Metrics
+
+The following metrics are not emitted by default. Each of them can be enabled by applying the following configuration:
+
+```yaml
+metrics:
+  <metric_name>:
+    enabled: true
+```
+
+### optional.metric
+
+[DEPRECATED] Gauge double metric disabled by default.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| 1 | Gauge | Double |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| string_attr | Attribute with any string value. | Any Str |
+| boolean_attr | Attribute with a boolean value. | Any Bool |
+
+### optional.metric.empty_unit
+
+[DEPRECATED] Gauge double metric disabled by default.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+|  | Gauge | Double |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| string_attr | Attribute with any string value. | Any Str |
+| boolean_attr | Attribute with a boolean value. | Any Bool |
+
+## Resource Attributes
+
+| Name | Description | Values | Enabled |
+| ---- | ----------- | ------ | ------- |
+| map.resource.attr | Resource attribute with a map value. | Any Map | true |
+| optional.resource.attr | Explicitly disabled ResourceAttribute. | Any Str | false |
+| slice.resource.attr | Resource attribute with a slice value. | Any Slice | true |
+| string.enum.resource.attr | Resource attribute with a known set of string values. | Str: ``one``, ``two`` | true |
+| string.resource.attr | Resource attribute with any string value. | Any Str | true |
+| string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true |
+| string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false |
+| string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true |

cmd/mdatagen/embeded_templates.go

@@ -0,0 +1,13 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import "embed"
+
+// templateFS ensures that the files needed
+// to generate metadata as an embedded filesystem since
+// `go get` doesn't require these files to be downloaded.
+//
+//go:embed templates/*.tmpl templates/testdata/*.tmpl
+var templateFS embed.FS

cmd/mdatagen/embeded_templates_test.go

@@ -0,0 +1,47 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"io/fs"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnsureTemplatesLoaded(t *testing.T) {
+	t.Parallel()
+
+	const (
+		rootDir = "templates"
+	)
+
+	var (
+		templateFiles = map[string]struct{}{
+			path.Join(rootDir, "component_test.go.tmpl"):       {},
+			path.Join(rootDir, "documentation.md.tmpl"):        {},
+			path.Join(rootDir, "metrics.go.tmpl"):              {},
+			path.Join(rootDir, "metrics_test.go.tmpl"):         {},
+			path.Join(rootDir, "resource.go.tmpl"):             {},
+			path.Join(rootDir, "resource_test.go.tmpl"):        {},
+			path.Join(rootDir, "config.go.tmpl"):               {},
+			path.Join(rootDir, "config_test.go.tmpl"):          {},
+			path.Join(rootDir, "readme.md.tmpl"):               {},
+			path.Join(rootDir, "status.go.tmpl"):               {},
+			path.Join(rootDir, "testdata", "config.yaml.tmpl"): {},
+		}
+		count = 0
+	)
+	assert.NoError(t, fs.WalkDir(templateFS, ".", func(path string, d fs.DirEntry, err error) error {
+		if d != nil && d.IsDir() {
+			return nil
+		}
+		count++
+		assert.Contains(t, templateFiles, path)
+		return nil
+	}))
+	assert.Equal(t, len(templateFiles), count, "Must match the expected number of calls")
+
+}

cmd/mdatagen/go.mod

@@ -0,0 +1,77 @@
+module go.opentelemetry.io/collector/cmd/mdatagen
+
+go 1.20
+
+require (
+	github.com/google/go-cmp v0.6.0
+	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/collector/component v0.91.0
+	go.opentelemetry.io/collector/confmap v0.91.0
+	go.opentelemetry.io/collector/pdata v1.0.0
+	go.opentelemetry.io/collector/receiver v0.91.0
+	go.opentelemetry.io/collector/semconv v0.91.0
+	go.opentelemetry.io/otel/metric v1.21.0
+	go.opentelemetry.io/otel/trace v1.21.0
+	go.uber.org/multierr v1.11.0
+	go.uber.org/zap v1.26.0
+	golang.org/x/text v0.14.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.91.0 // indirect
+	go.opentelemetry.io/collector/consumer v0.91.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.0.0 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
+	google.golang.org/grpc v1.60.1 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+retract (
+	v0.76.2
+	v0.76.1
+	v0.65.0
+)
+
+replace go.opentelemetry.io/collector/featuregate => ../../featuregate
+
+replace go.opentelemetry.io/collector/component => ../../component
+
+replace go.opentelemetry.io/collector => ../..
+
+replace go.opentelemetry.io/collector/config/configretry => ../../config/configretry
+
+replace go.opentelemetry.io/collector/processor => ../../processor
+
+replace go.opentelemetry.io/collector/config/configtelemetry => ../../config/configtelemetry
+
+replace go.opentelemetry.io/collector/confmap => ../../confmap
+
+replace go.opentelemetry.io/collector/consumer => ../../consumer
+
+replace go.opentelemetry.io/collector/pdata => ../../pdata
+
+replace go.opentelemetry.io/collector/extension => ../../extension
+
+replace go.opentelemetry.io/collector/exporter => ../../exporter
+
+replace go.opentelemetry.io/collector/semconv => ../../semconv
+
+replace go.opentelemetry.io/collector/receiver => ../../receiver