Automate creation of standards overview pages (#116)

* Automate creation of standards overview pages

resolves #97

Signed-off-by: Matthias Büchse <matthias.buechse@cloudandheat.com>
Signed-off-by: Max Wolfs <mail@maxwolfs.com>
Co-authored-by: Max Wolfs <mail@maxwolfs.com>

.gitignore

@@ -14,6 +14,8 @@
 /docs/04-operating-scs/components
 /docs/06-releases
 /standards/*.md
+/standards/*/*.md
+/standards/scs-*.yaml
 
 # Dependencies
 node_modules
@@ -24,6 +26,8 @@ node_modules
 # Generated files
 .docusaurus
 .cache-loader
+sidebarsStandardsItems.js
+sidebarsCertificationItems.js
 
 # Misc
 .DS_Store

docs.package.json

@@ -25,7 +25,10 @@
   },
   {
     "repo": "SovereignCloudStack/standards",
-    "source": "Standards/*.md",
+    "source": [
+      "Standards/*.md",
+      "Tests/scs-*.yaml"
+    ],
     "target": "standards",
     "label": ""
   },

getDocs.js

@@ -31,8 +31,15 @@ repos.forEach((repo) => {
   fs.mkdirSync(subDirPath, { recursive: true })
 
   // Copy docs content from A to B
-  const copyDocsCommand = `cp -r ${repoDir}/${repo.source} ${subDirPath}`
-  execSync(copyDocsCommand)
+  // allow multiple sources here so the same repo need not be checked out multiple times
+  // however, it would be better if this script automatically grouped all entries by repo and then only
+  // checked out each repo only once; I leave this as a TODO because I don't fully grasp the meaning of
+  // label, for instance, and the label is used for the temporary repo directory
+  let sources = Array.isArray(repo.source) ? repo.source : [repo.source]
+  sources.forEach((source) => {
+    const copyDocsCommand = `cp -r ${repoDir}/${source} ${subDirPath}`
+    execSync(copyDocsCommand)
+  })
 
   // Remove the cloned repository
   const removeRepoCommand = 'rm -rf repo_to_be_edited'

package.json

@@ -29,7 +29,7 @@
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "typecheck": "tsc",
-    "postinstall": "node getDocs.js",
+    "postinstall": "node getDocs.js && node populateStds.js && node populateCerts.js",
     "test": "echo \"Error: no test specified\" && exit 1",
     "lint:md": "markdownlint-cli2 \"**/*.md\"",
     "fix:md": "markdownlint-cli2-fix \"**/*.md\"",
@@ -57,7 +57,8 @@
     "prettier": "^2.8.4",
     "prism-react-renderer": "^1.3.5",
     "react": "^17.0.2",
-    "react-dom": "^17.0.2"
+    "react-dom": "^17.0.2",
+    "yaml": "^2.3.4"
   },
   "devDependencies": {
     "@docusaurus/eslint-plugin": "^2.4.3",

populateCerts.js

@@ -0,0 +1,119 @@
+const fs = require('fs')
+const YAML = require('yaml')
+
+// how many outdated versions of any scope to include
+const MAX_OLD = 1
+
+const filenames = fs
+    .readdirSync('standards/')
+    .filter((fn) => fn.startsWith('scs-') && fn.endsWith('.yaml'))
+
+const scopes = filenames.map((filename) => {
+    return {
+        ...YAML.parseDocument(fs.readFileSync(`standards/${filename}`, 'utf8')).toJSON(),
+        filename,
+        id: filename.substring(0, filename.length - 5),
+    }
+})
+
+const today = new Date().toISOString().slice(0, 10)
+
+const sidebarItems = scopes.map((scope) => {
+    const matrix = {}
+    const versionsShown = {}
+    var numOld = 0
+    // sort in descending order, so we get the MAX_OLD most recent obsolete versions
+    scope.versions.sort((a, b) => b.version.localeCompare(a.version));
+    scope.versions.forEach((version) => {
+        version.isStable = version.stabilized_at !== undefined && version.stabilized_at <= today
+        version.isObsolete = version.obsoleted_at !== undefined && version.obsoleted_at < today
+        version.isEffective = version.isStable && !version.isObsolete
+        version.isPreview = version.stabilized_at === undefined || today < version.stabilized_at
+        if (!version.isEffective && !version.isPreview) {
+            numOld += 1
+            if (numOld > MAX_OLD) return
+        }
+        version.state = (
+            version.stabilized_at === undefined ? 'Draft' :
+            version.isEffective ? 'Effective' :
+            version.isObsolete ? 'Deprecated' :
+            'Stable'
+        )
+        if (version.standards === undefined) return
+        versionsShown[version.version] = version
+        version.standards.forEach((standard) => {
+            const components = standard.url.split('/')
+            const filename = components[components.length - 1]
+            // first, sensible (but not pretty) defaults
+            var key = standard.url
+            var name = standard.name
+            var ver = '✓'
+            var url = standard.url
+            if (filename.startsWith('scs-') && filename.endsWith('.md')) {
+                // special case for internal standards
+                const components2 = filename.split('-')
+                key = `scs-${components2[1]}`
+                name = `${key}: ${name}`
+                ver = components2[2]
+                url = `/standards/${filename.substring(0, filename.length - 3)}`
+            } else {
+                // special case mainly for OpenStack Powered Compute, but anything ending in 'vXYZ'
+                const components2 = name.split(' ')
+                const v = components2.splice(components2.length - 1)
+                if (v[0].startsWith('v')) {
+                    key = components2.join(' ')
+                    name = key
+                    ver = v[0]
+                }
+            }
+            if (matrix[key] === undefined) {
+                matrix[key] = {name, columns: {}}
+            }
+            matrix[key].columns[version.version] = {
+                version: ver,
+                url,
+            }
+        })
+    })
+
+    const rows = Object.values(matrix)
+    const columns = Object.keys(versionsShown)
+    rows.sort((a, b) => a.name.localeCompare(b.name));
+    columns.sort((a, b) => a.localeCompare(b));
+
+    lines = [`# ${scope.name}
+
+Note that the state _Stable_ is shown here if _stabilized at_ is in the future, whereas _Effective_ is shown here if _stabilized at_ is in the past and _deprecated at_ is unset or in the future.
+`]
+    lines.push('| Scope versions ->  | ' + columns.join('  | ') + '  |')
+    lines.push('| :-- | ' + columns.map(() => ':--').join(' | ') + ' |')
+    lines.push('| State              | ' + columns.map((c) => versionsShown[c].state).join('  | ') + '  |')
+    lines.push('| Stabilized at      | ' + columns.map((c) => versionsShown[c].stabilized_at || '').join('  | ') + '  |')
+    lines.push('| Obsoleted at       | ' + columns.map((c) => versionsShown[c].obsoleted_at || '').join('  | ') + '  |')
+    // md doesn't allow intermediate header rows
+    // lines.push('| :-- | ' + columns.map(() => ':--').join(' | ') + ' |')
+    lines.push('| **Standards**      | ' + columns.map((c) => ' '.repeat(c.length)).join('  | ') + '  |')
+    // md doesn't allow intermediate header rows
+    // lines.push('| :-- | ' + columns.map(() => ':--').join(' | ') + ' |')
+    rows.forEach((row) => {
+        lines.push(`| ${row.name}  | ` + columns.map((c) => row.columns[c]).map((col) => {
+            if (col === undefined) {
+                // this version of the cert does not include this standard
+                return ''
+            }
+            return `[${col.version}](${col.url})`
+        }).join('  | ') + '  |')
+    })
+    lines.push('')  // file should end with a single newline character
+    fs.writeFileSync(`standards/${scope.id}.md`, lines.join('\n'), 'utf8')
+
+    const state = columns.filter((c) => versionsShown[c].isEffective).length ? '📜' : '✏️'
+    return {
+        type: 'doc',
+        label: scope.name,
+        id: scope.id,
+    }
+})
+
+var newSidebars = `module.exports = ${JSON.stringify(sidebarItems, null, '  ')}`
+fs.writeFileSync('./sidebarsCertificationItems.js', newSidebars, 'utf8')

populateStds.js

@@ -0,0 +1,155 @@
+const fs = require('fs')
+const YAML = require('yaml')
+
+const intro = `# Overview
+
+Standards are the core deliverable of SCS. By standardizing the open source software components of a cloud computing stack, their versions, how they are to be configured, deployed and utilized, SCS guarantees the reproducibility of a certain behavior of this technology.
+
+SCS standards are discussed, developed and maintained in the community by the corresponding teams (see Track in the table below), which naturally include existing users of SCS.`
+const trackIntros = {
+    'Global': 'This track encompasses the foundational standards that guide the overall structure, documentation, and general topics related to the Sovereign Cloud Stack. It serves as the core framework, ensuring consistency, clarity, and comprehensibility across all aspects of the cloud stack, fostering an environment where information is easily accessible and understood.',
+    'IaaS': 'The IaaS Layer Standards track focuses on the protocols, guidelines, and specifications that govern the infrastructure as a service layer. This encompasses standards for virtual machines, storage, networking, and other foundational resources, ensuring seamless, efficient, and secure operation, interoperability, and management of the underlying cloud infrastructure.',
+    'KaaS': 'Standards in this track are concerned with Kubernetes as a Service layer, outlining norms and best practices for deploying, managing, and operating Kubernetes clusters. These standards aim to ensure that the orchestration of containers is streamlined, secure, and compatible across various cloud environments and platforms.',
+    'IAM': 'This track revolves around Identity and Access Management (IAM) standards, providing guidelines for ensuring secure and efficient user authentication, authorization, and administration. It addresses issues related to user identity, permissions, roles, and policies, aiming to safeguard and streamline access to cloud resources and services.',
+    'Ops': 'Operational Tooling Standards cover the protocols and guidelines associated with tools and utilities used for monitoring, management, and maintenance of the cloud environment. This includes standards for status pages, alerts, logs, and other operational tools, aiming to optimize the reliability, performance, and security of cloud services and resources.',
+}
+const headerLegend = '*Legend to the column headings: Draft, Stable (but not effective), Effective, Deprecated (and no longer effective).'
+
+var filenames = fs
+    .readdirSync('standards/')
+    .filter((fn) => fn.startsWith('scs-') && fn.endsWith('.md') && !fn.startsWith('scs-X'))
+
+keys = ['title', 'type', 'status', 'track', 'stabilized_at', 'obsoleted_at', 'replaces', 'authors', 'state']
+
+// use the ISO string, because so do the standard documents, and we can use string comparison with ISO dates
+today = new Date().toISOString().slice(0, 10)
+
+// collect all the information sorted into a track/adr-id/version hierarchy
+tracks = {}
+filenames.forEach((filename) => {
+    var components = filename.split('-')
+    var obj = {
+        ...YAML.parseDocument(fs.readFileSync(`standards/${filename}`, 'utf8')).toJSON(),
+        filename,
+        id: filename.substring(0, filename.length - 3),
+        adrId: components[1],
+        version: components[2],
+    }
+    obj.isStable = obj.stabilized_at !== undefined && obj.stabilized_at <= today
+    obj.isObsolete = obj.obsoleted_at !== undefined && obj.obsoleted_at <= today
+    obj.isEffective = obj.isStable && !obj.isObsolete
+    var track = obj.track
+    if (track === undefined) return
+    if (tracks[track] === undefined) tracks[track] = {}
+    var standards = tracks[track]
+    if (standards[obj.adrId] === undefined) standards[obj.adrId] = {versions: []}
+    standards[obj.adrId].versions.push(obj)
+})
+
+function readPrefixLines(fn) {
+    var lines = []
+    if (fs.existsSync(fn)) {
+        lines = fs.readFileSync(fn, 'utf8').split('\n')
+        var tableIdx = lines.findIndex((line) => line.trim().startsWith('|'))
+        if (tableIdx >= 0) {
+            lines.splice(tableIdx)
+        }
+    } else console.log(`WARNING: file ${fn} not found`)
+    return lines
+}
+
+function mkLinkList(versions) {
+    var links = versions.map((v) => `[${v.version}](/standards/${v.id})`)
+    return links.join(', ')
+}
+
+// walk down the hierarchy, building adr overview pages, track overview pages, and total overview page
+// as well as the new sidebar
+sidebarItems = []
+var lines = readPrefixLines('standards/standards/overview.md')
+if (!lines.length) lines.push(`${intro}
+
+${headerLegend}
+`)
+lines.push('| Standard  | Track  | Description  | Draft | Stable* | Effective | Deprecated* |')
+lines.push('| --------- | ------ | ------------ | ----- | ------- | --------- | ----------- |')
+Object.entries(tracks).forEach((trackEntry) => {
+    var track = trackEntry[0]
+    var trackPath = `standards/${track.toLowerCase()}`
+    fs.mkdirSync(trackPath, { recursive: true })
+    var trackItem = {
+        type: 'category',
+        label: track,
+        link: {
+            type: 'doc',
+            id: `${track.toLowerCase()}/index`,
+        },
+        items: [],
+    }
+    sidebarItems.push(trackItem)
+    var tlines = readPrefixLines(`standards/${track.toLowerCase()}/index.md`)
+    if (!tlines.length) {
+        tlines.push(`# ${track} Standards
+
+${trackIntros[track]}
+
+${headerLegend}
+`)
+    }
+    tlines.push('| Standard  | Description  | Draft | Stable* | Effective | Deprecated* |')
+    tlines.push('| --------- | ------------ | ----- | ------- | --------- | ----------- |')
+    Object.entries(trackEntry[1]).forEach((standardEntry) => {
+        var versions = standardEntry[1].versions
+        // unfortunately, some standards are obsolete without being stable
+        var draftVersions = versions.filter((v) => v.stabilized_at === undefined && v.obsoleted_at === undefined)
+        var stableVersions = versions.filter((v) => v.stabilized_at !== undefined && !v.isEffective)
+        var effectiveVersions = versions.filter((v) => v.isEffective)
+        var deprecatedVersions = versions.filter((v) => v.isObsolete)
+        var ref = versions[versions.length - 1]
+        if (effectiveVersions.length) {
+            ref = effectiveVersions[effectiveVersions.length - 1]
+        }
+        var adrId = standardEntry[0]
+        var standardItem = {
+            type: 'category',
+            label: `scs-${adrId}`,
+            link: {
+                type: 'doc',
+                id: `${track.toLowerCase()}/scs-${adrId}`,
+            },
+            items: [],
+        }
+        trackItem.items.push(standardItem)
+        var slines = readPrefixLines(`standards/${track.toLowerCase()}/scs-${adrId}.md`)
+        if (!slines.length) {
+            slines.push(`# scs-${adrId}: ${ref.title}\n`)
+            if (ref.description !== undefined) {
+                slines.push(ref.description)
+            }
+        }
+        slines.push('| Version  | Type  | State   | stabilized | obsoleted |')
+        slines.push('| -------- | ----- | ------- | ---------- | --------- |')
+        var link = `[scs-${adrId}](/standards/${track.toLowerCase()}/scs-${adrId})`
+        var versionList = `${mkLinkList(draftVersions) || '-'} | ${mkLinkList(stableVersions) || '-'} | ${mkLinkList(effectiveVersions) || '-'} | ${mkLinkList(deprecatedVersions) || '-'}`
+        lines.push(`| ${link}  | ${track}  | ${ref.title}  | ${versionList}  |`)
+        tlines.push(`| ${link}  | ${ref.title}  | ${versionList}  |`)
+        standardEntry[1].versions.forEach((obj) => {
+            var versionItem = {
+                type: 'doc',
+                label: obj.version.toUpperCase(),
+                id: obj.id,
+            }
+            standardItem.items.push(versionItem)
+            slines.push(`| [scs-${adrId}-${obj.version}](/standards/${obj.id})  | ${obj.type}  | ${obj.status || obj.state}  | ${obj.stabilized_at || '-'}  | ${obj.obsoleted_at || '-'}  |`)
+        })
+        slines.push('')  // file should end with a single newline character
+        fs.writeFileSync(`${trackPath}/scs-${adrId}.md`, slines.join('\n'), 'utf8')
+    })
+    tlines.push('')  // file should end with a single newline character
+    fs.writeFileSync(`${trackPath}/index.md`, tlines.join('\n'), 'utf8')
+})
+lines.push('')  // file should end with a single newline character
+fs.writeFileSync(`standards/standards/overview.md`, lines.join('\n'), 'utf8')
+
+var newSidebars = `module.exports = ${JSON.stringify(sidebarItems, null, '  ')}`
+fs.writeFileSync('./sidebarsStandardsItems.js', newSidebars, 'utf8')