08__linguist_and_cloc__02__extract.sh

#!/usr/bin/env bash
# Copyright 2019-2024 VMware, Inc.
# SPDX-License-Identifier: Apache-2.0

##############################################################################################################
# Extract key results from the files generated by ...
#   "GitHub Linguist" - https://github.com/github/linguist
#   "Count Lines of Code" (CLOC) - https://github.com/AlDanial/cloc
##############################################################################################################

# ----- Please adjust

# ------ Do not modify
VERSION="${LINGUIST_VERSION}"
STEP=$(get_step)

BASE_DIR="${REPORTS_DIR}/${STEP}__LINGUIST"
export LOG_FILE="${BASE_DIR}.log"
INPUT_CLOC_FILE="${BASE_DIR}/_CLOC_results_extracted.csv"
INPUT_LINGUSIT_FILE="${BASE_DIR}/_LINGUIST_results_extracted.csv"
OUTPUT_CLOC_FILE="${BASE_DIR}/_CLOC_results_generated.txt"
OUTPUT_LINGUIST_FILE="${BASE_DIR}/_LINGUIST_results_generated.txt"

function tokenize() {
	for WORD in $1; do echo "${WORD}"; done
}

# This method consolidates the CLOC results and prepares them for their visualisation.
function generate_csv_data__CLOC() {
	INPUT_FILE="${1}"
	OUTPUT_FILE="${2}"
	log_console_info "Generate CLOC CSV file"
	INDEX_FIRST_TECH=3

	ALL_MIXED_LANGS=$(cut -d';' -f${INDEX_FIRST_TECH}- <"${INPUT_FILE}" | tr ' ' '_' | sed 's/\([^\;]*\;[^\;]*\);/\1\ /g' | tr -d '\n')
	ALL_LANGS=$(tokenize "${ALL_MIXED_LANGS}" | cut -d';' -f1 | sort | uniq)
	HEADER="App"
	# /!\ DO NOT DOUBLE QUOTE ${ALL_LANGS}
	for ALL_LANG in ${ALL_LANGS}; do
		HEADER+=",${ALL_LANG}"
	done
	echo "${HEADER}" >"${OUTPUT_FILE}"

	while read -r LINE; do
		VALUES=()
		#GROUP=$(echo $LINE | cut -d';' -f1)
		APP=$(echo "${LINE}" | cut -d';' -f2)
		LANGS=$(echo "${LINE}" | cut -d';' -f${INDEX_FIRST_TECH}- | sed 's/\([^\;]*\;[^\;]*\);/\1\ /g')

		# /!\ DO NOT DOUBLE QUOTE ${ALL_LANGS}
		for ALL_LANG in ${ALL_LANGS}; do
			if echo "${LINE}" | grep -q -v "${ALL_LANG};"; then
				VALUES+=("0")
				continue
			else
				TOTAL=$(echo " $LANGS" | grep -o " ${ALL_LANG};[0-9.]*" | cut -d';' -f2 | tr -d '\n')
				VALUES+=("${TOTAL}")
			fi
		done
		ENTRY="${APP}"
		for VALUE in "${VALUES[@]}"; do
			ENTRY="${ENTRY}","${VALUE}"
		done
		echo "${ENTRY}" >>"${OUTPUT_FILE}"
	done < <(sort -r <"${INPUT_FILE}" | awk 'NF' | uniq)
}

function generate_csv_data__LINGUIST() {
	INPUT_FILE="${1}"
	OUTPUT_FILE="${2}"
	log_console_info "Generate Linguist CSV file"

	ALL_MIXED_LANGS=$(cut -d';' -f7- <"${INPUT_FILE}" | tr ' ' '_' | sed 's/\([^\;]*\;[^\;]*\);/\1\ /g' | tr -d '\n')
	ALL_LANGS=$(tokenize "${ALL_MIXED_LANGS}" | cut -d';' -f1 | sort | uniq)
	HEADER="App"
	# /!\ DO NOT DOUBLE QUOTE ${ALL_LANGS}
	for ALL_LANG in ${ALL_LANGS}; do
		HEADER+=",${ALL_LANG}"
	done
	echo "${HEADER},Other" >"${OUTPUT_FILE}"

	while read -r LINE; do
		#GROUP=$(echo $LINE | cut -d';' -f1)
		#STATS=$(echo "${LINE}" | cut -d';' -f3-5)
		APP=$(echo "${LINE}" | cut -d';' -f2)
		VALUES=()
		LOCS=$(echo "${LINE}" | cut -d';' -f6)
		LANGS=$(echo "${LINE}" | cut -d';' -f7- | sed 's/\([^\;]*\;[^\;]*\);/\1\ /g')

		# /!\ DO NOT DOUBLE QUOTE ${ALL_LANGS}
		NO_MATCH="true"
		for ALL_LANG in ${ALL_LANGS}; do
			if echo "${LINE}" | grep -q -v "${ALL_LANG};"; then
				VALUES+=("0")
				continue
			else
				PERCENT=$(echo " $LANGS" | grep -o " ${ALL_LANG};[0-9.]*%" | cut -d';' -f2 | tr -d '%')
				TOTAL=$(awk -v locs="${LOCS}" -v pct="${PERCENT}" 'BEGIN{total=(locs*pct/100); printf "%.0f\n", total}')
				VALUES+=("${TOTAL}")
				NO_MATCH="false"
			fi
		done

		# Adding entry for "Other" category
		if [[ "${NO_MATCH}" == "true" ]]; then
			VALUES+=("${LOCS}")
		else
			VALUES+=("0")
		fi

		ENTRY="${APP}"
		for VALUE in "${VALUES[@]}"; do
			ENTRY="${ENTRY}","${VALUE}"
		done
		echo "${ENTRY}" >>"${OUTPUT_FILE}"
	done < <(sort -r <"${INPUT_FILE}" | awk 'NF' | uniq)
}

function main() {
	if [[ -f "${INPUT_CLOC_FILE}" ]]; then
		generate_csv_data__CLOC "${INPUT_CLOC_FILE}" "${OUTPUT_CLOC_FILE}"
	else
		log_console_error "CLOC result file missing: ${INPUT_CLOC_FILE}"
	fi

	if [[ -f "${INPUT_LINGUSIT_FILE}" ]]; then
		generate_csv_data__LINGUIST "${INPUT_LINGUSIT_FILE}" "${OUTPUT_LINGUIST_FILE}"
	else
		log_console_error "Linguist result file missing: ${INPUT_LINGUSIT_FILE}"
	fi
}

main