From d0dfea6c7f3fd02b46431a898019560ed2c3c31d Mon Sep 17 00:00:00 2001 From: Patrik Stas Date: Tue, 6 Sep 2022 23:04:46 +0200 Subject: [PATCH 1/3] List transactions based on 'serialized' format Signed-off-by: Patrik Stas --- .../BadgedValueDisplay/BadgedValueDisplay.js | 2 +- .../components/TxDisplay/TxDisplay.js | 11 ++- .../components/TxListCompact/TxListCompact.js | 1 - .../components/TxListItem/TxListItem.js | 11 ++- .../components/TxPreview/TxPreview.js | 40 ++------- .../components/TxPreviewList/TxPreviewList.js | 26 +++--- indyscan-webapp/context/socket-client.js | 12 ++- indyscan-webapp/nodemon.json | 3 +- indyscan-webapp/package.json | 2 +- indyscan-webapp/pages/home.js | 87 ++++++++++--------- indyscan-webapp/pages/tx.js | 84 +++++++++--------- indyscan-webapp/pages/txs.js | 2 +- indyscan-webapp/server/config.js | 2 +- indyscan-webapp/server/index.js | 16 ++-- indyscan-webapp/txtools/index.js | 29 +++++-- 15 files changed, 177 insertions(+), 151 deletions(-) diff --git a/indyscan-webapp/components/BadgedValueDisplay/BadgedValueDisplay.js b/indyscan-webapp/components/BadgedValueDisplay/BadgedValueDisplay.js index cb69e2c3..76a8f3d7 100644 --- a/indyscan-webapp/components/BadgedValueDisplay/BadgedValueDisplay.js +++ b/indyscan-webapp/components/BadgedValueDisplay/BadgedValueDisplay.js @@ -17,7 +17,7 @@ export function BadgedValueDisplay (props) { < span className="tooltiptext">Copied } { navigator.clipboard.writeText(value) diff --git a/indyscan-webapp/components/TxDisplay/TxDisplay.js b/indyscan-webapp/components/TxDisplay/TxDisplay.js index 54d1d658..fe2e7794 100644 --- a/indyscan-webapp/components/TxDisplay/TxDisplay.js +++ b/indyscan-webapp/components/TxDisplay/TxDisplay.js @@ -1,8 +1,7 @@ import React from 'react' import './TxDisplay.scss' import { - Button, - GridRow, Icon, + GridRow, Item, ItemContent, ItemDescription, @@ -17,8 +16,12 @@ import { BadgedValueDisplay } from '../BadgedValueDisplay/BadgedValueDisplay' -const TxDisplay = ({ txIndyscan, txLedger }) => { - const keyValTxDetailsHumanReadable = extractTxDataDetailsHumanReadable(txIndyscan) +const TxDisplay = ({ txIndyscan }) => { + const txExpansion = txIndyscan?.idata?.expansion + if (!txExpansion) { + return
+ } + const keyValTxDetailsHumanReadable = extractTxDataDetailsHumanReadable(txExpansion) const keyValTxBasic = extractTxDataBasic(txIndyscan) const { txnTimeIso8601, typeName } = keyValTxBasic // eslint-disable-line const keyValBasicHumanReadable = converTxDataBasicToHumanReadable(keyValTxBasic) diff --git a/indyscan-webapp/components/TxListCompact/TxListCompact.js b/indyscan-webapp/components/TxListCompact/TxListCompact.js index c34d0b65..2b5f2c23 100644 --- a/indyscan-webapp/components/TxListCompact/TxListCompact.js +++ b/indyscan-webapp/components/TxListCompact/TxListCompact.js @@ -25,7 +25,6 @@ class TxListCompact extends Component { network={this.props.network} ledger={this.props.ledger} txn={txn} - description='Todo' /> ) })} diff --git a/indyscan-webapp/components/TxListItem/TxListItem.js b/indyscan-webapp/components/TxListItem/TxListItem.js index af2f5ce1..30c0f8c5 100644 --- a/indyscan-webapp/components/TxListItem/TxListItem.js +++ b/indyscan-webapp/components/TxListItem/TxListItem.js @@ -27,9 +27,14 @@ class TxListItem extends Component { render () { const { baseUrl, description, ledger, network, txn } = this.props const { seqNo, txnTimeIso8601, typeName, from } = extractTxDataBasic(txn) - const data = extractTxDataDetailsHumanReadable(txn, 5) const { href, as } = getTxLinkData(baseUrl, network, ledger, seqNo) - const filteredData = filterTxDetails(data) + let filteredData + try { + const data = extractTxDataDetailsHumanReadable(txn, 5) + filteredData = filterTxDetails(data) + } catch (err) { + + } return ( {seqNo} @@ -45,7 +50,7 @@ class TxListItem extends Component { {from} - {renderKeyValuesAsBadges(seqNo, filteredData, palette[2])} + {filteredData && renderKeyValuesAsBadges(seqNo, filteredData, palette[2])} ) } diff --git a/indyscan-webapp/components/TxPreview/TxPreview.js b/indyscan-webapp/components/TxPreview/TxPreview.js index f20f68fb..364a7c14 100644 --- a/indyscan-webapp/components/TxPreview/TxPreview.js +++ b/indyscan-webapp/components/TxPreview/TxPreview.js @@ -15,44 +15,22 @@ class TxPreview extends Component { this.state = {} } - calculateTimeSinceLastTransaction = function calculateTimeSinceLastTransaction (expansionTx) { - const txnTime = (expansionTx.idata && expansionTx.idata.txnMetadata) ? - (Date.parse(expansionTx.idata.txnMetadata.txnTime)) - : undefined - const utimeNow = Math.floor(new Date()) - return secondsToDhms((utimeNow - txnTime) / 1000) - } - - refreshTimesSinceLast () { - let sinceSinceTx = this.calculateTimeSinceLastTransaction(this.props.indyscanTx) - sinceSinceTx = (sinceSinceTx) ? sinceSinceTx : 'Unknown' - this.setState({ sinceSinceTx }) - } - - componentDidMount () { - this.refreshTimesSinceLast() - this.interval = setInterval(this.refreshTimesSinceLast.bind(this), 1000) - } - - componentWillUnmount () { - clearInterval(this.interval) - } - render () { - const { baseUrl, network, ledger, indyscanTx } = this.props - const { seqNo, txnTimeIso8601, typeName, from } = extractTxDataBasic(indyscanTx) + const { baseUrl, network, ledger, seqNo, txnTimeIso8601, typeName, from } = this.props const href = `${baseUrl}/tx?network=${network}&ledger=${ledger}&seqNo=${seqNo}` const as = `/tx/${network}/${ledger}/${seqNo}` const fromDidDisplayed = from ? (from.length < MAX_DID_LENTH) ? from : `${from.substring(0, (MAX_DID_LENTH - 3))}...` : 'N/A' - // const utcTime = moment.utc(txnTimeIso8601).format('do MMMM YYYY, H:mm:ss') let txnDateLocalString = "N/A" - let txnDate - if (indyscanTx.idata && indyscanTx.idata.txnMetadata && indyscanTx.idata.txnMetadata.txnTime) { - txnDate = new Date(indyscanTx.idata.txnMetadata.txnTime) - txnDateLocalString = moment.utc(txnTimeIso8601).tz(moment.tz.guess()).format('DD MMMM YYYY, HH:mm:ss a') + // let utcDateString = "N/A" + const sinceEpoch = txnTimeIso8601 ? (Date.parse(txnTimeIso8601)) : null + if (txnTimeIso8601) { + txnDateLocalString = moment.utc(txnTimeIso8601).tz(moment.tz.guess()).format('do MMMM YYYY, HH:mm:ss a') + // todo: display this + // utcDateString = moment.utc(txnTimeIso8601).format('do MMMM YYYY, HH:mm:ss a') } + console.log(`txnTimeIso8601 = ${txnTimeIso8601}, sinceEpoch=${sinceEpoch}`) return ( @@ -66,7 +44,7 @@ class TxPreview extends Component {
From DID: {fromDidDisplayed} Local Time: {txnDateLocalString} - +
diff --git a/indyscan-webapp/components/TxPreviewList/TxPreviewList.js b/indyscan-webapp/components/TxPreviewList/TxPreviewList.js index af862c17..b8bfd33b 100644 --- a/indyscan-webapp/components/TxPreviewList/TxPreviewList.js +++ b/indyscan-webapp/components/TxPreviewList/TxPreviewList.js @@ -19,23 +19,29 @@ class TxPreviewList extends Component { render () { const { indyscanTxs, network, subledger, animateFirst } = this.props - const firstTx = indyscanTxs[0] + const { seqNo, txnTimeIso8601, typeName, from } = extractTxDataBasic(indyscanTxs[0]) return ( - {firstTx && - - - + {seqNo && + + + } { indyscanTxs.slice(1).map((indyscanTx, index) => { - const { seqNo } = extractTxDataBasic(indyscanTx) + const { seqNo, txnTimeIso8601, typeName, from } = extractTxDataBasic(indyscanTx) return ( ) diff --git a/indyscan-webapp/context/socket-client.js b/indyscan-webapp/context/socket-client.js index f330a38b..d4a4ae03 100644 --- a/indyscan-webapp/context/socket-client.js +++ b/indyscan-webapp/context/socket-client.js @@ -3,7 +3,7 @@ import io from 'socket.io-client' let socketClient -function getWebsocketClient () { +function assureWebsocketClient () { if (socketClient) { return socketClient } @@ -11,4 +11,12 @@ function getWebsocketClient () { return socketClient } -export default getWebsocketClient + +function getWebsocketClient () { + return socketClient +} + +export { + assureWebsocketClient, + getWebsocketClient +} diff --git a/indyscan-webapp/nodemon.json b/indyscan-webapp/nodemon.json index 2b2fe497..c5e3311e 100644 --- a/indyscan-webapp/nodemon.json +++ b/indyscan-webapp/nodemon.json @@ -4,8 +4,7 @@ "watch": ["server/**/*", "index.js"], "ext": "js json", "env": { - "INDYSCAN_API_URL": "http://localhost:3708", - "DAEMON_WS_URL": "http://localhost:3709", + "INDYSCAN_API_URL": "https://indyscan.io", "PORT" : 3707, "LOG_LEVEL": "debug", "LOG_HTTP_REQUESTS" : true, diff --git a/indyscan-webapp/package.json b/indyscan-webapp/package.json index 57114414..9c0de750 100644 --- a/indyscan-webapp/package.json +++ b/indyscan-webapp/package.json @@ -6,7 +6,7 @@ "scripts": { "lint": "standard", "lint:fix": "standard --fix", - "dev": "nodemon server/index.js", + "dev": "NODE_OPTIONS='--inspect' nodemon server/index.js", "test:unit": "jest tests/unit", "build": "next build", "start": "next start", diff --git a/indyscan-webapp/pages/home.js b/indyscan-webapp/pages/home.js index 1fa183b1..e150f774 100644 --- a/indyscan-webapp/pages/home.js +++ b/indyscan-webapp/pages/home.js @@ -9,8 +9,7 @@ import Footer from '../components/Footer/Footer' import fetch from 'isomorphic-fetch' import _ from 'lodash' import { CSSTransition } from 'react-transition-group' -import util from 'util' -import getWebsocketClient from '../context/socket-client' +import { assureWebsocketClient, getWebsocketClient } from '../context/socket-client' import NetworkInfo from '../components/NetworkInfo/NetworkInfo' import SubledgerHeader from '../components/SubledgerHeader/SubledgerHeader' @@ -18,18 +17,20 @@ class HomePage extends Component { static async getInitialProps ({ req, query }) { const baseUrl = getBaseUrl(req) const { network } = query - const networkDetails = await getNetwork(baseUrl, network) - const domainExpansionTxs = await getTxs(baseUrl, network, 'domain', 0, 13, [], 'expansion') - const poolExpansionTxs = await getTxs(baseUrl, network, 'pool', 0, 13, [], 'expansion') - const configExpansionTxs = await getTxs(baseUrl, network, 'config', 0, 13, [], 'expansion') + const features = await fetch(`${baseUrl}/features`) const versionRes = await fetch(`${baseUrl}/version`) const version = (await versionRes.json()).version + const networkDetails = await getNetwork(baseUrl, network) + const domainTxs = await getTxs(baseUrl, network, 'domain', 0, 13, [], 'full') + const poolTxs = await getTxs(baseUrl, network, 'pool', 0, 13, [], 'full') + const configTxs = await getTxs(baseUrl, network, 'config', 0, 13, [], 'full') return { + features, networkDetails, network, - domainExpansionTxs, - poolExpansionTxs, - configExpansionTxs, + domainTxs, + poolTxs, + configTxs, baseUrl, version } @@ -38,15 +39,15 @@ class HomePage extends Component { constructor (props) { super() this.state = { - domainExpansionTxs: props.domainExpansionTxs, - poolExpansionTxs: props.poolExpansionTxs, - configExpansionTxs: props.configExpansionTxs + domainTxs: props.domainTxs, + poolTxs: props.poolTxs, + configTxs: props.configTxs } } addNewDomainTx (txData) { - let domainExpansionTxs = _.cloneDeep(this.state.domainExpansionTxs) - if (domainExpansionTxs[0].imeta.seqNo === txData.imeta.seqNo) { + let domainTxs = _.cloneDeep(this.state.domainTxs) + if (domainTxs[0].imeta.seqNo === txData.imeta.seqNo) { // When scanner runs too fast (it might happen that one transaction in daemon is processed twice, causing // duplicate notification about the same transaction from UI perspective. If we'd add this transaction, // we bump into problem with animations, because 2 transactions in list would have generated the same @@ -54,35 +55,35 @@ class HomePage extends Component { // This early return is preventing this from happening return } - domainExpansionTxs.unshift(txData) - if (domainExpansionTxs.length > 10) { - domainExpansionTxs.pop() + domainTxs.unshift(txData) + if (domainTxs.length > 10) { + domainTxs.pop() } - this.setState({ domainExpansionTxs }) + this.setState({ domainTxs }) } addNewConfigTx (txData) { - let configExpansionTxs = _.cloneDeep(this.state.configExpansionTxs) - if (configExpansionTxs[0].imeta.seqNo === txData.imeta.seqNo) { + let configTxs = _.cloneDeep(this.state.configTxs) + if (configTxs[0].imeta.seqNo === txData.imeta.seqNo) { return } - configExpansionTxs.unshift(txData) - if (configExpansionTxs.length > 10) { - configExpansionTxs.pop() + configTxs.unshift(txData) + if (configTxs.length > 10) { + configTxs.pop() } - this.setState({ configExpansionTxs }) + this.setState({ configTxs }) } addNewPoolTx (txData) { - let poolExpansionTxs = _.cloneDeep(this.state.poolExpansionTxs) - if (poolExpansionTxs[0].imeta.seqNo === txData.imeta.seqNo) { + let poolTxs = _.cloneDeep(this.state.poolTxs) + if (poolTxs[0].imeta.seqNo === txData.imeta.seqNo) { return } - poolExpansionTxs.unshift(txData) - if (poolExpansionTxs.length > 10) { - poolExpansionTxs.pop() + poolTxs.unshift(txData) + if (poolTxs.length > 10) { + poolTxs.pop() } - this.setState({ poolExpansionTxs }) + this.setState({ poolTxs }) } onTxProcessed (payload) { @@ -134,9 +135,9 @@ class HomePage extends Component { componentWillReceiveProps (newProps) { console.log(`componentWillReceiveProps`) - this.setState({ domainExpansionTxs: newProps.domainExpansionTxs }) - this.setState({ poolExpansionTxs: newProps.poolExpansionTxs }) - this.setState({ configExpansionTxs: newProps.configExpansionTxs }) + this.setState({ domainTxs: newProps.domainTxs }) + this.setState({ poolTxs: newProps.poolTxs }) + this.setState({ configTxs: newProps.configTxs }) this.setState({ animateFirst: false }) } @@ -145,7 +146,7 @@ class HomePage extends Component { if (networkDetails) { const { id: indyNetworkId } = networkDetails if (indyNetworkId) { - let socket = getWebsocketClient() + let socket = assureWebsocketClient() console.log(`home.js configureSocketForCurrentNetwork ${indyNetworkId}`) socket.on('connection', function (_socket) { @@ -167,8 +168,12 @@ class HomePage extends Component { } componentDidMount () { - const { networkDetails } = this.props - this.configureSocketForCurrentNetwork(networkDetails) + const { networkDetails, features } = this.props + if (features.websockets === true) { + this.configureSocketForCurrentNetwork(networkDetails) + } else { + console.log("Feature websockets is not enabled.") + } this.interval = setInterval(this.recalcProgress.bind(this), 350) } @@ -177,7 +182,7 @@ class HomePage extends Component { clearInterval(this.interval) const socket = getWebsocketClient() if (socket) { - console.log(`CLEANING socket listeres. Had listeners=${socket.hasListeners()}`) + console.log(`Cleaning socket listeners. Had listeners=${socket.hasListeners()}`) socket.off('rescan-scheduled') socket.off('tx-processed') socket.off('switched-room-notification') @@ -186,7 +191,7 @@ class HomePage extends Component { render () { const { network, networkDetails, baseUrl } = this.props - const { domainExpansionTxs, poolExpansionTxs, configExpansionTxs } = this.state + const { domainTxs, poolTxs, configTxs } = this.state const { scanProgressDomain, scanProgressPool, scanProgressConfig } = this.state const isInteractive = (!!this.state.activeWsRoom) return ( @@ -209,7 +214,7 @@ class HomePage extends Component { - @@ -220,7 +225,7 @@ class HomePage extends Component { - @@ -231,7 +236,7 @@ class HomePage extends Component { - diff --git a/indyscan-webapp/pages/tx.js b/indyscan-webapp/pages/tx.js index e8b230c3..346dd82a 100644 --- a/indyscan-webapp/pages/tx.js +++ b/indyscan-webapp/pages/tx.js @@ -16,13 +16,10 @@ class Tx extends Component { static async getInitialProps ({ req, query }) { const { network, ledger, seqNo } = query const baseUrl = getBaseUrl(req) - let txExpansion - let txSerialized let displayMessage + let indyscanTx try { - let { idata } = await getTx(baseUrl, network, ledger, seqNo, 'full') - txExpansion = idata.expansion - txSerialized = idata.serialized + indyscanTx = await getTx(baseUrl, network, ledger, seqNo, 'full') } catch (e) { displayMessage = This transaction is not yet available @@ -30,10 +27,9 @@ class Tx extends Component { } return { + indyscanTx, displayMessage, - txExpansion, baseUrl, - txSerialized, network, ledger, seqNo @@ -74,8 +70,10 @@ class Tx extends Component { value: `color:${palette[3]};`, boolean: `color:${palette[4]};` } + const { baseUrl, network, ledger, seqNo, indyscanTx, displayMessage } = this.props - const { baseUrl, txSerialized, network, ledger, seqNo, txExpansion, displayMessage } = this.props + const txExpansion = indyscanTx?.idata?.expansion + const txSerialized = indyscanTx?.idata?.serialized const { href: hrefPrev, as: asPrev } = getTxLinkData(baseUrl, network, ledger, parseInt(seqNo) - 1) const { href: hrefNext, as: asNext } = getTxLinkData(baseUrl, network, ledger, parseInt(seqNo) + 1) return ( @@ -86,57 +84,57 @@ class Tx extends Component { - + Next tx - +

{`${network} / ${ledger} / ${seqNo}`}

- + Prev tx
{displayMessage && displayMessage} - {txExpansion && - - - + {indyscanTx && + + + } {txExpansion && - - - -

Enriched data

-
- -
+ + + +

Enriched data

+
+ +
} {txExpansion && - - - - {} - - - + + + + {} + + + } {txSerialized && - - - -

Original ledger data

-
- -
+ + + +

Original ledger data

+
+ +
} {txSerialized && - - - - {} - - - + + + + {} + + + } diff --git a/indyscan-webapp/pages/txs.js b/indyscan-webapp/pages/txs.js index 08fe1f76..3ea241df 100644 --- a/indyscan-webapp/pages/txs.js +++ b/indyscan-webapp/pages/txs.js @@ -71,7 +71,7 @@ class Txs extends Component { const filterTxNames = (query.filterTxNames) ? JSON.parse(query.filterTxNames) : [] const search = query.search const sortFromRecent = (query.sortFromRecent) ? query.sortFromRecent : 'true' - const indyscanTxs = await getTxs(baseUrl, network, ledger, skip, pageSize, filterTxNames, 'expansion', search, sortFromRecent) + const indyscanTxs = await getTxs(baseUrl, network, ledger, skip, pageSize, filterTxNames, 'full', search, sortFromRecent) const txCount = await getTxCount(baseUrl, network, ledger, filterTxNames, search) return { indyscanTxs, diff --git a/indyscan-webapp/server/config.js b/indyscan-webapp/server/config.js index b0be3925..d2d2e4b6 100644 --- a/indyscan-webapp/server/config.js +++ b/indyscan-webapp/server/config.js @@ -15,7 +15,7 @@ logger.info(`Loaded configuration:\n${JSON.stringify(appConfig, null, 2)}`) const configValidation = Joi.object().keys({ PORT: Joi.number().integer().min(1025).max(65535).required(), INDYSCAN_API_URL: Joi.string().uri().required(), - DAEMON_WS_URL: Joi.string().uri().required(), + DAEMON_WS_URL: Joi.string().uri(), LOG_LEVEL: Joi.string().valid(['trace', 'debug', 'info', 'warn', 'error']).required(), LOG_HTTP_REQUESTS: Joi.string().valid(['true', 'false']).required(), LOG_HTTP_RESPONSES: Joi.string().valid(['true', 'false']).required() diff --git a/indyscan-webapp/server/index.js b/indyscan-webapp/server/index.js index 1beb9330..a6824b59 100644 --- a/indyscan-webapp/server/index.js +++ b/indyscan-webapp/server/index.js @@ -45,6 +45,10 @@ async function startServer () { return res.status(200).send({ version: module.exports.version }) }) + server.get('/features', (req, res) => { + return res.status(200).send({ websockets: !!appConfig.DAEMON_WS_URL }) + }) + server.get('/home/:network', (req, res) => { const mergedQuery = Object.assign({}, req.query, req.params) logger.info(`Custom express routing handler: /home/:network\nmerged query: ${JSON.stringify(mergedQuery)}`) @@ -63,16 +67,18 @@ async function startServer () { return app.render(req, res, '/tx', mergedQuery) }) + logger.info(`Creating proxy, /api to ${appConfig.INDYSCAN_API_URL}, /socket.io to: ${appConfig.DAEMON_WS_URL}`) server.use( '/api', proxy({ target: appConfig.INDYSCAN_API_URL, changeOrigin: true }) ) - - server.use( - '/socket.io', - proxy({ target: appConfig.DAEMON_WS_URL, ws: true, changeOrigin: true }) - ) + if (appConfig.DAEMON_WS_URL) { + server.use( + '/socket.io', + proxy({ target: appConfig.DAEMON_WS_URL, ws: true, changeOrigin: true }) + ) + } server.get('*', (req, res) => { return handle(req, res) diff --git a/indyscan-webapp/txtools/index.js b/indyscan-webapp/txtools/index.js index a891d1c4..39292220 100644 --- a/indyscan-webapp/txtools/index.js +++ b/indyscan-webapp/txtools/index.js @@ -1,3 +1,5 @@ +import { txTypeToTxName } from 'indyscan-txtype' + function extractClassDataNym (txExpansion) { return [ { priority: 5, label: 'Target DID', value: txExpansion.idata.txn.data.dest }, @@ -109,11 +111,28 @@ const txDataDescriptiveExtractors = { 'UNKNOWN': empty } -export function extractTxDataBasic (txExpansion) { - const { typeName } = txExpansion.idata.txn - const { txnId, txnTime: txnTimeIso8601, seqNo } = txExpansion.idata.txnMetadata - const from = txExpansion.idata.txn.metadata ? txExpansion.idata.txn.metadata.from : 'not-available' - return { txnId, seqNo, txnTimeIso8601, typeName, from } +export function extractTxDataBasic (txFull) { + const { seqNo } = txFull.imeta + let txnId, txnTimeIso8601, typeName, from, indexedFields + if (txFull?.idata?.expansion) { + typeName = txFull.idata.expansion.idata.txn.typeName + txnId = txFull.idata.expansion.idata.txnMetadata.txnId + const epoch = txFull.idata.expansion?.idata?.txnMetadata?.txnTime + txnTimeIso8601 = epoch ? new Date(epoch).toISOString() : null + from = txFull?.idata?.expansion?.idata?.txn?.metadata?.from || '-' + indexedFields = true + } else if (txFull?.idata?.serialized) { + const deserializedOriginal = JSON.parse(txFull.idata.serialized.idata.json) + txnId = deserializedOriginal.txnMetadata.txnId + const epoch = deserializedOriginal.txnMetadata.txnTime * 1000 + txnTimeIso8601 = epoch ? new Date(epoch).toISOString() : null + typeName = txTypeToTxName(deserializedOriginal.txn.type) + from = deserializedOriginal.txn.metadata.from + indexedFields = false + } else { + throw Error("Malformed transaction format, does not contain expansion nor serialized format.") + } + return { txnId, seqNo, txnTimeIso8601, typeName, from, indexedFields } } export function converTxDataBasicToHumanReadable (txDataBasic) { From 45952dafc28ab0d637369b40307777984c9c2039 Mon Sep 17 00:00:00 2001 From: Patrik Stas Date: Thu, 8 Sep 2022 00:54:18 +0200 Subject: [PATCH 2/3] Drive UI updates by ledgercpy workers events Signed-off-by: Patrik Stas --- .../app-config/sovrin-buildernet.json | 20 +++++++++ indyscan-api/nodemon-buildernet.json | 15 +++++++ indyscan-api/package.json | 1 + indyscan-api/src/service/service-storages.js | 2 +- indyscan-daemon/app-configs/sovbuilder.json | 9 ---- indyscan-daemon/package.json | 1 + indyscan-daemon/src/constants.js | 8 ++++ indyscan-daemon/src/index.js | 12 ++--- indyscan-daemon/src/logging/logger-builder.js | 8 ++-- indyscan-daemon/src/server/server.js | 28 +++++++++--- indyscan-daemon/src/server/wsockets.js | 10 ++--- .../src/worker-templates/rtw-db-expansion.js | 3 +- .../rtw-ledger-to-serialized.js | 4 +- indyscan-daemon/src/workers/worker-rtw.js | 20 ++++++--- .../test/unit/workers/timing.spec.js | 45 +++++++++++++++++++ indyscan-webapp/nodemon-indyscan.json | 13 ++++++ indyscan-webapp/nodemon.json | 3 +- indyscan-webapp/pages/home.js | 15 ++++--- indyscan-webapp/txtools/index.js | 18 ++++---- 19 files changed, 178 insertions(+), 57 deletions(-) create mode 100644 indyscan-api/app-config/sovrin-buildernet.json create mode 100644 indyscan-api/nodemon-buildernet.json create mode 100644 indyscan-daemon/src/constants.js create mode 100644 indyscan-daemon/test/unit/workers/timing.spec.js create mode 100644 indyscan-webapp/nodemon-indyscan.json diff --git a/indyscan-api/app-config/sovrin-buildernet.json b/indyscan-api/app-config/sovrin-buildernet.json new file mode 100644 index 00000000..e682f02a --- /dev/null +++ b/indyscan-api/app-config/sovrin-buildernet.json @@ -0,0 +1,20 @@ +[ + { + "id": "SOVRIN_BUILDERNET", + "ui": { + "priority": 1, + "display": "BuilderNet", + "display-long": "Sovrin BuilderNet", + "description": "For active development of your solution.", + "tutorial": "Get your DID and start writing on the network", + "tutorial-link": "https://selfserve.sovrin.org/", + "logo-address": "/static/sovrin.png" + }, + "aliases": [ + "sovbuilder" + ], + "es" : { + "index": "txs-sovbuilder" + } + } +] diff --git a/indyscan-api/nodemon-buildernet.json b/indyscan-api/nodemon-buildernet.json new file mode 100644 index 00000000..138e8730 --- /dev/null +++ b/indyscan-api/nodemon-buildernet.json @@ -0,0 +1,15 @@ +{ + "verbose": true, + "ignore": ["node_modules", ".next"], + "watch": ["src/**/*"], + "ext": "js json", + "env": { + "ES_URL": "http://localhost:9200", + "DAEMON_URL": "http://localhost:3709", + "LOG_LEVEL": "info", + "PORT" : 3708, + "LOG_HTTP_REQUESTS" : true, + "LOG_HTTP_RESPONSES" : true, + "NETWORKS_CONFIG_PATH": "./app-config/sovrin-buildernet.json" + } +} diff --git a/indyscan-api/package.json b/indyscan-api/package.json index e2588af2..4cc6f112 100644 --- a/indyscan-api/package.json +++ b/indyscan-api/package.json @@ -7,6 +7,7 @@ "lint": "standard", "lint:fix": "standard --fix", "dev": "nodemon src/index.js", + "dev:sovrin:builder": "nodemon --config nodemon-buildernet.json src/index.js", "test:unit": "jest tests/unit", "start": "cross-env NODE_ENV=production node src/index.js" }, diff --git a/indyscan-api/src/service/service-storages.js b/indyscan-api/src/service/service-storages.js index b36296f5..711eaf8e 100644 --- a/indyscan-api/src/service/service-storages.js +++ b/indyscan-api/src/service/service-storages.js @@ -8,7 +8,7 @@ const { createStorageReadEs } = require('indyscan-storage/src') async function createLedgerStorageManager (esUrl) { const storages = {} - logger.info(`Connecting to ElasticSearh '${esUrl}'.`) + logger.info(`Connecting to ElasticSearch '${esUrl}'.`) const esClient = new elasticsearch.Client({ node: esUrl }) async function addIndyNetwork (networkId, networkEsIndex) { diff --git a/indyscan-daemon/app-configs/sovbuilder.json b/indyscan-daemon/app-configs/sovbuilder.json index aefda9c2..eb18a2b9 100644 --- a/indyscan-daemon/app-configs/sovbuilder.json +++ b/indyscan-daemon/app-configs/sovbuilder.json @@ -12,15 +12,6 @@ "genesisPath": "{{{cfgdir}}}/genesis/{{{INDY_NETWORK}}}.txn", "esIndex": "{{{ES_INDEX}}}", "esUrl": "{{{ES_URL}}}", - "workerTiming": "SLOW" - } - }, - { - "builder": "rtwExpansion", - "params": { - "indyNetworkId": "{{{INDY_NETWORK}}}", - "esUrl": "{{{ES_URL}}}", - "esIndex": "{{{ES_INDEX}}}", "workerTiming": "MEDIUM" } } diff --git a/indyscan-daemon/package.json b/indyscan-daemon/package.json index f1c021ab..3f258014 100644 --- a/indyscan-daemon/package.json +++ b/indyscan-daemon/package.json @@ -14,6 +14,7 @@ "lint:fix": "standard --fix", "dev": "cross-env NODE_ENV=development nodemon src/index.js", "dev:sovrin:staging:builder": "cross-env NODE_ENV=development WORKER_CONFIGS=app-configs/sovstaging.json,app-configs/sovbuilder.json nodemon src/index.js", + "dev:sovrin:builder": "cross-env NODE_ENV=development WORKER_CONFIGS=app-configs/sovbuilder.json nodemon src/index.js", "dev:sovrin:staging": "cross-env NODE_ENV=development WORKER_CONFIGS=app-configs/sovstaging.json nodemon src/index.js", "dev:sovrin:sovmain": "cross-env NODE_ENV=development WORKER_CONFIGS=app-configs/sovmain.json nodemon src/index.js", "dev:sovrin": "cross-env NODE_ENV=development WORKER_CONFIGS=app-configs/sovmain.json,app-configs/sovstaging.json,app-configs/sovbuilder.json nodemon src/index.js", diff --git a/indyscan-daemon/src/constants.js b/indyscan-daemon/src/constants.js new file mode 100644 index 00000000..4dcca267 --- /dev/null +++ b/indyscan-daemon/src/constants.js @@ -0,0 +1,8 @@ +const OPERATION_TYPES = { + LEDGER_CPY: 'ledgercpy', + EXPANSION: 'expansion' +} + +module.exports = { + OPERATION_TYPES +} diff --git a/indyscan-daemon/src/index.js b/indyscan-daemon/src/index.js index 755f69d0..d6d59949 100644 --- a/indyscan-daemon/src/index.js +++ b/indyscan-daemon/src/index.js @@ -27,8 +27,8 @@ const { createNetOpRtwSerialization } = require('./worker-templates/rtw-ledger-t // } // }) -async function buildWorkers (builder, builderParams) { - logger.info(`Going to build workers by ${builder} from ${JSON.stringify(builderParams)}`) +async function buildWorker (builder, builderParams) { + logger.info(`Going to build worker by ${builder} from ${JSON.stringify(builderParams, null, 2)}`) if (builder === 'rtwSerialization') { return createNetOpRtwSerialization(builderParams) } else if (builder === 'rtwExpansion') { @@ -50,14 +50,14 @@ async function run () { await sleep(2000) logger.info(`Will bootstrap app from following operations definitions ${JSON.stringify(workerConfigPaths, null, 2)}`) - for (const workerConfigPath of workerConfigPaths) { + for (const workerConfigPath of workerConfigPaths) { // per each worker config file, render the file const workersConfig = fs.readFileSync(workerConfigPath) const { workersBuildersTemplate, env } = JSON.parse(workersConfig) env.cfgdir = path.dirname(workerConfigPath) - const workerBuilders = JSON.parse(Mustache.render(JSON.stringify(workersBuildersTemplate), env)) - for (const workerBuilder of workerBuilders) { + const workerBuilders = JSON.parse(Mustache.render(JSON.stringify(workersBuildersTemplate), env)) // render template + for (const workerBuilder of workerBuilders) { // one file can define multiple workers const { builder, params } = workerBuilder - const { workers, sources, targets, transformers, iterators } = await buildWorkers(builder, params) + const { workers, sources, targets, transformers, iterators } = await buildWorker(builder, params) allWorkers.push(workers) allSources.push(sources) allTargets.push(targets) diff --git a/indyscan-daemon/src/logging/logger-builder.js b/indyscan-daemon/src/logging/logger-builder.js index e9c4aa1d..f1be450e 100644 --- a/indyscan-daemon/src/logging/logger-builder.js +++ b/indyscan-daemon/src/logging/logger-builder.js @@ -2,10 +2,10 @@ const winston = require('winston') const mkdirp = require('mkdirp') const Elasticsearch = require('winston-elasticsearch') const { format } = require('winston') -const { timestamp, printf } = format +const { timestamp, printf, label } = format -const myFormat = printf(({ level, message, timestamp, metadaemon }) => { - return `${timestamp} [${metadaemon && metadaemon.workerId ? metadaemon.workerId : '--'}] ${level}: ${message}` +const myFormat = printf(({ label, level, message, timestamp, metadaemon }) => { + return `${timestamp} [${label}] ${level}: ${message}` }) function createLogger (loggerName, consoleLogsLevel, enableLogFiles) { @@ -13,8 +13,8 @@ function createLogger (loggerName, consoleLogsLevel, enableLogFiles) { transports: [ new winston.transports.Console({ level: consoleLogsLevel, - label: loggerName, format: winston.format.combine( + label({ label: loggerName }), timestamp(), myFormat, winston.format.colorize({ all: true }) diff --git a/indyscan-daemon/src/server/server.js b/indyscan-daemon/src/server/server.js index 57c03dd5..db25320c 100644 --- a/indyscan-daemon/src/server/server.js +++ b/indyscan-daemon/src/server/server.js @@ -6,6 +6,7 @@ const logger = require('../logging/logger-main') var pretty = require('express-prettify') const { createSocketioManager } = require('./wsockets') const { logRequests, logResponses } = require('./middleware') +const { OPERATION_TYPES } = require('../constants') function setupLoggingMiddlleware (app, enableRequestLogging, enableResponseLogging) { if (enableRequestLogging) { @@ -15,18 +16,30 @@ function setupLoggingMiddlleware (app, enableRequestLogging, enableResponseLoggi app.use(logResponses) } } +function linkLedgerCpyWorkersToSockets(socketioManager, serviceWorkers) { + logger.info(`Linking workers of operationType ${OPERATION_TYPES.LEDGER_CPY} with sockets.`) + const workerQuery = { operationTypes: [OPERATION_TYPES.LEDGER_CPY] } + const workers = serviceWorkers.getWorkers(workerQuery) + for (const worker of workers) { + const emitter = worker.getEventEmitter() + const { workerId, subledger, operationType, indyNetworkId } = worker.getWorkerInfo() + if (operationType === OPERATION_TYPES.LEDGER_CPY) { + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-processed', 'tx-ledger-processed', indyNetworkId, subledger) + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-rescan-scheduled', 'tx-ledger-rescan-scheduled', indyNetworkId, subledger) + } + } +} function linkExpansionWorkersToSockets (socketioManager, serviceWorkers) { - logger.info('Linking workers with sockets.') - const workerQuery = { operationTypes: ['expansion'] } + logger.info(`Linking workers of operationType ${OPERATION_TYPES.EXPANSION} with sockets.`) + const workerQuery = { operationTypes: [OPERATION_TYPES.EXPANSION] } const workers = serviceWorkers.getWorkers(workerQuery) for (const worker of workers) { const emitter = worker.getEventEmitter() - const { subledger, operationType, indyNetworkId } = worker.getWorkerInfo() - logger.info(`Setting up event->ws forward for ${operationType}/${indyNetworkId}/${subledger} `) - if (operationType === 'expansion') { - socketioManager.forwardEmitterEventToWebsocket(emitter, 'tx-processed', indyNetworkId, subledger) - socketioManager.forwardEmitterEventToWebsocket(emitter, 'rescan-scheduled', indyNetworkId, subledger) + const { workerId, subledger, operationType, indyNetworkId } = worker.getWorkerInfo() + if (operationType === OPERATION_TYPES.EXPANSION) { + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-processed', 'tx-processed', indyNetworkId, subledger) + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-rescan-scheduled', 'tx-rescan-scheduled', indyNetworkId, subledger) } } } @@ -47,6 +60,7 @@ function setupWebsockets (expressServer, serviceWorkers) { const socketioManager = createSocketioManager(expressServer) socketioManager.setupBasicSocketioListeners(createRoomJoinReactor(serviceWorkers)) linkExpansionWorkersToSockets(socketioManager, serviceWorkers) + linkLedgerCpyWorkersToSockets(socketioManager, serviceWorkers) } function startServer (serviceWorkers) { diff --git a/indyscan-daemon/src/server/wsockets.js b/indyscan-daemon/src/server/wsockets.js index c5f80739..71b2b9ca 100644 --- a/indyscan-daemon/src/server/wsockets.js +++ b/indyscan-daemon/src/server/wsockets.js @@ -6,17 +6,17 @@ function createSocketioManager (expressServer) { logger.info('Creating socketio manager') const io = socketio(expressServer) - function forwardEmitterEventToWebsocket (emitter, eventName, forwardToRoom, subledger) { - logger.info(`Linking worker emitter to sockets for indyNetworkId=${forwardToRoom} subledger=${subledger}, `) + function forwardEmitterEventToWebsocket (emitter, workerId, sourceEmitterEventName, targetSocketEventName, forwardToRoom) { + logger.info(`Worker ${workerId} events of name ${sourceEmitterEventName} will be broadcasted to room ${targetSocketEventName} as event ${targetSocketEventName} `) - emitter.on(eventName, (payload) => { + emitter.on(sourceEmitterEventName, (payload) => { io.of('/').in(`${forwardToRoom}`).clients((error, clients) => { if (error) { logger.error('Problem listing clients to print info.') } - logger.info(`Broadcasting into room ${forwardToRoom}: "${eventName}" to ids=${JSON.stringify(clients)}`) + logger.info(`Worker ${workerId} emitting sockets event ${targetSocketEventName} to room ${forwardToRoom} to ${clients.length} clients`) }) - io.to(forwardToRoom).emit(eventName, payload) + io.to(forwardToRoom).emit(targetSocketEventName, payload) }) } diff --git a/indyscan-daemon/src/worker-templates/rtw-db-expansion.js b/indyscan-daemon/src/worker-templates/rtw-db-expansion.js index a44cf217..bc5e8b74 100644 --- a/indyscan-daemon/src/worker-templates/rtw-db-expansion.js +++ b/indyscan-daemon/src/worker-templates/rtw-db-expansion.js @@ -5,9 +5,10 @@ const { createTargetElasticsearch } = require('../targets/target-elasticsearch') const { createWorkerRtw } = require('../workers/worker-rtw') const { createIteratorGuided } = require('../iterators/iterator-guided') const { createSourceElasticsearch } = require('../sources/source-elasticsearch') +const { OPERATION_TYPES } = require('../constants') async function createNetOpRtwExpansion ({ indyNetworkId, esUrl, esIndex, workerTiming }) { - const operationType = 'expansion' + const operationType = OPERATION_TYPES.EXPANSION const sourceEs = await createSourceElasticsearch({ indyNetworkId, diff --git a/indyscan-daemon/src/worker-templates/rtw-ledger-to-serialized.js b/indyscan-daemon/src/worker-templates/rtw-ledger-to-serialized.js index ecc8611d..dcf4a361 100644 --- a/indyscan-daemon/src/worker-templates/rtw-ledger-to-serialized.js +++ b/indyscan-daemon/src/worker-templates/rtw-ledger-to-serialized.js @@ -4,9 +4,11 @@ const { createIteratorGuided } = require('../iterators/iterator-guided') const { createTargetElasticsearch } = require('../targets/target-elasticsearch') const { createSourceElasticsearch } = require('../sources/source-elasticsearch') const { createSourceLedger } = require('../sources/source-ledger') +const { OPERATION_TYPES } = require('../constants') + async function createNetOpRtwSerialization ({ indyNetworkId, genesisPath, esUrl, esIndex, workerTiming }) { - const operationType = 'ledgercpy' + const operationType = OPERATION_TYPES.LEDGER_CPY const sourceLedger = await createSourceLedger({ name: indyNetworkId, genesisPath diff --git a/indyscan-daemon/src/workers/worker-rtw.js b/indyscan-daemon/src/workers/worker-rtw.js index 5d042d27..98759ce3 100644 --- a/indyscan-daemon/src/workers/worker-rtw.js +++ b/indyscan-daemon/src/workers/worker-rtw.js @@ -10,12 +10,15 @@ const { envConfig } = require('../config/env') function getExpandedTimingConfig (providedTimingSetup) { let presetData - if (!providedTimingSetup || (typeof providedTimingSetup !== 'string')) { + if (!providedTimingSetup) { presetData = getDefaultPreset() - } else { + } else if (typeof providedTimingSetup === 'string') { presetData = resolvePreset(providedTimingSetup) || getDefaultPreset() + } else if (typeof providedTimingSetup === 'object') { + const defaultPreset = getDefaultPreset() + presetData = { ... defaultPreset, ... providedTimingSetup } } - return Object.assign(presetData, providedTimingSetup) + return presetData } function validateTimingConfig (timingConfig) { @@ -38,10 +41,10 @@ function validateTimingConfig (timingConfig) { } async function createWorkerRtw ({ indyNetworkId, subledger, operationType, iterator, iteratorTxFormat, transformer, target, timing }) { - console.log(`BUILDING WORKER for ${indyNetworkId}... timit= ${timing}`) - const eventEmitter = new EventEmitter() const workerId = `${indyNetworkId}.${subledger}.${operationType}` const logger = createLogger(workerId, envConfig.LOG_LEVEL, envConfig.ENABLE_LOGFILES) + logger.info(`Building RTW worker ${workerId} for network: ${indyNetworkId}`) + const eventEmitter = new EventEmitter() const loggerMetadata = { metadaemon: { workerId, @@ -81,7 +84,7 @@ async function createWorkerRtw ({ indyNetworkId, subledger, operationType, itera throw Error(errMsg) } timing = getExpandedTimingConfig(timing) - logger.info(`Worker ${workerId} using timing ${JSON.stringify(timing)}`) + logger.info(`Effective timing configuration ${JSON.stringify(timing, null, 2)}`) validateTimingConfig(timing) const { timeoutOnSuccess, timeoutOnTxIngestionError, timeoutOnLedgerResolutionError, timeoutOnTxNoFound, jitterRatio } = timing @@ -379,4 +382,7 @@ async function createWorkerRtw ({ indyNetworkId, subledger, operationType, itera } } -module.exports.createWorkerRtw = createWorkerRtw +module.exports = { + createWorkerRtw, + getExpandedTimingConfig +} diff --git a/indyscan-daemon/test/unit/workers/timing.spec.js b/indyscan-daemon/test/unit/workers/timing.spec.js new file mode 100644 index 00000000..cbb568e8 --- /dev/null +++ b/indyscan-daemon/test/unit/workers/timing.spec.js @@ -0,0 +1,45 @@ +const { getExpandedTimingConfig } = require('../../../src/workers/worker-rtw') +const sleep = require('sleep-promise') + +describe('worker timing configuration', () => { + it('should expand timing configuration', async () => { + const timing = getExpandedTimingConfig("FAST") + const expected = { + "timeoutOnSuccess": 1000, + "timeoutOnTxIngestionError": 30000, + "timeoutOnLedgerResolutionError": 30000, + "timeoutOnTxNoFound": 3000, + "jitterRatio": 0.1 + } + expect(timing).toStrictEqual(expected) + }) + + it('should expand timing configuration', async () => { + const timing = { + "timeoutOnSuccess": 1234, + "timeoutOnTxIngestionError": 2345, + "timeoutOnLedgerResolutionError": 3456, + "timeoutOnTxNoFound": 9999, + "jitterRatio": 0.1 + } + const expandedTimingConfig = getExpandedTimingConfig(timing) + expect(expandedTimingConfig).toStrictEqual(timing) + }) + + it('should expand missing timing configuration with defaults', async () => { + const timing = { + "timeoutOnTxIngestionError": 60000, + "timeoutOnLedgerResolutionError": 60000, + "timeoutOnTxNoFound": 9000, + } + const expandedTimingConfig = getExpandedTimingConfig(timing) + const expected = { + "timeoutOnSuccess": 4000, + "timeoutOnTxIngestionError": 60000, + "timeoutOnLedgerResolutionError": 60000, + "timeoutOnTxNoFound": 9000, + "jitterRatio": 0.1 + } + expect(expandedTimingConfig).toStrictEqual(expected) + }) +}) diff --git a/indyscan-webapp/nodemon-indyscan.json b/indyscan-webapp/nodemon-indyscan.json new file mode 100644 index 00000000..c5e3311e --- /dev/null +++ b/indyscan-webapp/nodemon-indyscan.json @@ -0,0 +1,13 @@ +{ + "verbose": true, + "ignore": ["node_modules", ".next"], + "watch": ["server/**/*", "index.js"], + "ext": "js json", + "env": { + "INDYSCAN_API_URL": "https://indyscan.io", + "PORT" : 3707, + "LOG_LEVEL": "debug", + "LOG_HTTP_REQUESTS" : true, + "LOG_HTTP_RESPONSES" : true + } +} diff --git a/indyscan-webapp/nodemon.json b/indyscan-webapp/nodemon.json index c5e3311e..2b2fe497 100644 --- a/indyscan-webapp/nodemon.json +++ b/indyscan-webapp/nodemon.json @@ -4,7 +4,8 @@ "watch": ["server/**/*", "index.js"], "ext": "js json", "env": { - "INDYSCAN_API_URL": "https://indyscan.io", + "INDYSCAN_API_URL": "http://localhost:3708", + "DAEMON_WS_URL": "http://localhost:3709", "PORT" : 3707, "LOG_LEVEL": "debug", "LOG_HTTP_REQUESTS" : true, diff --git a/indyscan-webapp/pages/home.js b/indyscan-webapp/pages/home.js index e150f774..aaaa4f24 100644 --- a/indyscan-webapp/pages/home.js +++ b/indyscan-webapp/pages/home.js @@ -17,13 +17,14 @@ class HomePage extends Component { static async getInitialProps ({ req, query }) { const baseUrl = getBaseUrl(req) const { network } = query - const features = await fetch(`${baseUrl}/features`) + const featuresRes = await fetch(`${baseUrl}/features`) + const features = (await featuresRes.json()) const versionRes = await fetch(`${baseUrl}/version`) const version = (await versionRes.json()).version const networkDetails = await getNetwork(baseUrl, network) - const domainTxs = await getTxs(baseUrl, network, 'domain', 0, 13, [], 'full') - const poolTxs = await getTxs(baseUrl, network, 'pool', 0, 13, [], 'full') - const configTxs = await getTxs(baseUrl, network, 'config', 0, 13, [], 'full') + const domainTxs = await getTxs(baseUrl, network, 'domain', 0, 13, [], 'serialized') + const poolTxs = await getTxs(baseUrl, network, 'pool', 0, 13, [], 'serialized') + const configTxs = await getTxs(baseUrl, network, 'config', 0, 13, [], 'serialized') return { features, networkDetails, @@ -86,10 +87,11 @@ class HomePage extends Component { this.setState({ poolTxs }) } - onTxProcessed (payload) { + onTxDiscovered (payload) { this.setState({ animateFirst: true }) // const {workerData, txData} = payload const { txData } = payload + console.log(`onTxDiscovered >>> ${JSON.stringify(txData)}`) if (txData.imeta.subledger === 'domain') { this.addNewDomainTx(txData) } @@ -157,7 +159,8 @@ class HomePage extends Component { console.log(`switched-room-notification: Entered room ${activeWsRoom}`) this.setState({activeWsRoom}) socket.on('rescan-scheduled', this.onRescanScheduled.bind(this)) - socket.on('tx-processed', this.onTxProcessed.bind(this)) + // socket.on('tx-processed', this.onTxProcessed.bind(this)) + socket.on('tx-ledger-processed', this.onTxDiscovered.bind(this)) console.log(`Registered hooks on the socket! ${socket.hasListeners()}`) }) diff --git a/indyscan-webapp/txtools/index.js b/indyscan-webapp/txtools/index.js index 39292220..aec12480 100644 --- a/indyscan-webapp/txtools/index.js +++ b/indyscan-webapp/txtools/index.js @@ -111,18 +111,18 @@ const txDataDescriptiveExtractors = { 'UNKNOWN': empty } -export function extractTxDataBasic (txFull) { - const { seqNo } = txFull.imeta +export function extractTxDataBasic (tx) { + const { seqNo } = tx.imeta let txnId, txnTimeIso8601, typeName, from, indexedFields - if (txFull?.idata?.expansion) { - typeName = txFull.idata.expansion.idata.txn.typeName - txnId = txFull.idata.expansion.idata.txnMetadata.txnId - const epoch = txFull.idata.expansion?.idata?.txnMetadata?.txnTime + if (tx?.idata?.expansion) { + typeName = tx.idata.expansion.idata.txn.typeName + txnId = tx.idata.expansion.idata.txnMetadata.txnId + const epoch = tx.idata.expansion?.idata?.txnMetadata?.txnTime txnTimeIso8601 = epoch ? new Date(epoch).toISOString() : null - from = txFull?.idata?.expansion?.idata?.txn?.metadata?.from || '-' + from = tx?.idata?.expansion?.idata?.txn?.metadata?.from || '-' indexedFields = true - } else if (txFull?.idata?.serialized) { - const deserializedOriginal = JSON.parse(txFull.idata.serialized.idata.json) + } else if (tx?.idata?.json || tx?.idata?.serialized) { + const deserializedOriginal = JSON.parse(tx?.idata?.json || tx?.idata?.serialized?.idata?.json) txnId = deserializedOriginal.txnMetadata.txnId const epoch = deserializedOriginal.txnMetadata.txnTime * 1000 txnTimeIso8601 = epoch ? new Date(epoch).toISOString() : null From 4cac9a9c85148a36ded366f9a762d166cbf72c24 Mon Sep 17 00:00:00 2001 From: Patrik Stas Date: Thu, 8 Sep 2022 18:09:45 +0200 Subject: [PATCH 3/3] Fix scan-scheduled event listening, cleanups, refactring Signed-off-by: Patrik Stas --- indyscan-daemon/app-configs/sovbuilder.json | 9 ++++++++ indyscan-daemon/src/constants.js | 19 ++++++++++++++++- indyscan-daemon/src/server/server.js | 18 +++++++++------- indyscan-daemon/src/server/wsockets.js | 4 ++-- indyscan-daemon/src/workers/worker-rtw.js | 23 +++++++++++---------- indyscan-webapp/pages/home.js | 17 +++++---------- indyscan-webapp/sockets/constants.js | 10 +++++++++ indyscan-webapp/txtools/index.js | 14 ++++--------- 8 files changed, 70 insertions(+), 44 deletions(-) create mode 100644 indyscan-webapp/sockets/constants.js diff --git a/indyscan-daemon/app-configs/sovbuilder.json b/indyscan-daemon/app-configs/sovbuilder.json index eb18a2b9..db4a40ed 100644 --- a/indyscan-daemon/app-configs/sovbuilder.json +++ b/indyscan-daemon/app-configs/sovbuilder.json @@ -14,6 +14,15 @@ "esUrl": "{{{ES_URL}}}", "workerTiming": "MEDIUM" } + }, + { + "builder": "rtwExpansion", + "params": { + "indyNetworkId": "{{{INDY_NETWORK}}}", + "esUrl": "{{{ES_URL}}}", + "esIndex": "{{{ES_INDEX}}}", + "workerTiming": "MEDIUM" + } } ] } diff --git a/indyscan-daemon/src/constants.js b/indyscan-daemon/src/constants.js index 4dcca267..f8e5e520 100644 --- a/indyscan-daemon/src/constants.js +++ b/indyscan-daemon/src/constants.js @@ -1,8 +1,25 @@ +const SOCKETIO_EVENT = { + LEDGER_TX_SCANNED: 'ledger-tx-scanned', + LEDGER_TX_SCAN_SCHEDULED: 'ledger-tx-scan-scheduled', + SCANNED_TX_PROCESSED: 'tx-processed', + SCANNED_TX_PROCESSING_SCHEDULED: 'tx-rescan-scheduled' +} + +const INTERNAL_EVENT = { + TX_RESCAN_SCHEDULED: 'tx-rescan-scheduled', + TX_NOT_AVAILABLE: 'tx-not-available', + TX_PROCESSED: 'tx-processed', + TX_RESOLUTION_ERROR: 'tx-resolution-error', + TX_INGESTION_ERROR: 'tx-ingestion-error' +} + const OPERATION_TYPES = { LEDGER_CPY: 'ledgercpy', EXPANSION: 'expansion' } module.exports = { - OPERATION_TYPES + OPERATION_TYPES, + INTERNAL_EVENT, + SOCKETIO_EVENT } diff --git a/indyscan-daemon/src/server/server.js b/indyscan-daemon/src/server/server.js index db25320c..2f3e4658 100644 --- a/indyscan-daemon/src/server/server.js +++ b/indyscan-daemon/src/server/server.js @@ -3,10 +3,10 @@ const apiWorkers = require('./api/api-workers') const express = require('express') const bodyParser = require('body-parser') const logger = require('../logging/logger-main') -var pretty = require('express-prettify') +const pretty = require('express-prettify') const { createSocketioManager } = require('./wsockets') const { logRequests, logResponses } = require('./middleware') -const { OPERATION_TYPES } = require('../constants') +const { OPERATION_TYPES, INTERNAL_EVENT, SOCKETIO_EVENT } = require('../constants') function setupLoggingMiddlleware (app, enableRequestLogging, enableResponseLogging) { if (enableRequestLogging) { @@ -16,7 +16,8 @@ function setupLoggingMiddlleware (app, enableRequestLogging, enableResponseLoggi app.use(logResponses) } } -function linkLedgerCpyWorkersToSockets(socketioManager, serviceWorkers) { + +function linkLedgerCpyWorkersToSockets (socketioManager, serviceWorkers) { logger.info(`Linking workers of operationType ${OPERATION_TYPES.LEDGER_CPY} with sockets.`) const workerQuery = { operationTypes: [OPERATION_TYPES.LEDGER_CPY] } const workers = serviceWorkers.getWorkers(workerQuery) @@ -24,8 +25,8 @@ function linkLedgerCpyWorkersToSockets(socketioManager, serviceWorkers) { const emitter = worker.getEventEmitter() const { workerId, subledger, operationType, indyNetworkId } = worker.getWorkerInfo() if (operationType === OPERATION_TYPES.LEDGER_CPY) { - socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-processed', 'tx-ledger-processed', indyNetworkId, subledger) - socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-rescan-scheduled', 'tx-ledger-rescan-scheduled', indyNetworkId, subledger) + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, INTERNAL_EVENT.TX_PROCESSED, SOCKETIO_EVENT.LEDGER_TX_SCANNED, indyNetworkId, subledger) + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, INTERNAL_EVENT.TX_RESCAN_SCHEDULED, SOCKETIO_EVENT.LEDGER_TX_SCAN_SCHEDULED, indyNetworkId, subledger) } } } @@ -38,21 +39,22 @@ function linkExpansionWorkersToSockets (socketioManager, serviceWorkers) { const emitter = worker.getEventEmitter() const { workerId, subledger, operationType, indyNetworkId } = worker.getWorkerInfo() if (operationType === OPERATION_TYPES.EXPANSION) { - socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-processed', 'tx-processed', indyNetworkId, subledger) - socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, 'tx-rescan-scheduled', 'tx-rescan-scheduled', indyNetworkId, subledger) + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, INTERNAL_EVENT.TX_PROCESSED, SOCKETIO_EVENT.SCANNED_TX_PROCESSED, indyNetworkId, subledger) + socketioManager.forwardEmitterEventToWebsocket(emitter, workerId, INTERNAL_EVENT.TX_RESCAN_SCHEDULED, SOCKETIO_EVENT.SCANNED_TX_PROCESSING_SCHEDULED, indyNetworkId, subledger) } } } function createRoomJoinReactor (serviceWorkers) { function onRoomJoined (room, socket) { - const workerQuery = { operationTypes: ['expansion'], indyNetworkIds: [room] } + const workerQuery = { operationTypes: [OPERATION_TYPES.EXPANSION], indyNetworkIds: [room] } const workers = serviceWorkers.getWorkers(workerQuery) for (const worker of workers) { const rescanScheduledPayload = worker.requestRescheduleStatus() socket.emit('rescan-scheduled', rescanScheduledPayload) } } + return onRoomJoined } diff --git a/indyscan-daemon/src/server/wsockets.js b/indyscan-daemon/src/server/wsockets.js index 71b2b9ca..fa27c9e0 100644 --- a/indyscan-daemon/src/server/wsockets.js +++ b/indyscan-daemon/src/server/wsockets.js @@ -7,14 +7,14 @@ function createSocketioManager (expressServer) { const io = socketio(expressServer) function forwardEmitterEventToWebsocket (emitter, workerId, sourceEmitterEventName, targetSocketEventName, forwardToRoom) { - logger.info(`Worker ${workerId} events of name ${sourceEmitterEventName} will be broadcasted to room ${targetSocketEventName} as event ${targetSocketEventName} `) + logger.info(`Forwarding worker events ${workerId} / ${sourceEmitterEventName} -----> sockets room ${forwardToRoom} / ${targetSocketEventName} `) emitter.on(sourceEmitterEventName, (payload) => { io.of('/').in(`${forwardToRoom}`).clients((error, clients) => { if (error) { logger.error('Problem listing clients to print info.') } - logger.info(`Worker ${workerId} emitting sockets event ${targetSocketEventName} to room ${forwardToRoom} to ${clients.length} clients`) + logger.info(`Worker ${workerId} emitting sockets event ${forwardToRoom} / ${targetSocketEventName} (${clients.length} clients)`) }) io.to(forwardToRoom).emit(targetSocketEventName, payload) }) diff --git a/indyscan-daemon/src/workers/worker-rtw.js b/indyscan-daemon/src/workers/worker-rtw.js index 98759ce3..4ed620fa 100644 --- a/indyscan-daemon/src/workers/worker-rtw.js +++ b/indyscan-daemon/src/workers/worker-rtw.js @@ -7,6 +7,7 @@ const sleep = require('sleep-promise') const EventEmitter = require('events') const { createLogger } = require('../logging/logger-builder') const { envConfig } = require('../config/env') +const { INTERNAL_EVENT } = require('../constants') function getExpandedTimingConfig (providedTimingSetup) { let presetData @@ -157,9 +158,9 @@ async function createWorkerRtw ({ indyNetworkId, subledger, operationType, itera subledger } } - logger.info(`Transaction seqno=${txMeta.seqNo} processed. Emitting 'tx-processed', 'rescan-scheduled'`) - eventEmitter.emit('tx-processed', { workerData, txData }) - eventEmitter.emit('rescan-scheduled', { workerData, msTillRescan: timerLock.getMsTillUnlock() }) + logger.info(`Transaction ${txMeta.seqNo} processed.`) + eventEmitter.emit(INTERNAL_EVENT.TX_PROCESSED, { workerData, txData }) + eventEmitter.emit(INTERNAL_EVENT.TX_RESCAN_SCHEDULED, { workerData, msTillRescan: timerLock.getMsTillUnlock() }) } // TODO: in all of this even-reacting functions, rename txMeta or reduce signature requirmenets to require just "seqNo" if possible @@ -167,19 +168,19 @@ async function createWorkerRtw ({ indyNetworkId, subledger, operationType, itera txNotAvailableCount++ timerLock.addBlockTime(timeoutOnTxNoFound, jitterRatio) const workerData = eventSharedPayload() - logger.info(`Transaction seqno=${queryMeta.seqNo} not available. Emitting 'tx-not-available', 'rescan-scheduled'`) - eventEmitter.emit('tx-not-available', { workerInfo: getWorkerInfo() }) - eventEmitter.emit('rescan-scheduled', { workerData, msTillRescan: timerLock.getMsTillUnlock() }) + logger.info(`Transaction ${queryMeta.seqNo} not available.`) + eventEmitter.emit(INTERNAL_EVENT.TX_NOT_AVAILABLE, { workerInfo: getWorkerInfo() }) + eventEmitter.emit(INTERNAL_EVENT.TX_RESCAN_SCHEDULED, { workerData, msTillRescan: timerLock.getMsTillUnlock() }) } function resolutionError (queryMeta, error) { cycleExceptionCount++ timerLock.addBlockTime(timeoutOnLedgerResolutionError, jitterRatio) const workerData = eventSharedPayload() - logger.warn(`Transaction seqno=${queryMeta.seqNo} resolution error ${util.inspect(error)}. ` + + logger.warn(`Transaction ${queryMeta.seqNo} resolution error ${util.inspect(error)}. ` + 'Emitting \'tx-resolution-error\', \'rescan-scheduled\'') - eventEmitter.emit('tx-resolution-error', getWorkerInfo()) - eventEmitter.emit('rescan-scheduled', { workerData, msTillRescan: timerLock.getMsTillUnlock() }) + eventEmitter.emit(INTERNAL_EVENT.TX_RESOLUTION_ERROR, getWorkerInfo()) + eventEmitter.emit(INTERNAL_EVENT.TX_RESCAN_SCHEDULED, { workerData, msTillRescan: timerLock.getMsTillUnlock() }) } function ingestionError (error, queryMeta, processedTx) { @@ -189,8 +190,8 @@ async function createWorkerRtw ({ indyNetworkId, subledger, operationType, itera logger.error(`Transaction ${queryMeta.seqNo} ingestion error. Couldn't ingest transaction` + `${JSON.stringify(processedTx)} due to storage ingestion error ${util.inspect(error)} ` + 'Emitting: \'tx-ingestion-error\', \'rescan-scheduled\'.') - eventEmitter.emit('tx-ingestion-error', getWorkerInfo()) - eventEmitter.emit('rescan-scheduled', { workerData, msTillRescan: timerLock.getMsTillUnlock() }) + eventEmitter.emit(INTERNAL_EVENT.TX_INGESTION_ERROR, getWorkerInfo()) + eventEmitter.emit(INTERNAL_EVENT.TX_RESCAN_SCHEDULED, { workerData, msTillRescan: timerLock.getMsTillUnlock() }) } async function processTransaction (txData, txFormat) { diff --git a/indyscan-webapp/pages/home.js b/indyscan-webapp/pages/home.js index aaaa4f24..c2f631ab 100644 --- a/indyscan-webapp/pages/home.js +++ b/indyscan-webapp/pages/home.js @@ -12,6 +12,7 @@ import { CSSTransition } from 'react-transition-group' import { assureWebsocketClient, getWebsocketClient } from '../context/socket-client' import NetworkInfo from '../components/NetworkInfo/NetworkInfo' import SubledgerHeader from '../components/SubledgerHeader/SubledgerHeader' +import { SOCKETIO_EVENT } from '../sockets/constants' class HomePage extends Component { static async getInitialProps ({ req, query }) { @@ -105,7 +106,6 @@ class HomePage extends Component { onRescanScheduled (payload) { const { workerData: { subledger }, msTillRescan } = payload - console.log(`rescan-scheduled = ${subledger} ${msTillRescan}`) const rescanStart = Math.round((new Date()).getTime()) const rescanDone = rescanStart + Math.round(msTillRescan) if (subledger === 'domain') { @@ -143,27 +143,20 @@ class HomePage extends Component { this.setState({ animateFirst: false }) } - configureSocketForCurrentNetwork(networkDetails) { if (networkDetails) { const { id: indyNetworkId } = networkDetails if (indyNetworkId) { let socket = assureWebsocketClient() - console.log(`home.js configureSocketForCurrentNetwork ${indyNetworkId}`) - socket.on('connection', function (_socket) { logger.info(`app.js WS connection established.`) }) - socket.on('switched-room-notification', (activeWsRoom) => { console.log(`switched-room-notification: Entered room ${activeWsRoom}`) this.setState({activeWsRoom}) - socket.on('rescan-scheduled', this.onRescanScheduled.bind(this)) - // socket.on('tx-processed', this.onTxProcessed.bind(this)) - socket.on('tx-ledger-processed', this.onTxDiscovered.bind(this)) - console.log(`Registered hooks on the socket! ${socket.hasListeners()}`) + socket.on(SOCKETIO_EVENT.LEDGER_TX_SCAN_SCHEDULED, this.onRescanScheduled.bind(this)) + socket.on(SOCKETIO_EVENT.LEDGER_TX_SCANNED, this.onTxDiscovered.bind(this)) }) - console.log(`Sending switch-room request for ${indyNetworkId}`) socket.emit('switch-room', indyNetworkId) } @@ -186,8 +179,8 @@ class HomePage extends Component { const socket = getWebsocketClient() if (socket) { console.log(`Cleaning socket listeners. Had listeners=${socket.hasListeners()}`) - socket.off('rescan-scheduled') - socket.off('tx-processed') + socket.off(SOCKETIO_EVENT.LEDGER_TX_SCANNED) + socket.off(SOCKETIO_EVENT.LEDGER_TX_SCAN_SCHEDULED) socket.off('switched-room-notification') } } diff --git a/indyscan-webapp/sockets/constants.js b/indyscan-webapp/sockets/constants.js new file mode 100644 index 00000000..bdbeccbf --- /dev/null +++ b/indyscan-webapp/sockets/constants.js @@ -0,0 +1,10 @@ +const SOCKETIO_EVENT = { + LEDGER_TX_SCANNED: 'ledger-tx-scanned', + LEDGER_TX_SCAN_SCHEDULED: 'ledger-tx-scan-scheduled', + SCANNED_TX_PROCESSED: 'tx-processed', + SCANNED_TX_PROCESSING_SCHEDULED: 'tx-rescan-scheduled' +} + +module.exports = { + SOCKETIO_EVENT +} diff --git a/indyscan-webapp/txtools/index.js b/indyscan-webapp/txtools/index.js index aec12480..27bd361e 100644 --- a/indyscan-webapp/txtools/index.js +++ b/indyscan-webapp/txtools/index.js @@ -114,15 +114,9 @@ const txDataDescriptiveExtractors = { export function extractTxDataBasic (tx) { const { seqNo } = tx.imeta let txnId, txnTimeIso8601, typeName, from, indexedFields - if (tx?.idata?.expansion) { - typeName = tx.idata.expansion.idata.txn.typeName - txnId = tx.idata.expansion.idata.txnMetadata.txnId - const epoch = tx.idata.expansion?.idata?.txnMetadata?.txnTime - txnTimeIso8601 = epoch ? new Date(epoch).toISOString() : null - from = tx?.idata?.expansion?.idata?.txn?.metadata?.from || '-' - indexedFields = true - } else if (tx?.idata?.json || tx?.idata?.serialized) { - const deserializedOriginal = JSON.parse(tx?.idata?.json || tx?.idata?.serialized?.idata?.json) + const serializedOriginal = tx?.idata?.json || tx?.idata?.serialized?.idata?.json + if (serializedOriginal) { + const deserializedOriginal = JSON.parse(serializedOriginal) txnId = deserializedOriginal.txnMetadata.txnId const epoch = deserializedOriginal.txnMetadata.txnTime * 1000 txnTimeIso8601 = epoch ? new Date(epoch).toISOString() : null @@ -130,7 +124,7 @@ export function extractTxDataBasic (tx) { from = deserializedOriginal.txn.metadata.from indexedFields = false } else { - throw Error("Malformed transaction format, does not contain expansion nor serialized format.") + throw Error("Expected transaction in 'serialized' or 'full.serialized' format") } return { txnId, seqNo, txnTimeIso8601, typeName, from, indexedFields } }