util_rebuild.py

from sqlalchemy import create_engine, desc, func 
from sqlalchemy.orm import sessionmaker 
from models import SystemData, TokenBase, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, TokenContractAssociation, ContractBase, ContractStructure, ContractParticipants, ContractTransactionHistory, ContractDeposits, ConsumedInfo, ContractWinners, ContinuosContractBase, ContractStructure2, ContractParticipants2, ContractDeposits2, ContractTransactionHistory2, SystemBase, ActiveContracts, SystemData, ContractAddressMapping, TokenAddressMapping, DatabaseTypeMapping, TimeActions, RejectedContractTransactionHistory, RejectedTransactionHistory, LatestCacheBase, LatestTransactions, LatestBlocks 
import json 
from tracktokens_smartcontracts import processTransaction, checkLocal_expiry_trigger_deposit, newMultiRequest
import os 
import logging 
import argparse 
import configparser 
import shutil 
import sys 
import pdb


# helper functions
def check_database_existence(type, parameters):
    if type == 'token':
        return os.path.isfile(f"./tokens/{parameters['token_name']}.db")

    if type == 'smart_contract':
        return os.path.isfile(f"./smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db")


def create_database_connection(type, parameters):
    if type == 'token':
        engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True)
    elif type == 'smart_contract':
        engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True)
    elif type == 'system_dbs':
        engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False)

    connection = engine.connect()
    return connection


def create_database_session_orm(type, parameters, base):
    if type == 'token':
        engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True)
        base.metadata.create_all(bind=engine)
        session = sessionmaker(bind=engine)()

    elif type == 'smart_contract':
        engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True)
        base.metadata.create_all(bind=engine)
        session = sessionmaker(bind=engine)()
    
    elif type == 'system_dbs':
        engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False)
        base.metadata.create_all(bind=engine)
        session = sessionmaker(bind=engine)()
    
    return session


# MAIN EXECUTION STARTS
# Configuration of required variables 
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')

file_handler = logging.FileHandler('tracking.log')
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)

stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)

logger.addHandler(file_handler)
logger.addHandler(stream_handler)


#  Rule 1 - Read command line arguments to reset the databases as blank
#  Rule 2     - Read config to set testnet/mainnet
#  Rule 3     - Set flo blockexplorer location depending on testnet or mainnet
#  Rule 4     - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
#  Rule 5     - Set the block number to scan from


# Read command line arguments
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
parser.add_argument('-rb', '--toblocknumer', nargs='?', type=int, help='Forward to the specified block number')
parser.add_argument('-r', '--blockcount', nargs='?', type=int, help='Forward to the specified block count') 
args = parser.parse_args() 

if (args.blockcount and args.toblocknumber):
    print("You can only specify one of the options -b or -c")
    sys.exit(0)
elif args.blockcount:
    forward_block = lastscannedblock + args.blockcount
elif args.toblocknumer:
    forward_block = args.toblocknumer
else:
    latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache'}, LatestCacheBase)
    forward_block = int(latestCache_session.query(LatestBlocks.blockNumber).order_by(LatestBlocks.blockNumber.desc()).first()[0])
    latestCache_session.close()


args = parser.parse_args()

apppath = os.path.dirname(os.path.realpath(__file__))
dirpath = os.path.join(apppath, 'tokens')
if not os.path.isdir(dirpath):
    os.mkdir(dirpath)
dirpath = os.path.join(apppath, 'smartContracts')
if not os.path.isdir(dirpath):
    os.mkdir(dirpath)

# rename all the old databases 
# system.db , latestCache.db, smartContracts, tokens 
if os.path.isfile('./system.db'):
    os.rename('system.db', 'system1.db')
if os.path.isfile('./latestCache.db'):
    os.rename('latestCache.db', 'latestCache1.db')
if os.path.isfile('./smartContracts'):
    os.rename('smartContracts', 'smartContracts1')
if os.path.isfile('./tokens'):
    os.rename('tokens', 'tokens1')

# Read configuration
config = configparser.ConfigParser()
config.read('config.ini')

# todo - write all assertions to make sure default configs are right 
if (config['DEFAULT']['NET'] != 'mainnet') and (config['DEFAULT']['NET'] != 'testnet'):
    logger.error("NET parameter in config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
    sys.exit(0)

# Specify mainnet and testnet server list for API calls and websocket calls 
serverlist = None
if config['DEFAULT']['NET'] == 'mainnet':
    serverlist = config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
elif config['DEFAULT']['NET'] == 'testnet':
    serverlist = config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
serverlist = serverlist.split(',')
neturl = config['DEFAULT']['FLOSIGHT_NETURL']
tokenapi_sse_url = config['DEFAULT']['TOKENAPI_SSE_URL']

# Delete database and smartcontract directory if reset is set to 1
#if args.reset == 1:
logger.info("Resetting the database. ")
apppath = os.path.dirname(os.path.realpath(__file__))
dirpath = os.path.join(apppath, 'tokens')
shutil.rmtree(dirpath)
os.mkdir(dirpath)
dirpath = os.path.join(apppath, 'smartContracts')
shutil.rmtree(dirpath)
os.mkdir(dirpath)
dirpath = os.path.join(apppath, 'system.db')
if os.path.exists(dirpath):
    os.remove(dirpath)
dirpath = os.path.join(apppath, 'latestCache.db')
if os.path.exists(dirpath):
    os.remove(dirpath)

# Read start block no
startblock = int(config['DEFAULT']['START_BLOCK'])
session = create_database_session_orm('system_dbs', {'db_name': "system"}, SystemBase)
session.add(SystemData(attribute='lastblockscanned', value=startblock - 1))
session.commit()
session.close()

# Initialize latest cache DB
session = create_database_session_orm('system_dbs', {'db_name': "latestCache"}, LatestCacheBase)
session.commit()
session.close()

# get all blocks and transaction data 
latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache1'}, LatestCacheBase)
if forward_block:
    lblocks = latestCache_session.query(LatestBlocks).filter(LatestBlocks.blockNumber <= forward_block).all()
    ltransactions = latestCache_session.query(LatestTransactions).filter(LatestTransactions.blockNumber <= forward_block).all()
else:
    lblocks = latestCache_session.query(LatestBlocks).all()
    ltransactions = latestCache_session.query(LatestTransactions).all()
latestCache_session.close()

# make a list of all internal tx block numbers
systemDb_session = create_database_session_orm('system_dbs', {'db_name':'system1'}, SystemBase)
internal_action_blocks = systemDb_session.query(ActiveContracts.blockNumber).all()
internal_action_blocks = [block[0] for block in internal_action_blocks]
internal_action_blocks = sorted(internal_action_blocks)

lblocks_dict = {}
for block in lblocks:
    block_dict = block.__dict__
    print(block_dict['blockNumber'])
    lblocks_dict[block_dict['blockNumber']] = {'blockHash':f"{block_dict['blockHash']}", 'jsonData':f"{block_dict['jsonData']}"}

# process and rebuild all transactions 
prev_block = 0

for transaction in ltransactions:
    transaction_dict = transaction.__dict__
    current_block = transaction_dict['blockNumber']

    # Check if any internal action block lies between prev_block and current_block
    for internal_block in internal_action_blocks:
        if prev_block < internal_block <= current_block:
            logger.info(f'Processing block {internal_block}') 
            # Get block details 
            response = newMultiRequest(f"block-index/{internal_block}") 
            blockhash = response['blockHash'] 
            blockinfo = newMultiRequest(f"block/{blockhash}")
            # Call your function here, passing the internal block to it
            checkLocal_expiry_trigger_deposit(blockinfo)

    transaction_data = json.loads(transaction_dict['jsonData'])
    parsed_flodata = json.loads(transaction_dict['parsedFloData'])
    try:
        block_info = json.loads(lblocks_dict[transaction_dict['blockNumber']]['jsonData'])
        processTransaction(transaction_data, parsed_flodata, block_info)
        prev_block = current_block
    except:
        prev_block = current_block
        continue

# copy the old block data 
old_latest_cache = create_database_connection('system_dbs', {'db_name':'latestCache1'})
old_latest_cache.execute("ATTACH DATABASE 'latestCache.db' AS new_db")
old_latest_cache.execute("INSERT INTO new_db.latestBlocks SELECT * FROM latestBlocks WHERE blockNumber <= ?", (forward_block,))
old_latest_cache.close()

# delete 
# system.db , latestCache.db, smartContracts, tokens 
if os.path.isfile('./system1.db'):
    os.remove('system1.db')
if os.path.isfile('./latestCache1.db'):
    os.remove('latestCache1.db')
if os.path.isfile('./smartContracts1'):
    shutil.rmtree('smartContracts1')
if os.path.isfile('./tokens1'):
    shutil.rmtree('tokens1')

# Update system.db's last scanned block 
connection = create_database_connection('system_dbs', {'db_name': "system"})
connection.execute(f"UPDATE systemData SET value = {int(list(lblocks_dict.keys())[-1])} WHERE attribute = 'lastblockscanned';")
connection.close()