Deployment scripts with services and do-everything cmd

README.md

@@ -29,6 +29,12 @@ your platform from the [latest `influence-eth` release](https://github.com/moons
 
 ## Building a dataset of Influence.eth events
 
+Find deployment block:
+
+```bash
+influence-eth find-deployment-block --contract $INFLUENCE_DISPATCHER_ADDRESS
+```
+
 To crawl all events for an Influence.eth contract, you can use:
 
 ```bash

cmd.go

@@ -6,9 +6,12 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"log"
+	"math/big"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/NethermindEth/juno/core/felt"
@@ -29,12 +32,14 @@ func CreateRootCommand() *cobra.Command {
 
 	completionCmd := CreateCompletionCommand(rootCmd)
 	versionCmd := CreateVersionCommand()
+	blockNumberCmd := CreateBlockNumberCommand()
+	doEverythingCmd := CreateDoEverythingCommand()
 	eventsCmd := CreateEventsCommand()
 	findDeploymentBlockCmd := CreateFindDeploymentCmd()
 	parseCmd := CreateParseCommand()
 	leaderboardCmd := CreateLeaderboardCommand()
 	leaderboardsCmd := CreateLeaderboardsCommand()
-	rootCmd.AddCommand(completionCmd, versionCmd, eventsCmd, findDeploymentBlockCmd, parseCmd, leaderboardCmd, leaderboardsCmd)
+	rootCmd.AddCommand(completionCmd, versionCmd, doEverythingCmd, blockNumberCmd, eventsCmd, findDeploymentBlockCmd, parseCmd, leaderboardCmd, leaderboardsCmd)
 
 	// By default, cobra Command objects write to stderr. We have to forcibly set them to output to
 	// stdout.
@@ -109,6 +114,52 @@ func CreateVersionCommand() *cobra.Command {
 	return versionCmd
 }
 
+func CreateBlockNumberCommand() *cobra.Command {
+	var providerURL string
+	var timeout uint64
+
+	blockNumberCmd := &cobra.Command{
+		Use:   "block-number",
+		Short: "Get the current block number on your Starknet RPC provider",
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if providerURL == "" {
+				providerURLFromEnv := os.Getenv("STARKNET_RPC_URL")
+				if providerURLFromEnv == "" {
+					return errors.New("you must provide a provider URL using -p/--provider or set the STARKNET_RPC_URL environment variable")
+				}
+				providerURL = providerURLFromEnv
+			}
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			client, clientErr := rpc.NewClient(providerURL)
+			if clientErr != nil {
+				return clientErr
+			}
+
+			provider := rpc.NewProvider(client)
+
+			ctx := context.Background()
+			if timeout > 0 {
+				ctx, _ = context.WithDeadline(ctx, time.Now().Add(time.Duration(timeout)*time.Second))
+			}
+
+			blockNumber, err := provider.BlockNumber(ctx)
+
+			if err != nil {
+				return err
+			}
+
+			cmd.Println(blockNumber)
+			return nil
+		},
+	}
+	blockNumberCmd.Flags().StringVarP(&providerURL, "provider", "p", "", "The URL of your Starknet RPC provider (defaults to value of STARKNET_RPC_URL environment variable)")
+	blockNumberCmd.Flags().Uint64VarP(&timeout, "timeout", "t", 0, "The timeout for requests to your Starknet RPC provider")
+
+	return blockNumberCmd
+}
+
 func CreateEventsCommand() *cobra.Command {
 	var providerURL, contractAddress string
 	var timeout, fromBlock, toBlock uint64
@@ -328,6 +379,158 @@ func CreateParseCommand() *cobra.Command {
 	return parseCmd
 }
 
+func CreateDoEverythingCommand() *cobra.Command {
+	var providerURL, contractAddress, outfile, fromBlockFilePath string
+	var batchSize, coldInterval, hotInterval, hotThreshold, confirmations int
+
+	doEverythingCmd := &cobra.Command{
+		Use:   "do-everything",
+		Short: "Just do everything with events",
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if providerURL == "" {
+				providerURLFromEnv := os.Getenv("STARKNET_RPC_URL")
+				if providerURLFromEnv == "" {
+					return errors.New("you must provide a provider URL using -p/--provider or set the STARKNET_RPC_URL environment variable")
+				}
+				providerURL = providerURLFromEnv
+			}
+
+			if fromBlockFilePath == "" {
+				return errors.New("flag --from-block-file should be set")
+			}
+
+			if outfile == "" {
+				return errors.New("flag -o/--outfile should be set")
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			client, clientErr := rpc.NewClient(providerURL)
+			if clientErr != nil {
+				return clientErr
+			}
+
+			provider := rpc.NewProvider(client)
+			ctx := context.Background()
+
+			eventsChan := make(chan RawEvent)
+
+			var fromBlock uint64
+			fromBlockFile, err := os.Open(fromBlockFilePath)
+			if err != nil {
+				return err
+			}
+			defer fromBlockFile.Close()
+
+			scanner := bufio.NewScanner(fromBlockFile)
+			if scanner.Scan() {
+				blockNumberStr := scanner.Text()
+				fromBlock, err = strconv.ParseUint(blockNumberStr, 10, 64)
+				if err != nil {
+					return err
+				}
+			}
+
+			latestBlock, err := provider.BlockNumber(ctx)
+			if err != nil {
+				return err
+			}
+
+			if fromBlock > latestBlock {
+				return fmt.Errorf("fromBlock %d can not be less then latest block %d", fromBlock, latestBlock)
+			}
+
+			ofp, err := os.Create(outfile)
+			if err != nil {
+				return err
+			}
+			defer ofp.Close()
+
+			fmt.Printf("Starting processing events from block %d to block %d\n", fromBlock, latestBlock)
+
+			go ContractEvents(ctx, provider, contractAddress, eventsChan, hotThreshold, time.Duration(hotInterval)*time.Millisecond, time.Duration(coldInterval)*time.Millisecond, fromBlock, latestBlock, confirmations, batchSize)
+
+			parser, newParserErr := NewEventParser()
+			if newParserErr != nil {
+				return newParserErr
+			}
+
+			newline := []byte("\n")
+
+			batchCounter := 0
+			eventsCounter := big.NewInt(0)
+			for event := range eventsChan {
+				if batchCounter >= 1000 {
+					fmt.Printf("Processed another 1000 events with total %s, working block number %d\n", eventsCounter.String(), event.BlockNumber)
+					batchCounter = 0
+				}
+				batchCounter++
+				eventsCounter.Add(eventsCounter, big.NewInt(1))
+
+				unparsedEvent := ParsedEvent{Name: EVENT_UNKNOWN, Event: event}
+
+				passThrough := true
+
+				parsedEvent, parseErr := parser.Parse(event)
+				if parseErr == nil {
+					passThrough = false
+
+					parsedEventBytes, marshalErr := json.Marshal(parsedEvent)
+					if marshalErr != nil {
+						return marshalErr
+					}
+
+					if _, writeErr := ofp.Write(parsedEventBytes); writeErr != nil {
+						fmt.Printf("Error writing to file: %v\n", writeErr)
+						continue
+					}
+					if _, writeErr := ofp.Write(newline); writeErr != nil {
+						fmt.Printf("Error writing newline to file: %v\n", writeErr)
+						continue
+					}
+				}
+
+				if passThrough {
+					serializedEvent, marshalErr := json.Marshal(unparsedEvent)
+					if marshalErr != nil {
+						return marshalErr
+					}
+					if _, writeErr := ofp.Write(serializedEvent); writeErr != nil {
+						fmt.Printf("Error writing to file: %v\n", writeErr)
+						continue
+					}
+					if _, writeErr := ofp.Write(newline); writeErr != nil {
+						fmt.Printf("Error writing newline to file: %v\n", writeErr)
+						continue
+					}
+				}
+			}
+
+			fmt.Printf("Processed %s events from block %d to block %d\n", eventsCounter.String(), fromBlock, latestBlock)
+
+			writeBlockErr := os.WriteFile(fromBlockFilePath, []byte(fmt.Sprintf("%d", latestBlock)), 0644)
+			if writeBlockErr != nil {
+				return writeBlockErr
+			}
+			fmt.Printf("Updated old block number %d to %d in file %s\n", fromBlock, latestBlock, fromBlockFilePath)
+
+			return nil
+		},
+	}
+	doEverythingCmd.Flags().StringVarP(&providerURL, "provider", "p", "", "The URL of your Starknet RPC provider (defaults to value of STARKNET_RPC_URL environment variable)")
+	doEverythingCmd.Flags().StringVarP(&contractAddress, "contract", "c", "", "The address of the contract from which to crawl events (if not provided, no contract constraint will be specified)")
+	doEverythingCmd.Flags().IntVarP(&batchSize, "batch-size", "N", 100, "The number of events to fetch per batch (defaults to 100)")
+	doEverythingCmd.Flags().IntVar(&hotThreshold, "hot-threshold", 2, "Number of successive iterations which must return events before we consider the crawler hot")
+	doEverythingCmd.Flags().IntVar(&hotInterval, "hot-interval", 100, "Milliseconds at which to poll the provider for updates on the contract while the crawl is hot")
+	doEverythingCmd.Flags().IntVar(&coldInterval, "cold-interval", 10000, "Milliseconds at which to poll the provider for updates on the contract while the crawl is cold")
+	doEverythingCmd.Flags().IntVar(&confirmations, "confirmations", 5, "Number of confirmations to wait for before considering a block canonical")
+	doEverythingCmd.Flags().StringVarP(&fromBlockFilePath, "from-block-file", "f", "", "File contains the block number from which to start crawling")
+	doEverythingCmd.Flags().StringVarP(&outfile, "outfile", "o", "", "File to write reparsed events to")
+
+	return doEverythingCmd
+}
+
 type LeaderboardCommandCreator func(infile, outfile, accessToken, leaderboardId *string) error
 
 type LeaderboardCommandFunc struct {

deploy/deploy.bash

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Deployment script
+
+# Colors
+C_RESET='\033[0m'
+C_RED='\033[1;31m'
+C_GREEN='\033[1;32m'
+C_YELLOW='\033[1;33m'
+
+# Logs
+PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
+
+# Main
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-west-1}"
+APP_DIR="${APP_DIR:-/home/ubuntu/influence-eth}"
+SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/influence-eth-secrets}"
+PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
+FROM_BLOCK_FILE_PATH="${SECRETS_DIR}/from-block.txt"
+SCRIPT_DIR="$(realpath $(dirname $0))"
+USER_SYSTEMD_DIR="${USER_SYSTEMD_DIR:-/home/ubuntu/.config/systemd/user}"
+
+# Service files
+EVENTS_SERVICE_FILE="influence-eth-events.service"
+EVENTS_TIMER_FILE="influence-eth-events.timer"
+LEADERBOARDS_SERVICE_FILE="influence-eth-leaderboards.service"
+LEADERBOARDS_TIMER_FILE="influence-eth-leaderboards.timer"
+
+set -eu
+
+echo
+echo
+echo -e "${PREFIX_INFO} Building executable script with Go"
+EXEC_DIR=$(pwd)
+cd "${APP_DIR}"
+HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_DIR}/influence-eth" .
+cd "${EXEC_DIR}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} If file from-block.txt does not exists, create new one and push deployment block of contract"
+if [ ! -f "${FROM_BLOCK_FILE_PATH}" ]; then
+  touch "${FROM_BLOCK_FILE_PATH}"
+  echo -e "${PREFIX_WARN} Created new from-block file at ${FROM_BLOCK_FILE_PATH}"
+
+  source "${APP_DIR}/starknet.sepolia.env"
+  HOME=/home/ubuntu "${APP_DIR}/influence-eth" find-deployment-block --contract "${INFLUENCE_DISPATCHER_ADDRESS}" > "${FROM_BLOCK_FILE_PATH}"
+fi
+
+echo
+echo
+echo -e "${PREFIX_INFO} Prepare user systemd directory"
+if [ ! -d "${USER_SYSTEMD_DIR}" ]; then
+  mkdir -p "${USER_SYSTEMD_DIR}"
+  echo -e "${PREFIX_WARN} Created new user systemd directory"
+fi
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing influence-eth-events service and timer with ${EVENTS_SERVICE_FILE}, ${EVENTS_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${EVENTS_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${EVENTS_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${EVENTS_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${EVENTS_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${EVENTS_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user restart --no-block "${EVENTS_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing influence-eth-events service and timer with ${LEADERBOARDS_SERVICE_FILE}, ${LEADERBOARDS_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_SERVICE_FILE}" "${SCRIPT_DIR}/${LEADERBOARDS_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${LEADERBOARDS_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${LEADERBOARDS_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${LEADERBOARDS_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${LEADERBOARDS_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user restart --no-block "${LEADERBOARDS_TIMER_FILE}"

deploy/influence-eth-events.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Fulfill file with Infulence-eth events
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/influence-eth
+EnvironmentFile=/home/ubuntu/influence-eth-secrets/app.env
+ExecStart=/home/ubuntu/influence-eth-env/influence-eth do-everything --contract $INFLUENCE_DISPATCHER_ADDRESS --outfile $INFLUENCE_EVENTS_FILE --batch-size 1000 --from-block-file /home/ubuntu/influence-eth-secrets/from-block.txt
+m-block.txt
+CPUWeight=50
+SyslogIdentifier=influence-eth-events

deploy/influence-eth-events.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run events fulfill service
+
+[Timer]
+OnBootSec=120s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

deploy/influence-eth-leaderboards.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Update scores at leaderboards
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/influence-eth
+EnvironmentFile=/home/ubuntu/influence-eth-secrets/app.env
+ExecStart=/home/ubuntu/influence-eth-env/influence-eth leaderboards --infile $INFLUENCE_EVENTS_FILE --leaderboards-map leaderboards-map.json
+m-block.txt
+CPUWeight=50
+SyslogIdentifier=influence-eth-leaderboards

deploy/influence-eth-leaderboards.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Update scores at leaderboards
+
+[Timer]
+OnBootSec=480s
+OnUnitActiveSec=10m
+
+[Install]
+WantedBy=timers.target

sample.env

@@ -1,2 +1,3 @@
 export STARKNET_RPC_URL="<json_rpc_url_to_starknet_node>"
 export MOONSTREAM_ACCESS_TOKEN="<moonstream_user_access_token_uuid>"
+export INFLUENCE_EVENTS_FILE="<path_to_events_file>"