diff --git a/README.md b/README.md index 613cc5c..bca0106 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,12 @@ your platform from the [latest `influence-eth` release](https://github.com/moons ## Building a dataset of Influence.eth events +Find deployment block: + +```bash +influence-eth find-deployment-block --contract $INFLUENCE_DISPATCHER_ADDRESS +``` + To crawl all events for an Influence.eth contract, you can use: ```bash diff --git a/cmd.go b/cmd.go index 82e0ca6..9bc85aa 100644 --- a/cmd.go +++ b/cmd.go @@ -6,9 +6,12 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io/ioutil" "log" + "math/big" "os" + "strconv" "time" "github.com/NethermindEth/juno/core/felt" @@ -29,12 +32,14 @@ func CreateRootCommand() *cobra.Command { completionCmd := CreateCompletionCommand(rootCmd) versionCmd := CreateVersionCommand() + blockNumberCmd := CreateBlockNumberCommand() + doEverythingCmd := CreateDoEverythingCommand() eventsCmd := CreateEventsCommand() findDeploymentBlockCmd := CreateFindDeploymentCmd() parseCmd := CreateParseCommand() leaderboardCmd := CreateLeaderboardCommand() leaderboardsCmd := CreateLeaderboardsCommand() - rootCmd.AddCommand(completionCmd, versionCmd, eventsCmd, findDeploymentBlockCmd, parseCmd, leaderboardCmd, leaderboardsCmd) + rootCmd.AddCommand(completionCmd, versionCmd, doEverythingCmd, blockNumberCmd, eventsCmd, findDeploymentBlockCmd, parseCmd, leaderboardCmd, leaderboardsCmd) // By default, cobra Command objects write to stderr. We have to forcibly set them to output to // stdout. @@ -109,6 +114,52 @@ func CreateVersionCommand() *cobra.Command { return versionCmd } +func CreateBlockNumberCommand() *cobra.Command { + var providerURL string + var timeout uint64 + + blockNumberCmd := &cobra.Command{ + Use: "block-number", + Short: "Get the current block number on your Starknet RPC provider", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if providerURL == "" { + providerURLFromEnv := os.Getenv("STARKNET_RPC_URL") + if providerURLFromEnv == "" { + return errors.New("you must provide a provider URL using -p/--provider or set the STARKNET_RPC_URL environment variable") + } + providerURL = providerURLFromEnv + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + client, clientErr := rpc.NewClient(providerURL) + if clientErr != nil { + return clientErr + } + + provider := rpc.NewProvider(client) + + ctx := context.Background() + if timeout > 0 { + ctx, _ = context.WithDeadline(ctx, time.Now().Add(time.Duration(timeout)*time.Second)) + } + + blockNumber, err := provider.BlockNumber(ctx) + + if err != nil { + return err + } + + cmd.Println(blockNumber) + return nil + }, + } + blockNumberCmd.Flags().StringVarP(&providerURL, "provider", "p", "", "The URL of your Starknet RPC provider (defaults to value of STARKNET_RPC_URL environment variable)") + blockNumberCmd.Flags().Uint64VarP(&timeout, "timeout", "t", 0, "The timeout for requests to your Starknet RPC provider") + + return blockNumberCmd +} + func CreateEventsCommand() *cobra.Command { var providerURL, contractAddress string var timeout, fromBlock, toBlock uint64 @@ -328,6 +379,158 @@ func CreateParseCommand() *cobra.Command { return parseCmd } +func CreateDoEverythingCommand() *cobra.Command { + var providerURL, contractAddress, outfile, fromBlockFilePath string + var batchSize, coldInterval, hotInterval, hotThreshold, confirmations int + + doEverythingCmd := &cobra.Command{ + Use: "do-everything", + Short: "Just do everything with events", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if providerURL == "" { + providerURLFromEnv := os.Getenv("STARKNET_RPC_URL") + if providerURLFromEnv == "" { + return errors.New("you must provide a provider URL using -p/--provider or set the STARKNET_RPC_URL environment variable") + } + providerURL = providerURLFromEnv + } + + if fromBlockFilePath == "" { + return errors.New("flag --from-block-file should be set") + } + + if outfile == "" { + return errors.New("flag -o/--outfile should be set") + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + client, clientErr := rpc.NewClient(providerURL) + if clientErr != nil { + return clientErr + } + + provider := rpc.NewProvider(client) + ctx := context.Background() + + eventsChan := make(chan RawEvent) + + var fromBlock uint64 + fromBlockFile, err := os.Open(fromBlockFilePath) + if err != nil { + return err + } + defer fromBlockFile.Close() + + scanner := bufio.NewScanner(fromBlockFile) + if scanner.Scan() { + blockNumberStr := scanner.Text() + fromBlock, err = strconv.ParseUint(blockNumberStr, 10, 64) + if err != nil { + return err + } + } + + latestBlock, err := provider.BlockNumber(ctx) + if err != nil { + return err + } + + if fromBlock > latestBlock { + return fmt.Errorf("fromBlock %d can not be less then latest block %d", fromBlock, latestBlock) + } + + ofp, err := os.Create(outfile) + if err != nil { + return err + } + defer ofp.Close() + + fmt.Printf("Starting processing events from block %d to block %d\n", fromBlock, latestBlock) + + go ContractEvents(ctx, provider, contractAddress, eventsChan, hotThreshold, time.Duration(hotInterval)*time.Millisecond, time.Duration(coldInterval)*time.Millisecond, fromBlock, latestBlock, confirmations, batchSize) + + parser, newParserErr := NewEventParser() + if newParserErr != nil { + return newParserErr + } + + newline := []byte("\n") + + batchCounter := 0 + eventsCounter := big.NewInt(0) + for event := range eventsChan { + if batchCounter >= 1000 { + fmt.Printf("Processed another 1000 events with total %s, working block number %d\n", eventsCounter.String(), event.BlockNumber) + batchCounter = 0 + } + batchCounter++ + eventsCounter.Add(eventsCounter, big.NewInt(1)) + + unparsedEvent := ParsedEvent{Name: EVENT_UNKNOWN, Event: event} + + passThrough := true + + parsedEvent, parseErr := parser.Parse(event) + if parseErr == nil { + passThrough = false + + parsedEventBytes, marshalErr := json.Marshal(parsedEvent) + if marshalErr != nil { + return marshalErr + } + + if _, writeErr := ofp.Write(parsedEventBytes); writeErr != nil { + fmt.Printf("Error writing to file: %v\n", writeErr) + continue + } + if _, writeErr := ofp.Write(newline); writeErr != nil { + fmt.Printf("Error writing newline to file: %v\n", writeErr) + continue + } + } + + if passThrough { + serializedEvent, marshalErr := json.Marshal(unparsedEvent) + if marshalErr != nil { + return marshalErr + } + if _, writeErr := ofp.Write(serializedEvent); writeErr != nil { + fmt.Printf("Error writing to file: %v\n", writeErr) + continue + } + if _, writeErr := ofp.Write(newline); writeErr != nil { + fmt.Printf("Error writing newline to file: %v\n", writeErr) + continue + } + } + } + + fmt.Printf("Processed %s events from block %d to block %d\n", eventsCounter.String(), fromBlock, latestBlock) + + writeBlockErr := os.WriteFile(fromBlockFilePath, []byte(fmt.Sprintf("%d", latestBlock)), 0644) + if writeBlockErr != nil { + return writeBlockErr + } + fmt.Printf("Updated old block number %d to %d in file %s\n", fromBlock, latestBlock, fromBlockFilePath) + + return nil + }, + } + doEverythingCmd.Flags().StringVarP(&providerURL, "provider", "p", "", "The URL of your Starknet RPC provider (defaults to value of STARKNET_RPC_URL environment variable)") + doEverythingCmd.Flags().StringVarP(&contractAddress, "contract", "c", "", "The address of the contract from which to crawl events (if not provided, no contract constraint will be specified)") + doEverythingCmd.Flags().IntVarP(&batchSize, "batch-size", "N", 100, "The number of events to fetch per batch (defaults to 100)") + doEverythingCmd.Flags().IntVar(&hotThreshold, "hot-threshold", 2, "Number of successive iterations which must return events before we consider the crawler hot") + doEverythingCmd.Flags().IntVar(&hotInterval, "hot-interval", 100, "Milliseconds at which to poll the provider for updates on the contract while the crawl is hot") + doEverythingCmd.Flags().IntVar(&coldInterval, "cold-interval", 10000, "Milliseconds at which to poll the provider for updates on the contract while the crawl is cold") + doEverythingCmd.Flags().IntVar(&confirmations, "confirmations", 5, "Number of confirmations to wait for before considering a block canonical") + doEverythingCmd.Flags().StringVarP(&fromBlockFilePath, "from-block-file", "f", "", "File contains the block number from which to start crawling") + doEverythingCmd.Flags().StringVarP(&outfile, "outfile", "o", "", "File to write reparsed events to") + + return doEverythingCmd +} + type LeaderboardCommandCreator func(infile, outfile, accessToken, leaderboardId *string) error type LeaderboardCommandFunc struct { diff --git a/deploy/deploy.bash b/deploy/deploy.bash new file mode 100755 index 0000000..7c7f364 --- /dev/null +++ b/deploy/deploy.bash @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Deployment script + +# Colors +C_RESET='\033[0m' +C_RED='\033[1;31m' +C_GREEN='\033[1;32m' +C_YELLOW='\033[1;33m' + +# Logs +PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]" +PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]" + +# Main +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-west-1}" +APP_DIR="${APP_DIR:-/home/ubuntu/influence-eth}" +SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/influence-eth-secrets}" +PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" +FROM_BLOCK_FILE_PATH="${SECRETS_DIR}/from-block.txt" +SCRIPT_DIR="$(realpath $(dirname $0))" +USER_SYSTEMD_DIR="${USER_SYSTEMD_DIR:-/home/ubuntu/.config/systemd/user}" + +# Service files +EVENTS_SERVICE_FILE="influence-eth-events.service" +EVENTS_TIMER_FILE="influence-eth-events.timer" +LEADERBOARDS_SERVICE_FILE="influence-eth-leaderboards.service" +LEADERBOARDS_TIMER_FILE="influence-eth-leaderboards.timer" + +set -eu + +echo +echo +echo -e "${PREFIX_INFO} Building executable script with Go" +EXEC_DIR=$(pwd) +cd "${APP_DIR}" +HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_DIR}/influence-eth" . +cd "${EXEC_DIR}" + +echo +echo +echo -e "${PREFIX_INFO} If file from-block.txt does not exists, create new one and push deployment block of contract" +if [ ! -f "${FROM_BLOCK_FILE_PATH}" ]; then + touch "${FROM_BLOCK_FILE_PATH}" + echo -e "${PREFIX_WARN} Created new from-block file at ${FROM_BLOCK_FILE_PATH}" + + source "${APP_DIR}/starknet.sepolia.env" + HOME=/home/ubuntu "${APP_DIR}/influence-eth" find-deployment-block --contract "${INFLUENCE_DISPATCHER_ADDRESS}" > "${FROM_BLOCK_FILE_PATH}" +fi + +echo +echo +echo -e "${PREFIX_INFO} Prepare user systemd directory" +if [ ! -d "${USER_SYSTEMD_DIR}" ]; then + mkdir -p "${USER_SYSTEMD_DIR}" + echo -e "${PREFIX_WARN} Created new user systemd directory" +fi + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing influence-eth-events service and timer with ${EVENTS_SERVICE_FILE}, ${EVENTS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${EVENTS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${EVENTS_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${EVENTS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${EVENTS_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${EVENTS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user restart --no-block "${EVENTS_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing influence-eth-events service and timer with ${LEADERBOARDS_SERVICE_FILE}, ${LEADERBOARDS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_SERVICE_FILE}" "${SCRIPT_DIR}/${LEADERBOARDS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${LEADERBOARDS_SERVICE_FILE}" "${USER_SYSTEMD_DIR}/${LEADERBOARDS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${LEADERBOARDS_TIMER_FILE}" "${USER_SYSTEMD_DIR}/${LEADERBOARDS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/$UID" systemctl --user restart --no-block "${LEADERBOARDS_TIMER_FILE}" diff --git a/deploy/influence-eth-events.service b/deploy/influence-eth-events.service new file mode 100644 index 0000000..414c556 --- /dev/null +++ b/deploy/influence-eth-events.service @@ -0,0 +1,12 @@ +[Unit] +Description=Fulfill file with Infulence-eth events +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/influence-eth +EnvironmentFile=/home/ubuntu/influence-eth-secrets/app.env +ExecStart=/home/ubuntu/influence-eth-env/influence-eth do-everything --contract $INFLUENCE_DISPATCHER_ADDRESS --outfile $INFLUENCE_EVENTS_FILE --batch-size 1000 --from-block-file /home/ubuntu/influence-eth-secrets/from-block.txt +m-block.txt +CPUWeight=50 +SyslogIdentifier=influence-eth-events \ No newline at end of file diff --git a/deploy/influence-eth-events.timer b/deploy/influence-eth-events.timer new file mode 100644 index 0000000..0c8daf0 --- /dev/null +++ b/deploy/influence-eth-events.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Run events fulfill service + +[Timer] +OnBootSec=120s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/influence-eth-leaderboards.service b/deploy/influence-eth-leaderboards.service new file mode 100644 index 0000000..d487824 --- /dev/null +++ b/deploy/influence-eth-leaderboards.service @@ -0,0 +1,12 @@ +[Unit] +Description=Update scores at leaderboards +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/influence-eth +EnvironmentFile=/home/ubuntu/influence-eth-secrets/app.env +ExecStart=/home/ubuntu/influence-eth-env/influence-eth leaderboards --infile $INFLUENCE_EVENTS_FILE --leaderboards-map leaderboards-map.json +m-block.txt +CPUWeight=50 +SyslogIdentifier=influence-eth-leaderboards \ No newline at end of file diff --git a/deploy/influence-eth-leaderboards.timer b/deploy/influence-eth-leaderboards.timer new file mode 100644 index 0000000..318b0e1 --- /dev/null +++ b/deploy/influence-eth-leaderboards.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Update scores at leaderboards + +[Timer] +OnBootSec=480s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/sample.env b/sample.env index 2021374..24b0bac 100644 --- a/sample.env +++ b/sample.env @@ -1,2 +1,3 @@ export STARKNET_RPC_URL="" export MOONSTREAM_ACCESS_TOKEN="" +export INFLUENCE_EVENTS_FILE=""