Configuration Improvement: Create config.toml on Application Start (#…

…363)

* configueration improvement

* fix docker build

* fmt

* changelog

* add docstrings

* docs improvement

.gitignore

@@ -5,3 +5,4 @@
 .env.*
 *.log
 credentials.json
+config.toml

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### What's Changed
 * Improved bulk insertion of state_changes, reducing database requests from hundreds to a maximum of 7 per block.
+* Configuration improvement. Create default config.toml on start application to loaded parameters from the environment variables.
 
 ## [0.3.0](https://github.com/near/read-rpc/releases/tag/v0.3.0)
 

README.md

@@ -34,7 +34,7 @@ The configuration module is responsible for managing the configuration settings
 ### Run the entire project
 
 Put TOML file `config.toml` with configuration in the home root of the project.
-See the example [here](./configuration/example.config.toml).
+See the example [here](./configuration/src/default_env_configs.rs).
 
 Run the docker compose:
 

configuration/README.md

@@ -13,10 +13,9 @@ For example, `${DATABASE_URL}` specifies the `DATABASE_URL` environment variable
 
 ## Files
 
-- `example.config.toml`: This file contains an example configuration for the NEAR ReadRPC. 
+- `default_env_config.rs`: This file contains an example configuration for the NEAR ReadRPC. 
 It includes settings for the general configuration, 
-RPC server, transaction indexer, state indexer, epoch indexer, rightsizing, lake framework, and database.
-
+RPC server, tx-indexer, state-indexer, rightsizing, lake-framework, and database.
 
 ## Configuration
 
@@ -36,6 +35,13 @@ The configuration settings are stored in a TOML file. The settings include:
 
 Put TOML file `config.toml` with configuration in the home root of the project.
 
+## Default Configuration
+Configuration improvement. Create default config.toml on start application to loaded parameters from the environment variables.
+If `config.toml` is not found, the application will create default config toml with env variables.
+This file to present all configuration around th environment variables [default_env_config.rs](configuration/src/default_env_config.rs)
+Not present environment variables will be set to default values
+See more details and information about each parameter in [default_env_config.rs](configuration/src/default_env_config.rs)
+
 ## Note
 
 Please ensure that the configuration file is correctly formatted and all the required settings are provided. 

configuration/src/configs/general.rs

@@ -152,7 +152,7 @@ impl CommonGeneralRpcServerConfig {
     }
 
     pub fn default_prefetch_state_size_limit() -> u64 {
-        1_000_000
+        100_000
     }
 }
 

configuration/src/configs/mod.rs

@@ -69,7 +69,7 @@ where
     Ok(match deserialize_data_or_env(data) {
         Ok(value) => Some(value),
         Err(err) => {
-            tracing::warn!("Failed to deserialize_optional_data_or_env: {:?}", err);
+            tracing::debug!("Failed to deserialize_optional_data_or_env: {:?}", err);
             None
         }
     })

configuration/example.config.toml → configuration/src/default_env_configs.rs

@@ -1,62 +1,64 @@
+// This file to present all confiuration around th environment variables
+// Not present environment variables will be set to default values
+// See more details and information about configuration in configuration/README.md
+
+pub const DEFAULT_CONFIG: &str = r#"
 ### General configuration for NEAR ReadRPC
 [general]
-
 ## Chain ID: testnet or mainnet
-chain_id = "mainnet"
+chain_id = "${CHAIN_ID}"
 
 ## Near network rpc url
 ## Using for proxying some requests to near network and to handle finality block
-near_rpc_url = "https://beta.rpc.mainnet.near.org"
+near_rpc_url = "${NEAR_RPC_URL}"
+
+## near network archival rpc url
+## Using for proxying some requests to near network and to handle historical data like proofs
+## default value is None
+near_archival_rpc_url = "${ARCHIVAL_NEAR_RPC_URL}"
 
 ## Referer header value
 ## We want to set a custom referer to let NEAR JSON RPC nodes know that we are a read-rpc instance
 ## Default value is "http://read-rpc.local"
-#referer_header_value = "http://read-rpc.local"
-
-## near network archival rpc url
-## Using under the hood in near network rpc
-## Not needed for regular users.
-## Please, don't uncoment this parameter or use same as near_rpc_url
-## default value is None
-#near_archival_rpc_url = "https://beta.rpc.mainnet.near.org"
+referer_header_value = "${REFERER_HEADER_VALUE}"
 
 ## redis url using for pub/sub optimistic_block and final_block
 ## from near_state_indexer to rpc_server
 ## Default value is redis://127.0.0.1/
-#redis_url = "redis://127.0.0.1/"
+redis_url = "${REDIS_URL}"
 
 ### Rpc server general configuration
 [general.rpc_server]
 
 ## Port for RPC server
 ## Default port is 8000
-#server_port = 8000
+server_port = "${SERVER_PORT}"
 
 ## Max gas burnt for contract function call
 ## We allow to use max gas bunt to run contract function call
 ## Default value is 300_000_000_000_000
-#max_gas_burnt = 300_000_000_000_000
+max_gas_burnt = "${MAX_GAS_BURNT}"
 
 ## Contract code cache in gigabytes
 ## By default we use 0.25 gigabyte (256MB or 268_435_456 bytes)
-#contract_code_cache_size = 0.25
+contract_code_cache_size = "${CONTRACT_CODE_CACHE_SIZE}"
 
 ## Block cache size in gigabytes
 ## By default we use 0.125 gigabyte (128MB or 134_217_728 bytes)
 ## One cache_block size is ≈ 96 bytes
 ## In 128MB we can put 1_398_101 cache_blocks
-#block_cache_size = 0.125
+block_cache_size = "${BLOCK_CACHE_SIZE}"
 
 ## How many requests we should check for data consistency
 ## By default we use 100% of requests
 ## If you want to check 1% of requests, you should set 1
 ## thet means for every method calls will be checked every 100th request
-#shadow_data_consistency_rate = 100
+shadow_data_consistency_rate = "${SHADOW_DATA_CONSISTENCY_RATE}"
 
 ## Max size (in bytes) for state prefetch during a view_call
 ## Limits the amount of data prefetched to speed up the view_call
-## By default, it is set to 1MB (1_000_000 bytes).
-#prefetch_state_size_limit = 1_000_000
+## By default, it is set to 100KB (100_000 bytes).
+prefetch_state_size_limit = "${PREFETCH_STATE_SIZE_LIMIT}"
 
 ### Tx indexer general configuration
 [general.tx_indexer]
@@ -65,11 +67,11 @@ near_rpc_url = "https://beta.rpc.mainnet.near.org"
 ## Unique indexer ID
 ## Default value is "tx-indexer"
 ## If you run multiple instances of the indexer, you should change this value for each instance
-#indexer_id = "tx-indexer"
+indexer_id = "${TX_INDEXER_ID}"
 
 ## Port for metrics server
 ## By default it 8080 for tx-indexer and 8081 for state-indexer
-#metrics_server_port = 8080
+metrics_server_port = "${TX_SERVER_PORT}"
 
 ### State indexer general configuration
 [general.state_indexer]
@@ -78,40 +80,43 @@ near_rpc_url = "https://beta.rpc.mainnet.near.org"
 ## Unique indexer ID
 ## Default value is "state-indexer"
 ## If you run multiple instances of the indexer, you should change this value for each instance
-#indexer_id = "state-indexer"
+indexer_id = "${STATE_INDEXER_ID}"
 
 ## Port for metrics server
 ## By default it 8080 for tx-indexer and 8081 for state-indexer
-#metrics_server_port = 8081
+metrics_server_port = "${STATE_SERVER_PORT}"
 
 ## Concurrency for state-indexer
 ## Default value is 1
-#concurrency = 1
+concurrency = "${CONCURRENCY}"
 
 ### Near state indexer general configuration
 [general.near_state_indexer]
 
 ## Port for metrics server
 ## By default it 8082
-#metrics_server_port = 8082
+metrics_server_port = "${NEAR_STATE_SERVER_PORT}"
 
 ## Concurrency for state-indexer
 ## Default value is 1
-#concurrency = 1
+concurrency = "${NEAR_STATE_CONCURRENCY}"
 
 ### Tracking acconunts and state changes configuration
 [rightsizing]
 
 ## Accounts to track. By default we track all accounts.
 ## You can specify a list of accounts to track.
 ## tracked_accounts = ["test.near"]
-#tracked_accounts = []
+## By default we track all accounts.
+tracked_accounts = "${TRACKED_ACCOUNTS}"
 
 ## State changes to track. By default we track all state changes.
 ## You can specify a list of state changes to track.
 ## Possible values: "state", "access_key", "contract_code"
 ## "accounts" are tracked from the `tracked_accounts` section
-#tracked_changes = ["state", "access_key", "contract_code"]
+## tracked_changes = ["state", "access_key", "contract_code"]
+## By default we track all state changes.
+tracked_changes = "${TRACKED_CHANGES}"
 
 ### Lake framework configuration
 [lake_config]
@@ -123,23 +128,24 @@ aws_access_key_id = "${AWS_ACCESS_KEY_ID}"
 aws_secret_access_key = "${AWS_SECRET_ACCESS_KEY}"
 
 ## Lake framework AWS default region
-aws_default_region = "eu-central-1"
+aws_default_region = "${AWS_DEFAULT_REGION}"
 
 ## Lake framework bucket name
-aws_bucket_name = "near-lake-data-mainnet"
+aws_bucket_name = "${AWS_BUCKET_NAME}"
 
+## Transaction details are stored in the Google Cloud Storage
 [tx_details_storage]
-## Transaction details are stored in the S3-compatibe object storage (Google Cloud Storage by default)
-# Storage Bucket Name
-bucket_name = "readrpc-tx-details"
+
+## Storage Bucket Name
+bucket_name = "${TX_BUCKET_NAME}"
 
 ## Database configuration
 [database]
 
 ## Database connection string
 ## You can use database connection URL
 ## postgresql://{user}:{password}@localhost:5432/{db_name}
-database_url = "postgresql://postgres:password@localhost:5432/near-indexer"
+database_url = "${META_DATABASE_URL}"
 
 ## Database max connections
 ## Default value is 10
@@ -154,7 +160,7 @@ database_url = "postgresql://postgres:password@localhost:5432/near-indexer"
 
 ## This parameter not needed to encrise for indexers
 ## 10 connections is enough for indexers to save changes to the database
-max_connections = 10
+max_connections = "${MAX_CONNECTIONS}"
 
 ## Database shards
 ## You can use multiple database shards
@@ -185,3 +191,4 @@ database_url = "${SHARD_4_DATABASE_URL}"
 [[database.shards]]
 shard_id = 5
 database_url = "${SHARD_5_DATABASE_URL}"
+"#;