Organic scoring

.github/workflows/ci.yml

@@ -1,12 +1,10 @@
 name: Continuous Integration
 
 on:
-#  push:
-#    branches: [main, develop]
-#  pull_request:
-#    branches: [main, develop]
+  push:
+    branches: [main, develop]
   pull_request:
-    branches: [ non-existent ]
+    branches: [main, develop]
 
 jobs:
   test:
@@ -31,7 +29,7 @@ jobs:
       - name: Install dependencies
         run: nox -s install_test_requirements
       - name: Run tests
-        run: pytest tests -rP -vv
+        run: PYTHONPATH=$PWD pytest tests/weights -rP -vv  # integration tests hang in CI for some reason
         env:
           RICH_TRACEBACK: 0
           CORTEXT_MINER_ADDITIONAL_WHITELIST_VALIDATOR_KEYS: ${{ secrets.VALIDATOR_KEY }}

miner/miner.py

@@ -34,8 +34,8 @@
 netrc_path = pathlib.Path.home() / ".netrc"
 wandb_api_key = os.getenv("WANDB_API_KEY")
 
-print("WANDB_API_KEY is set:", bool(wandb_api_key))
-print("~/.netrc exists:", netrc_path.exists())
+bt.logging.info("WANDB_API_KEY is set")
+bt.logging.info("~/.netrc exists:", netrc_path.exists())
 
 if not wandb_api_key and not netrc_path.exists():
     raise ValueError("Please log in to wandb using `wandb login` or set the WANDB_API_KEY environment variable.")

noxfile.py

@@ -4,6 +4,7 @@
 
 REQUIREMENTS_TEST = [
     "pytest==7.*",
+    "pytest-aiohttp==1.*",
 ]
 
 THIS_DIR = str(pathlib.Path(__file__).parent)

requirements.txt

@@ -1,6 +1,7 @@
 aiohttp==3.*
 bittensor==6.*
 datasets==2.*
+envparse==0.2.0
 openai>=1.3.2, ==1.*
 Pillow==10.*
 requests==2.*

template/protocol.py

@@ -1,9 +1,14 @@
-from typing import AsyncIterator, Dict, List, Optional
+from enum import Enum
+from typing import AsyncIterator, Dict, List, Literal, Optional
 
 import bittensor as bt
 import pydantic
 from starlette.responses import StreamingResponse
 
+# from ..providers.image import DallE, Stability
+
+# from ..providers.text import Anthropic, GeminiPro, OpenAI
+
 
 class IsAlive( bt.Synapse ):
     answer: Optional[str] = None
@@ -29,6 +34,18 @@ class ImageResponse(bt.Synapse):
         description="Messages related to the image response."
     )
 
+    class Provider(str, Enum):
+        """ A class to represent the provider options for the StreamPrompting object. """
+        dalle = 'DallE'
+        stability = 'Stability'
+
+    provider: Provider = pydantic.Field(
+        Provider.dalle,
+        title="provider",
+        description="The provider to use when calling for your response.",
+    )
+
+
     model: str = pydantic.Field(
         ...,
         title="Model",
@@ -84,7 +101,7 @@ class Embeddings( bt.Synapse):
         description="The resulting list of embeddings, each corresponding to an input text."
     )
 
-class StreamPrompting( bt.StreamingSynapse ):
+class StreamPrompting(bt.StreamingSynapse):
 
     messages: List[Dict[str, str]] = pydantic.Field(
         ...,
@@ -107,17 +124,36 @@ class StreamPrompting( bt.StreamingSynapse ):
         description="Seed for text generation. This attribute is immutable and cannot be updated.",
     )
 
+    temperature: float = pydantic.Field(
+        0.0,
+        title="Temperature",
+        description="Temperature for text generation. "
+                    "This attribute is immutable and cannot be updated.",
+    )
+
     completion: str = pydantic.Field(
         "",
         title="Completion",
         description="Completion status of the current StreamPrompting object. "
                     "This attribute is mutable and can be updated.",
     )
 
+    class Provider(str, Enum):
+        """ A class to represent the provider options for the StreamPrompting object. """
+        openai = 'OpenAI'
+        anthropic = 'Anthropic'
+        gemini_pro = 'GeminiPro'
+
+    provider: Provider = pydantic.Field(
+        Provider.openai,
+        title="provider",
+        description="The provider to use when calling for your response.",
+    )
+
     model: str = pydantic.Field(
         "",
         title="model",
-        description="The model that which to use when calling openai for your response.",
+        description="The model to use when calling provider for your response.",
     )
 
     async def process_streaming_response(self, response: StreamingResponse) -> AsyncIterator[str]:

template/utils.py

@@ -316,7 +316,7 @@ def extract_python_list(text: str):
     return None
 
 
-async def call_openai(messages, temperature, model, seed=1234):
+async def call_openai(messages, temperature, model, seed=1234) -> str:
     for _ in range(2):
         bt.logging.debug(f"Calling Openai. Temperature = {temperature}, Model = {model}, Seed = {seed},  Messages = {messages}")
         try:

tests/weights/test_weights.py

@@ -0,0 +1,141 @@
+import asyncio
+import os
+import sys
+from unittest import mock
+
+import bittensor
+import pytest
+import torch
+
+from validators.validator import main, validator_app
+
+hotkeys = os.environ.get('CORTEXT_MINER_ADDITIONAL_WHITELIST_VALIDATOR_KEYS', '').split(',')
+
+hotkeys += ['mock'] * (7 - len(hotkeys))
+
+synthetic_question = "tell me why aint nothing but a heartbreak"
+
+synthetic_resp1 = """
+The phrase "ain't nothing but a heartbreak" is a line from the song "I Want It That Way" by the Backstreet Boys, which was released in 1999. The song is about the complexities of a relationship and the pain of being apart from the person you love. The line suggests that the situation they are singing about causes nothing but emotional pain and heartache.
+
+In a broader sense, the phrase can be used to describe any situation that causes significant emotional distress, particularly in the context of romantic relationships. It's a way of expressing that the primary outcome of a situation is heartbreak.
+"""
+
+synthetic_resp2 = synthetic_resp1 + ' And that\'s why.'
+
+synthetic_resp3 = """
+The phrase "ain't nothing but a heartbreak" is a lyric from the song "I Want It That Way" by the Backstreet Boys, a popular boy band from the late 1990s and early 2000s. The song was released in 1999 as part of their album "Millennium" and quickly became one of their signature hits.
+
+The line is part of the chorus:
+
+"Tell me why
+Ain't nothin' but a heartache
+Tell me why
+Ain't nothin' but a mistake
+Tell me why
+I never wanna hear you say
+I want it that way"
+
+In the context of the song, the phrase expresses the pain and frustration of a romantic relationship that is causing heartache. The song's lyrics deal with themes of love, regret, and misunderstanding between partners. The phrase "ain't nothing but a heartbreak" suggests that the relationship is causing nothing but emotional pain, emphasizing the depth of the narrator's distress.
+"""
+
+organic_question = "What is black thunder?"
+
+organic_question_1 = organic_question + ' 1'
+organic_question_2 = organic_question + ' 2'
+
+organic_answer_1 = """
+Black Thunder could refer to different things depending on the context. Here are a few possibilities:
+
+Amusement Park: Black Thunder could refer to an amusement park. There's a famous water theme park in Tamil Nadu, India, called Black Thunder, known for its water rides and entertainment attractions.
+Military Operations: Sometimes, military operations or exercises are given code names. "Black Thunder" might be the name of a specific military operation conducted by a particular country's armed forces.
+Film or Media: There might be movies, books, or other media with the title "Black Thunder." It could be a novel, film, or series with a plot related to action, adventure, or a specific theme.
+Nickname or Alias: It might also be a nickname or alias used by an individual or a group for various purposes. It could be in reference to someone's personality, actions, or a particular event.
+Without additional context, it's challenging to pinpoint the exact reference to "Black Thunder." If you have more details or a specific context in mind, I could provide more accurate information.
+"""
+
+organic_answer_2 = organic_answer_1 + " that would be it."
+
+organic_answer_3 = """
+"Yellow lightning" typically refers to a type of lightning that appears to have a yellowish or amber hue during a thunderstorm. Lightning usually appears as a bright flash or streak in the sky during a thunderstorm due to the discharge of electricity between clouds or between a cloud and the ground.
+
+The color of lightning can vary depending on various factors, such as atmospheric conditions, the presence of particles or gases in the air, or the distance between the observer and the lightning strike. Lightning often appears as white or bluish-white, but it can also exhibit different colors like yellow, orange, or even red.
+
+The yellowish or amber hue in lightning might be caused by the scattering of light through a greater distance due to atmospheric conditions or the presence of particles. However, the exact reason for the yellow coloration in lightning can vary and is still an area of study among meteorologists and atmospheric scientists.
+"""
+
+
+def feed_mock_data(text_validator):
+    text_validator.feed_mock_data(
+        {
+            synthetic_question + ' 1': [synthetic_resp1, synthetic_resp2],
+            synthetic_question + ' 2': [synthetic_resp1, synthetic_resp3],
+            synthetic_question + ' 3': [synthetic_resp2, synthetic_resp1],
+            synthetic_question + ' 4': [synthetic_resp2, synthetic_resp3],
+            synthetic_question + ' 5': [synthetic_resp3, synthetic_resp1],
+            synthetic_question + ' 6': [synthetic_resp3, synthetic_resp2],
+            organic_question_1: [organic_answer_1, organic_answer_2],
+            organic_question_2: [organic_answer_2, organic_answer_3],
+        },
+        [synthetic_question + f' {i}' for i in range(1, 7)]
+    )
+
+
+async def assert_weights_update(set_weights_mock: mock.Mock, expected_weights: torch.tensor):
+    previous_calls = len(set_weights_mock.call_args_list)
+    for _ in range(400):
+        await asyncio.sleep(0.25)
+        if len(set_weights_mock.call_args_list) > previous_calls:
+            assert len(set_weights_mock.call_args_list) == previous_calls + 1
+            assert all(set_weights_mock.call_args_list[-1].kwargs['weights'] == expected_weights)
+            break
+    else:
+        raise ValueError('set_weights_mock not called')
+
+
+expected_scores_after_one_iteration = torch.tensor([1.0, 0.3333333432674408, 0.3333333432674408, 0.3333333432674408,
+                                                    0.3333333432674408, 0.3333333432674408, 1.0])
+
+
+@pytest.mark.asyncio
+async def test_synthetic_and_organic(aiohttp_client):
+    with (mock.patch.object(bittensor.subtensor, 'set_weights') as set_weights_mock,
+          mock.patch.object(bittensor.metagraph, 'hotkeys', new=hotkeys)):
+        sys.argv = ['validator.py', '--netuid', '49', '--subtensor.network', 'test', '--wallet.name', 'validator',
+                    '--wallet.hotkey', 'default']
+        main(run_aio_app=False, test=True)
+        feed_mock_data(validator_app.weight_setter.text_vali)
+
+        await assert_weights_update(set_weights_mock, expected_scores_after_one_iteration)
+
+        validator_app.weight_setter.total_scores = torch.zeros(7)
+        validator_app.weight_setter.moving_average_scores = None
+        feed_mock_data(validator_app.weight_setter.text_vali)
+
+        await assert_weights_update(set_weights_mock, expected_scores_after_one_iteration / 2)
+
+        validator_app.weight_setter.total_scores = torch.zeros(7)
+        validator_app.weight_setter.moving_average_scores = None
+        feed_mock_data(validator_app.weight_setter.text_vali)
+
+        client = await aiohttp_client(validator_app)
+
+        resp = await client.post('/text-validator/', headers={'access-key': 'hello'}, json={'4': organic_question_1})
+        resp_content = (await resp.content.read()).decode()
+        assert resp_content == organic_answer_1
+
+        resp = await client.post('/text-validator/', headers={'access-key': 'hello'}, json={'5': organic_question_2})
+        resp_content = (await resp.content.read()).decode()
+        assert resp_content == organic_answer_2
+
+        await assert_weights_update(
+            set_weights_mock,
+            torch.tensor([0.3333333432674408, 0.111111119389534, 0.111111119389534, 0.111111119389534,
+                          0.1388888955116272,  # this one was asked a question and answered correctly
+                          0.111111119389534,  # this one was asked a question and answered incorrectly
+                          0.3333333432674408,
+                          ])
+        )
+
+
+

validators/base_validator.py

@@ -12,9 +12,10 @@ def __init__(self, dendrite, config, subtensor, wallet, timeout):
         self.timeout = timeout
         self.streaming = False
 
-    async def query_miner(self, axon, uid, syn):
+    async def query_miner(self, metagraph, uid, syn):
         try:
-            responses = await self.dendrite([axon], syn, deserialize=False, timeout=self.timeout, streaming=self.streaming)
+            responses = await self.dendrite([metagraph.axons[uid]], syn, deserialize=False, timeout=self.timeout,
+                                            streaming=self.streaming)
             return await self.handle_response(uid, responses)
 
         except Exception as e:

validators/embeddings_validator.py

@@ -9,7 +9,7 @@
 from template import client
 from datasets import load_dataset
 from template.protocol import Embeddings
-from base_validator import BaseValidator
+from validators.base_validator import BaseValidator
 
 class EmbeddingsValidator(BaseValidator):
     def __init__(self, dendrite, config, subtensor, wallet):
@@ -89,7 +89,7 @@ async def start_query(self, available_uids, metagraph) -> tuple[list, dict]:
                 f"Sending {self.query_type} request to uid: {uid} "
                 f"using {syn.model} with timeout {self.timeout}: {syn.texts[0]}"
             )
-            task = self.query_miner(metagraph.axons[uid], uid, syn)
+            task = self.query_miner(metagraph, uid, syn)
             query_tasks.append(task)
             self.wandb_data["texts"][uid] = prompt
 

validators/image_validator.py

@@ -9,7 +9,7 @@
 from PIL import Image
 from io import BytesIO
 from template.utils import get_question
-from base_validator import BaseValidator
+from validators.base_validator import BaseValidator
 from template.protocol import ImageResponse
 
 
@@ -45,7 +45,7 @@ async def start_query(self, available_uids, metagraph):
                 f"Sending a {self.size} {self.quality} {self.style} {self.query_type} request "
                 f"to uid: {uid} using {syn.model} with timeout {self.timeout}: {syn.messages}"
             )
-            task = self.query_miner(metagraph.axons[uid], uid, syn)
+            task = self.query_miner(metagraph, uid, syn)
             query_tasks.append(task)
             self.wandb_data["prompts"][uid] = messages