Merge pull request #99 from OdyAsh/DB-and-API-Refactors

DB and API Refactors

.env.example

@@ -11,7 +11,7 @@ export DATABASE_URL="postgresql://user:password@localhost:5432/database_name"
 export SECRET_KEY="secret"         # Secret key for signing tokens
 
 # Origins to be allowed by the backend
-export ORIGINS="https://beta.ansari.chat,http://beta.ansari.chat,https://ansari.chat,http://ansari.chat,https://hajiansari.ai,http://hajiansari.ai,https://ansari.endeavorpal.com"
+export ORIGINS="https://beta.ansari.chat,http://beta.ansari.chat,https://ansari.chat,http://ansari.chat,https://hajiansari.ai,http://hajiansari.ai,https://ansari.endeavorpal.com,https://web.whatsapp.com"
 
 # Vectara search engine configuration
 export PGPASSWORD=""                # Password for PostgreSQL database
@@ -25,15 +25,41 @@ export template_dir="."             # Directory path for templates
 
 # Related to WhatsApp Business and Meta (leave empty if you're not planning to use WhatsApp)
 # Source 1: https://www.youtube.com/watch?v=KP6_BUw3i0U
+#   Watch Until 32:25
 # Source 2: https://glitch.com/edit/#!/insidious-tartan-alvarezsaurus
 # Source 3: https://developers.facebook.com/blog/post/2022/10/24/sending-messages-with-whatsapp-in-your-python-applications/#u_0_39_8q
+
+# Moreover, if want to test whatsapp's webhook locally, you can use zrok on a reserved URL with a zrok "share token" 
+# obtained by contacting its current holder: https://github.com/OdyAsh (source 1, 2 below)
+# Alternatively, you can change the webhook url all together (source 3, 4 below)
+# Check these sources for more details: 
+# Source 1: https://dev.to/odyash/quickly-share-your-app-with-zrok-4ihp
+# Source 2: https://openziti.discourse.group/t/how-do-i-use-a-reserved-share-on-different-devices/2379/2
+# Source 3: https://youtu.be/KP6_BUw3i0U?t=1294 
+#   (@21:33 and 25:30, however they use glitch instead of zrok, so you'll just need to change the webhook url to your zrok url)
+# Source 4 (where you can change callback url, given that your facebook account gets access by the app's admins): 
+#   https://developers.facebook.com/apps/871020755148175/whatsapp-business/wa-settings/
+#   Note 1: Obviously, that `871...175` is the testing app's public id, so if this link still doesn't work even after you gain access, 
+#   then the admins most probably created a new test app instance
+#   Note 2: If an unexpected 3rd party discovers the ZROK_SHARE_TOKEN, 
+#   a new one will have to be generated, then added to Meta's callback URL of the *testing* app
+#   (Noting that the *production* app's callback URL will be different anyway, so the 3rd party won't be able to access that app)
+#   (but we still don't want random calls to be made to our testing app, so that's why we'll still have to change an exposed token :])
+
 export WHATSAPP_RECIPIENT_WAID="<<YOUR-RECIPIENT-TEST-PHONE-NUMBER>>"
 export WHATSAPP_API_VERSION="<<CURRENT-VERSION-AS-MENTIONED-IN-SOURCE-URL-ABOVE>>" 
 export WHATSAPP_BUSINESS_PHONE_NUMBER_ID="<<YOUR-WHATSAPP-BUSINESS-PHONE-NUMBER-ID>>"
 export WHATSAPP_ACCESS_TOKEN_FROM_SYS_USER="<<YOUR-SYSTEM-USER-ACCESS-TOKEN>"
-export WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK="<<YOUR-VERIFIFY-TOKEN-TO-VERIFY-WHATSAPP-WEBHOOK>>"
+export WHATSAPP_VERIFY_TOKEN_FOR_WEBHOOK="<<The-VERIFIFY-TOKEN-CURRENTLY-USED-TO-VERIFY-META'S-CALLBACK-URL>>"
+export ZROK_SHARE_TOKEN="<<THE-ZROK-SHARE-TOKEN-CURRENTLY-USED-IN-META'S-CALLBACK-URL>>"
 
 # Related to internal code logic
 # Leave the values below when locally debugging the application
+# In production, don't add them to environment variables, or add them as "INFO"/"False" respectively
 export LOGGING_LEVEL="DEBUG"
-export DEBUG_MODE="True"
+export DEBUG_MODE="True"
+
+# To get rid of .py[cod] files (This should key should NOT be set in production!)
+# This is only to de-clutter your local development environment
+# Details: https://docs.python-guide.org/writing/gotchas/#disabling-bytecode-pyc-files
+PYTHONDONTWRITEBYTECODE=1

data/mawsuah/strip_tashkeel.py

@@ -6,7 +6,7 @@
 
 from ansari.ansari_logger import get_logger
 
-logger = get_logger(__name__)
+logger = get_logger()
 
 
 def strip_tashkeel_from_doc(input_file, output_file):

docs/structure_of_api_responses/ansari_website_api_structure_of_a_request_sent_using_zrok.json

@@ -0,0 +1,76 @@
+{
+	"scope": {
+		"type": "http",
+		"asgi": {
+			"version": "3.0",
+			"spec_version": "2.4"
+		},
+		"http_version": "1.1",
+		"server": ["127.0.0.1", 8000], // When running locally
+		"client": ["127.0.0.1", 11563], // The port here changes dynamically
+		"scheme": "http",
+		"method": "POST",
+		"root_path": "",
+		"path": "/api/v2/users/login",
+		"raw_path": "/api/v2/users/login",
+		"query_string": "",
+		"headers": [
+			["host", "localhost:8000"],
+			["connection", "keep-alive"],
+			["content-length", "83"],
+			["sec-ch-ua-platform", "\"Windows\""],
+			["user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"],
+			["x-mobile-ansari", "ANSARI"],
+			["sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\""],
+			["content-type", "application/json"],
+			["sec-ch-ua-mobile", "?0"],
+			["accept", "*/*"],
+			["origin", "http://localhost:3000"],
+			["sec-fetch-site", "same-site"],
+			["sec-fetch-mode", "cors"],
+			["sec-fetch-dest", "empty"],
+			["referer", "http://localhost:3000/"],
+			["accept-encoding", "gzip, deflate, br, zstd"],
+			["accept-language", "en-GB,en;q=0.9,ar-EG;q=0.8,ar;q=0.7,en-US;q=0.6"]
+		],
+		"state": {},
+		"app": "<fastapi.applications.FastAPI object>",
+		"starlette.exception_handlers": {
+			"<class 'starlette.exceptions.HTTPException'>": "<function http_exception_handler>",
+			"<class 'starlette.exceptions.WebSocketException'>": "<bound method ExceptionMiddleware.websocket_exception>",
+			"<class 'fastapi.exceptions.RequestValidationError'>": "<function request_validation_exception_handler>",
+			"<class 'fastapi.exceptions.WebSocketRequestValidationError'>": "<function websocket_request_validation_exception_handler>"
+		},
+		"router": "<fastapi.routing.APIRouter object>",
+		"endpoint": "<function login_user>",
+		"path_params": {},
+		"route": {
+			"path": "/api/v2/users/login",
+			"name": "login_user",
+			"methods": ["POST"]
+		}
+	},
+	"_receive": "<bound method RequestResponseCycle.receive>",
+	"_send": "<function wrap_app_handling_exceptions.<locals>.wrapped_app.<locals>.sender>",
+	"_stream_consumed": true,
+	"_is_disconnected": false,
+	"_form": null,
+	// _body's value (and other strings) were actually binary strings (i.e., start with b'...')
+	"_body": "{\"email\":\"guest_<<AUTO_GENERATED_KEY>>@endeavorpal.com\",\"password\":\"<<PASSWORD_SENT_VIA_ANSARI_WEBSITE>>\",\"guest\":true}",
+    // this is what actually gets returned when accessing headers property (e.g., `request.headers`)
+    // Check Starlette's implementation (which FastAPI uses) for details:
+    // https://github.com/encode/starlette/blob/b68a142a356ede730083347f254e1eae8b5c803e/starlette/requests.py#L12
+	"_headers": {
+		"host": "localhost:8000",
+		"connection": "...",
+        "...": "..."
+        // I.e., the value of the `_headers` key is a dictionary of the headers already mentioned above
+	},
+	"_json": {
+		"email": "guest_<<AUTO_GENERATED_KEY>>@endeavorpal.com",
+        "...": ["..."] 
+        // I.e.,  the value of the `_json` key is simply the dictionary equivalent of `_body`'s string value
+	},
+	"_query_params": "",
+	"_cookies": {}
+}

docs/structure_of_api_responses/meta_whatsapp_api_structure_of_a_request_sent_using_zrok.json

@@ -0,0 +1,73 @@
+{
+    "scope": {
+        "type": "http",
+        "asgi": {
+            "version": "3.0",
+            "spec_version": "2.4"
+        },
+        "http_version": "1.1",
+        "server": ["127.0.0.1", 8000], // When running locally
+        "client": ["<<WHATSAPP.USER.IP>>", 0],
+        "scheme": "https",
+        "method": "POST",
+        "root_path": "",
+        "path": "/whatsapp/v1",
+        "raw_path": "/whatsapp/v1",
+        "query_string": "",
+        "headers": [
+            ["host", "YOUR_ZROK_SHARE_TOKEN.share.zrok.io"],
+            ["user-agent", "facebookexternalua"],
+            ["content-length", "545"],
+            ["accept", "*/*"],
+            ["accept-encoding", "deflate, gzip"],
+            ["content-type", "application/json"],
+            ["x-amzn-trace-id", "Root=1-674b2035-0f0a8ab27075asce3324dcdb"], // trace value here is fake
+            ["x-forwarded-for", "173.REST.OF.IP, <<WHATSAPP.USER.IP>>"],
+            ["x-forwarded-port", "443"],
+            ["x-forwarded-proto", "https"],
+            ["x-hub-signature", "sha1=8a3e35da6fb5dfaaf5aaa46c8d059d519e18112d"], // sha1 hash here is fake
+            ["x-hub-signature-256", "sha256=51d62480d40ffd0f48d1cde1ea47656452fd65b5ac29077fe3c6b4e68d74c827"], // sha256 here is fake
+            ["x-proxy", "zrok"]
+        ],
+        "state": {},
+        "app": "<FastAPI object>",
+        "starlette.exception_handlers": {
+            "<class 'starlette.exceptions.HTTPException'>": "<function http_exception_handler>",
+            "<class 'starlette.exceptions.WebSocketException'>": "<bound method ExceptionMiddleware.websocket_exception>",
+            "<class 'fastapi.exceptions.RequestValidationError'>": "<function request_validation_exception_handler>",
+            "<class 'fastapi.exceptions.WebSocketRequestValidationError'>": "<function websocket_request_validation_exception_handler>"
+        },
+        "router": "<APIRouter object>",
+        "endpoint": "<function main_webhook>",
+        "path_params": {},
+        "route": {
+            "path": "/whatsapp/v1",
+            "name": "main_webhook",
+            "methods": ["POST"]
+        }
+    },
+    "_receive": "<bound method RequestResponseCycle.receive>",
+    "_send": "<function wrap_app_handling_exceptions.<locals>.wrapped_app.<locals>.sender>",
+    "_stream_consumed": true,
+    "_is_disconnected": false,
+    "_form": null,
+    "_query_params": "",
+    // this is what actually gets returned when accessing headers property (e.g., `request.headers`)
+    // Check Starlette's implementation (which FastAPI uses) for details:
+    // https://github.com/encode/starlette/blob/b68a142a356ede730083347f254e1eae8b5c803e/starlette/requests.py#L125
+    "_headers": {
+        "host": "...",
+        "user-agent": "...",
+        "...": "..."
+        // I.e., the value of the `_headers` key is a dictionary of the headers already mentioned above
+    },
+    "_cookies": {},
+    // _body's value (and other strings) were actually binary strings (i.e., start with b'...')
+    // Also, it contains content mentioned in other `meta_whatsapp_*.json` files
+    "_body": "{\"object\":\"whatsapp_business_account\", ...}",
+    "_json": {
+        "object": "whatsapp_business_account",
+        "...": ["..."] 
+        // I.e.,  the value of the `_json` key is simply the dictionary equivalent of `_body`'s string value
+    }
+}

docs/structure_of_api_responses/meta_whatsapp_structure_of_a_user_incoming_msg.json → docs/structure_of_api_responses/meta_whatsapp_api_structure_of_a_user_incoming_msg.json

@@ -9,7 +9,7 @@
                         "messaging_product": "whatsapp",
                         "metadata": {
                             "display_phone_number": "<<the WHATSAPP_BUSINESS_PHONE_NUMBER without non-numeric characters (e.g., +1 (555) 555-5555 -> 15555555555)>>",
-                            "phone_number_id": "<<the WHATSAPP_BUSINESS_PHONE_NUMBER_ID (this ID is a numeric value)>>"
+                            "phone_number_id": "<<the WHATSAPP_BUSINESS_PHONE_NUMBER_ID (this ID is a numeric value generated by Meta)>>"
                         },
                         "contacts": [
                             {

requirements.txt

@@ -8,6 +8,7 @@ jinja2
 # should be >= 2 in order for "from langfuse.decorators" to work
 langfuse>=2.0.0
 litellm
+loguru
 openai
 pandas
 psycopg2-binary

setup_database.py

@@ -5,7 +5,7 @@
 from ansari.ansari_logger import get_logger
 from ansari.config import get_settings
 
-logger = get_logger(__name__)
+logger = get_logger()
 
 
 def import_sql_files(directory, db_url):

sql/01_create_tables.sql

@@ -1,15 +1,16 @@
 CREATE TABLE users (
     id SERIAL PRIMARY KEY,
-    email VARCHAR(100) UNIQUE, -- can be null if it is a guest account
-    password_hash VARCHAR(255), -- can be null if it is a guest account
-    first_name VARCHAR(50), -- can be null if it is a guest account
-    last_name VARCHAR(50), -- can be null if it is a guest account
+    email VARCHAR(100) UNIQUE, -- Can be null if it is a guest account
+    password_hash VARCHAR(255), -- Can be null if it is a guest account
+    first_name VARCHAR(50), -- Can be null if it is a guest account
+    last_name VARCHAR(50), -- Can be null if it is a guest account
     preferred_language VARCHAR(10) DEFAULT 'en',
     is_guest BOOLEAN NOT NULL DEFAULT FALSE, 
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
 
+
 CREATE TABLE preferences (
     id SERIAL PRIMARY KEY,
     user_id INTEGER NOT NULL,
@@ -35,7 +36,7 @@ CREATE TABLE messages (
     user_id INTEGER NOT NULL,
     thread_id INTEGER NOT NULL,
     role TEXT NOT NULL, 
-    -- TODO (odyash): check if "function" can be renamed to "tool" like the rest of the codebase or not
+    -- #TODO (odyash): check if "function" can be renamed to "tool" like the rest of the codebase or not
     function_name TEXT, 
     content TEXT NOT NULL,
     timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -44,5 +45,3 @@ CREATE TABLE messages (
     FOREIGN KEY (user_id) REFERENCES users(id),
     FOREIGN KEY (thread_id) REFERENCES threads(id)
 );
-
-

src/ansari/agents/ansari.py

@@ -14,7 +14,8 @@
 from ansari.tools.search_vectara import SearchVectara
 from ansari.util.prompt_mgr import PromptMgr
 
-logger = get_logger(__name__ + ".Ansari")
+# previous logger name: __name__ + ".Ansari"
+logger = get_logger()
 
 
 class Ansari:

src/ansari/agents/ansari_workflow.py

@@ -17,7 +17,8 @@
 else:
     logging_level = logging.INFO
 
-logger = get_logger(__name__ + ".AnsariWorkflow", logging_level)
+# previous logger name: __name__ + ".AnsariWorkflow"
+logger = get_logger(logging_level)
 
 
 class AnsariWorkflow: