Fix tests.

config.py

@@ -23,13 +23,15 @@ class Settings(BaseSettings):
     """
 
     model_config = SettingsConfigDict(
-        env_file=".env", env_file_encoding="utf-8", case_sensitive=True
+        env_file=".env", env_file_encoding="utf-8", case_sensitive=True,extra='ignore', 
+        missing = 'ignore'
+
     )
 
-    DATABASE_URL: PostgresDsn
+    DATABASE_URL: PostgresDsn = Field(default="postgresql://postgres:password@localhost:5432/ansari")
     MAX_THREAD_NAME_LENGTH: int = Field(default=100)
 
-    SECRET_KEY: SecretStr
+    SECRET_KEY: SecretStr =  Field(default="secret")
     # Literal ensures the allowed value(s), and frozen ensures it can't be changed after initialization
     ALGORITHM: Literal["HS256"] = Field(default="HS256", frozen=True)
     ENCODING: Literal["utf-8"] = Field(default="utf-8", frozen=True)
@@ -42,7 +44,7 @@ class Settings(BaseSettings):
     API_SERVER_PORT: int = Field(default=8000)
 
     OPENAI_API_KEY: SecretStr
-    PGPASSWORD: SecretStr
+    PGPASSWORD: SecretStr = Field(default="password")
     KALEMAT_API_KEY: SecretStr
     VECTARA_AUTH_TOKEN: SecretStr
     VECTARA_CUSTOMER_ID: str
@@ -54,7 +56,7 @@ class Settings(BaseSettings):
     template_dir: DirectoryPath = Field(default="resources/templates")
     diskcache_dir: str = Field(default="diskcache_dir")
 
-    MODEL: str = Field(default="gpt-4o-2024-05-13")
+    MODEL: str = Field(default="gpt-4o")
     MAX_FUNCTION_TRIES: int = Field(default=3)
     MAX_FAILURES: int = Field(default=1)
     SYSTEM_PROMPT_FILE_NAME: str = Field(default="system_msg_fn")

tests/ask-question-en.txt

@@ -0,0 +1,11 @@
+Select the best answer to the following question: 
+
+{{ question }}
+
+from the list of options (one option per line)
+
+{% for option in options %}{{ option }}
+{% endfor %}
+
+Output your answer in json format, with a single field, "answer" 
+and the answer to the above question. 

tests/batik-v1-en.csv

@@ -0,0 +1,101 @@
+category,question,options,correct
+Tajweed,Which of the following letters is not one of the letters of ithhaar? ,"hamza, haa, ayn, qaaf, khaa",qaaf
+Tajweed,How many harakahs of madd if a long vowel is followed by a letter with a shaddah? ,"2, 3, 4, 6",6
+Tajweed,Which of the following letters is not one of the letters of qalqalah?,"qaaf, baa, thaa, jeem",thaa
+Tajweed,Which of the following letters is not one of the letters of idghaam after noon? ,"yaa, meem, laam, faa",faa
+Fiqh,What is the status of men wearing gold in Islam if there's no reason? ,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Fiqh,What is the status of men wearing silk in Islam if there's no reason? ,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Modern Issues,What is Islam's stance on homosexual acts?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Modern Issues,What is islam's stance on premarital sex?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Modern Issues,What is Islam's stance on non-spot transactions exchanging gold and silver?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Fiqh,Which madhab delays the start of Asr prayer compared to others?,"Hanafi, Shafi, Hanbali, Maliki",Hanafi
+Theology,What type of creation is Iblees or Satan?,"Human, Angel, Jinn, other",Jinn
+Islamic History,What year was Muhammad born?,"570 BCE, 570 CE, 203 CE, 1492 CE",570 CE
+Islamic History,What is the name of the year that was given to the year Muhammad was born?,"Year of the Horse, Year of the Elephant, Year of the Tiger, Year of the Snake",Year of the Elephant
+Islamic History,What was the name of Muhammad's first wife?,"Khadijah, Aisha, Sawda, Zainab",Khadijah
+Fiqh,Which prayer has an odd number of Rakahs?,"Fajr, Thuhr, Asr, Maghrib, Isha",Maghrib
+Fiqh,"A man without parents, spouse but with 2 boys and 3 girls dies. He has $70,000 of assets in his will. How much does each girl get?","Nothing, $5000, $10000, $14000, $20000",$10000
+Theology,Which of the following is not part of the Tahawi creed? ,"Tawheed al Uloohiyyah, Tawheed al Ruboobiyah, Tawheed al Qadar, Tawheed Al Asmaa was Sifaat",Tawheed al Qadar
+Qur'an,Which of the following is Ayat Ul Kursi?,"2:255, 2:286, 2:202, 2:153",2:255
+Qur'an,How many Surahs are there in the 29th chapter of the Qur'an,"8,10,11,15",11
+Qur'an,How many verses in the standardized version of the Qur'an,"5555,5876, 6236, 6666",6236
+Qur'an,Which of the following months is not one of the Sacred months,"Thul Qa'dah, Thul Hijjah, Muharram, Safar",Safar
+Qur'an,What is the longest verse of the Qur'an?,"2:282, 45:34, 55:38, 3:14",2:282
+Qur'an,What is the verse known as the Ayah of Debt?,"2:282, 45:34, 55:38, 3:14",2:282
+Qur'an,How many prophets are mentioned in the Qur'an?,"5,10,15,25,40",25
+Qur'an,Who was Yusuf's great grandfather according to the Qur'an?,"Ishmael, Isaac, Jacob, Abraham",Abraham
+Qur'an,What is the trilietaeral root of اتستبدلون ? ,"ta-ba-la, da-la-na, ba-da-la, ta-sa-ta",ba-da-la
+Islamic History,What was Madinah known by before the Prophet renamed it?,"Makkah, Taa'if, Yathrib, Yamama",Yathrib
+Islamic History,What was the name of the tribe Muhammad was from? ,"Quraysh, Thaqeef, Daws, Bani Kalb",Quraysh
+Islamic History,What was the name of the tribe that was based in Taa'if in the time of Muhammad? ,"Quraysh, Thaqeef, Daws, Bani Kalb",Thaqeef
+Islamic History,What was the name of the first person who was killed for his/her beliefs?,"Ammar bin Yaasir, Sumayyah bint Khayyaat, Bilal Ibn Rabah, Yasir bin Ameer",Sumayyah bint Khayyaat
+Qur'an,"According to the Qur'an, what is the name of the angel guarding hell?","Jibreel, Maalik, Meekaal, Azraaeel",Maalik
+Islamic History,What year was the conquest of Makkah?,"5H, 6H, 8H, 10H",8H
+Qur'an,What ayah of the Qur'an permits marriage to up to 4 wives?,"4:3, 4:133, 16:125, 2:255",4:3
+Qur'an,Which surah of the Qur'an tells the story of Musa and Khidr?,"The Cave, The Prophets, Musa, Cattle",The Cave
+Hadith,What does the first hadith of Imam Nawawi's 40 hadith cover?,"Pillars of Islam, Intention, The Day of Judgement, Perfection",Intention
+Qur'an,Is there an Ayah that banee israeel will cause corruption on the earth twice? ,"17:4, 14:7, 41:7, 4:17",17:4
+Qur'an,What is the verse that talks about if the seas were ink and the trees were pens and even if there was seven times as much it would not be enough?,"31:27, 17:4, 16:125",31:27
+Qur'an,What's the verse that says kullu nafsin thaaiqatul mawt?,"17:4,6:23, 55:22, 3:185",3:185
+Islamic History,What year did Yusuf al Qaradawi die? ,"He's still alive, 2018,2022, 2000",2022
+Islamic History,Who was the founder of the Muslim Brotherhood?,"Muhammad Kishk, Ali Tantawi, Hassan Al Banna, Sayyid Qutb",Hassan Al Banna
+Islamic History,Whose khilaafah was the longest? ,"Abu Bakr, Umar, Uthmaan, Ali",Uthmaan
+Islamic History,Which caliphate made Egypt the center of the caliphate?,"Fatimid, Abbasid, Umayyad, Ottoman",Fatimid
+Qur'an,"Name of surah whose name means ""staircase""?","Maarij, Salaalim, Madaarij, Taariq",Maarij
+Islamic History,Which was the treaty where Muhammad SAW asked Ali (RA) to rub rasool ullah from the contract?,"Hudaybiyah, Aqaba I, Aqaba II, Shajara",Hudaybiyah
+Islamic History,The Prophet took an oath to avenge the death of a Sahabi under the tree. Which Sahabi was it,"Ali ben Abi Taalib, Abu Bakr, Uthmaan, Sufyan",Uthmaan
+Qur'an,How many surahs in the Qur'an?,"86, 114, 113, 127",114
+Qur'an,Which of these tafseers are the most recent?,"Sa'di, Ibn Kathir, Qurtubi, Razi, ",Sa'di
+Modern Issues,Are tattoos allowed in Islam?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Modern Issues,Is pornography permitted in Islam?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Fiqh,What is the status of Salaat in Islam?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Obligatory
+Fiqh,What is the status of Zakat in Islam? ,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Obligatory
+Fiqh,What is Islam's position on gambling?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Fiqh,What is Islam's position on alcohol?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Fiqh,What is the status of nafl prayers before Thuhr? ,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Encouraged
+Hadith,Which is the highest level of hadith?,"Mutawaatir, Ahad, Hassan, Daeef",Mutawaatir
+Hadith,"In the hadith that talks about the scholars being inheritors of the prophets, what does it say the Prophets don't leave behind? ","Gold and silver, dinar and dirham, children, property",dinar and dirham
+Islamic History,Which of the follow tribes was not one of the Jewish tribes of Madinah?,"Banu Nadir, Banu Qaynuqa, Banu Qurayda, Banu Qaloon",Banu Qaloon
+Islamic History,What is the name of the tribe that came across Haajar and Ismail?,"Jurhum, Jinaan, Kinaanah, Hishaam",Jurhum
+Hadith,Who narrates the hadith about scholars being inheritors of Prophets?,"Abu Darda, Abu Tharr, Abu Quhaafah, Anas Ibn Maalik",Abu Darda
+Islamic History,Who took care of the Prophet (SAWS) immediately after his mother passed away?,"Abdul Muttalib, Abu Taalib, Hamzah, Abbas",Abdul Muttalib
+Islamic History,What is the name of Khadijah's sister that would meet the Prophet (SAWS) sometimes?,"Also Khadija, Hala, Ruqayyah, Maymuna",Hala
+Islamic History,Waht year did Salahuddin defeat the Europeans in the crusades?,"1187 CE, 1099 CE, 1492 CE, 912 CE",1187 CE
+Islamic History,Which Islamic scholar were logarithms named after?,"Al-Logharithmi, Al-Khawarizmi, Al-Gebri, Al-Farabi",Al-Khawarizmi
+Islamic History,Which Sahabi was entrusted with compiling the Qur'an after the Prophet's death?,"Zaid Ibn Harithah, Zayd Ibn Thabit, Thabit Ibn Qays, Abu Ayyub al Ansari",Zayd Ibn Thabit
+Qur'an,Who is the only Sahabi mentioned by name in the Qur'an?,"Abu Bakr, Umar, Zaid Ibn Harithah, Zaid Ibn Thabit",Zaid Ibn Harithah
+Qur'an,What does Allah call the Prophet (SAWS) for the Isra and Miraj?,"Prophet, Muhammad, Abd, Sayyid",Abd
+Qur'an,Is Abu Bakr mentioned in the Qur'an?,"Yes, No, indirectly as thaani-Ithnayn, yes by his name Abdullah",Indirectly as thaani-ithnayn
+Islamic History,Who is known as thaat an nitaaqayn (The one with two belts)? ,"Aisha bint Abi Bakr, Asma bint Abi Bakr, Um Salamah, Khadijah",Asma bint Abi Bakr
+Islamic History,Which wife of the Prophet (SAWS) advised him to shave his head at hudaybiyah and the people would follow?,"Aisha bint Abi Bakr, Asma bint Abi Bakr, Um Salamah, Zainab bin Jahsh",Um Salamah
+Theology,Do Muslims believe in original sin? ,"Yes since everybody is inclined to sin, No Muslims believe in the concept of fitra, ",No Muslims believe in the concept of fitra
+Qur'an,Which of these was one of the idols that the people of Noah said they should not abandon?,"Laat, Uzza, Hubal, Wadd",Wadd
+Qur'an,What is the verse of the Qur'an that lists the idols that the people of Noah said not to abandon?,"71:11, 71:23, 72:11, 72:23",71:23
+Islamic History,What is the first month of the Islamic calendar as currently understood?,"Ramadan, Muharram, Thul Hijjah, Safar",Muharram
+Islamic History,In whose reign was the current Islamic Calendar created?,"Abu Bakr, Umar, Uthmaan, Ali",Umar
+Islamic History,"Who is the sahabi that made the dua O Allah, grant me shahadah in Your path, and allow my death to be in the city of Your Messenger?","Abu Bakr, Umar, Ubaydah Ibn Al Jarrah, Abu Tharr al Ghifaari",Umar
+Islamic History,"When Jibreel would come to Muhammad in human form, which sahabi did he resemble?","Umar, Abu Bakr, Huthayfah Ibn al Yaman, Dihyah Al Kalbi",Dihyah Al Kalbi
+Hadith,"According to the hadith Qudsi that starts man aadaa lee waliyyan, Which of the following is not a characteristic of the awliyaa of Allah","hearing with which he hears, sight with which he sees, leg with which he walks, mind with which he thinks",mind with which he thinks
+Qur'an,Which of the following is not one of the 3 categories of people on the day of judgement mentioned in Surah al Waqi'ah?,"Assabiqoon as saabiqoon, ashaab al-maimanah, ashaab al-Mash'amah, Ashaab al-Jaheem",Ashaab al-jaheem
+Qur'an,Who made the first translation of the Qur'an into a European Language,"Robert of Ketton, Geoge Sale, Abdullah Yusuf Ali, Mohammad Marmaduke Pickthall",Robert of Ketton
+Modern Issues,What is Islam's stance on IVF treatment?,"Never permissible, Always permissible, permissible if and only if the sperm comes from the husband and the egg comes from the wife",permissible if and only if the sperm comes from the husband and the egg comes from the wife
+Hadith,"In the hadith about three stages of embryonic development, which is not one of the stages mentioned","nutfah (drop), alaqah (clot), mudghah (morsel of flesh), ithaam (bones)",ithaam (bones)
+Fiqh,Which of the following is not one of the 8 categories eligible for Zakat,"freeing captives, zakat administrators, those in debt, building mosques",building mosques
+Fiqh,Which of the following is not one of the 3 types of hajj,"ifraad, qiran, tamattu, matmum",matmum
+Fiqh,Which is the most preferred form of hajj,"ifraad, qiran, tamattu, matmum",tamattu
+Hadith,"According to hadith, how many doors does Jannah have","1,3,7,8",8
+Hadith,"According to hadith, what is required to enter through the door of Raiyan","praying, fasting, paying zakat, making thikr",fasting
+Qur'an,"According to the Qur'an, how many doors does hell have","1,3,7,8",7
+Qur'an,Which verse of the Qur'an tells us how many doors hell has,"15:44, 44:15, 4:15, 5:14",15:44
+Qur'an,According to the Qur'an how many angels are guardians over hell?,"1,3,7,19",19
+Qur'an,Which verse of the Qur'an tells us how many angels are guardians over hell?,"74:30, 34:70,30:74,47:3",74:30
+Qur'an,Which of the following is not one of the women discussed at the end of S Tahrim,"the wife of Pharaoh, Mary, the wife of Lot, the wife of Noah, the wife of Abraham ",The wife of Abraham
+Islamic History,What is the name of the cousin of Khadijah that was knowledgeable about scripture? ,"Waraqah ibn Nawfal, Khalid bin Khuwaylid, Thabit bin Qays, Ayyub ibn Hakeen",Waraqah ibn Nawfal
+Modern Issues,What is Islam's position on cross dressing?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Modern Issues,What is Islam's position on sculptures of living things?,"Prohibited, Disliked, Permitted, Encouraged, Obligatory",Prohibited
+Theology,Who were the first major group to break away from mainstream islam?,"Khawarij, Mutazilis, Shia, Jahmiyyah",Khawarij
+Theology,"In understanding evil, which of the following is not a form of Allah's will","Iraadah Shar'iyyah, Iraadah Kawniyyah, Iraadah Nafsiyyah",Iraadah Nafsiyyah
+Fiqh,Are women with menses supposed to go to Eid prayer,"Yes, No, Yes but they do not participate in the prayer itself",Yes but they do not participate in the prayer itself
+Islamic History,What is the name of the person who assassinated Umar?,"Dujanah Al Shami, Abu Luluah Al Majoosi, Layth al Saadi, Yamam bin Qays",Abu Luluah Al Majoosi
+Qur'an,What is the verse that says Laa yamassahu illal mutahharoon?,"56:79, 59:76,57:69,5:69",56:79 
+Qur'an ,Which is the verse that says that corruption has spread through the land as a result of people's actions?,"31:40, 3:140, 30:41, 30:14",30:41

tests/test_answer_quality.py

@@ -24,7 +24,7 @@
 def data():
     tenv = Environment(loader=FileSystemLoader("resources/templates/"))
     q_temp = tenv.get_template("ask_question.txt")
-    df = pd.read_csv("evals/batik/batik-v1.csv")
+    df = pd.read_csv("tests/batik-v1-en.csv")
     cache = {}
     return df, q_temp, cache