From d7f6109bcd8ed19c22588fdc8519b8f8fdbda3ff Mon Sep 17 00:00:00 2001 From: eric0210 <77634181+hsheric0210@users.noreply.github.com> Date: Thu, 7 Apr 2022 05:08:19 +0900 Subject: [PATCH] #16 --- .vs/AutoKkutu/v17/.suo | Bin 157696 -> 160256 bytes DatabaseManager.cs | 297 +++++++++++++++++++++-------------------- 2 files changed, 153 insertions(+), 144 deletions(-) diff --git a/.vs/AutoKkutu/v17/.suo b/.vs/AutoKkutu/v17/.suo index cceffc53e0173fa99f889e28fe9864e6711fe141..2254ecea89664d03e5565a58c3751ff640b46f5c 100644 GIT binary patch delta 1040 zcma)(ZAesE7{~Ab%pG;6YmJVnJI&a*=oS_2G?leBcH=u=lwj- zInmpW=!oNRpj4S$E=4g&&LSpKsgz{~v6`so#hr7{H%8IHBKE`Yed}KGovaN3o@A}? za>Le!%EUjmAAlfVWnY4cM9h97L$%8A63rjTr9y5ZkFbpSbQTbW|Lc}*5+B)kvsUw8 zcE7gu$kx2X((I6mFSwDDVk&q`?mo=m$=p_WxZ!SvBaw8!Xw)Vv^is!sEF8|V3`3+O zY7WlM!+XB7YZmXU)%ck%J;(BWytXHjT?Khq%(KrmDv$ItA+6>gku)h<&4Bdpc&#~b7%lD%#2#O}dg zabajmNUd608I;qq@De>P7#oD)M=>4?;m2m|mT!N8GfS$ac*lC0VSe-NN-9HFrtwtL zk2Nbh=#PF*ONKIEX;LD}_D{IQ+B&QkRduL>=bH6Q9GuJ za0lkevr+tLGY7y#?ID!Qn|*iyng0t$?eRWTr2{1ec0)XjV1d{h!(zFu6Gv^rAA=@N zUc*!Q`Z%&--i60JNp->p@Ritc1J~tq6S+3?Z)E(YchemA?n9UQ9q5d|3N4avBs3O( z3SH;}2xSJ9ZPx3^6$AJarpj1+m|W3e^v6e0AuC=3-~pzJ>PHA^8*Vw%(?;fRX|kG_ zF225wnepE-yD)86g8e=cmJ`)6RF?`Vh!@i9OiL|{r{p!7ENW3vduD<(Nn{~Btk4@H^Z#Z{GQsa zVuRvtCgzBTneczcM#H$IzKM<4#Kuv0UA{|)j`wQ2Zchn!E<9b}UM*eUu`zrA&@^JT delta 775 zcmYjLZAepL7`^A&y=xmemsy!fEmH)oT=gRteGL0FmQ5=yC`gH@SapYMR21D3G%O;# zvDZQ0#I+(scVjP6kks&B6hZ#$6Df=6!#+8>5wF0 z{$kYpmY9?eSRcqR@cXM+t^=QE1w}_pkQBj}W8+I*llj>jKjv4nozB5T(Z1qKLZ>~E zXKMy4pLek>PE0EoO#wAvRWj$NK2$};N+MxzTiM(HHIL;YyAf^{^TLZKjCs+FT|Bb` zE=Q$rb3?)HcV_|)yKc_$Px*30sAm6z<=T-Jp{#+@w#!2GtDBnMuinAtm12p*ll~a^ z>8=UzgAvSJh%VM$hxMpto;sAUiV&u<+Inno`6TOEV3!tS2Jf#&0PJxA1$nR+H(@kD zHd2C{LDcAB6h?(jk=j|^Zq%_u3EbuPkD(Oo)LztEK9{MVk>bOxm`(VvCM(jifA*7+ z2J!G?v=g^=i#>egJlwYU2tKfD+sVtCuA;%p*1_lnrADEFYaALGgDA8<8_z&-2L*WG z75ZU(hus(gv5s4a=?j}fe4vOHD27bLM$*V*z94G()?&Ii#&DCH-OFHwe&_Hq-Z7S9 tIr>-aHGf@Bg-# + { + try { - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Database doesn't contain table 'endword_list'", LOG_MODULE_NAME); - return; - } - int WordCount = 0; - int EndWordCount = 0; - using (SqliteDataReader reader2 = new SqliteCommand($"SELECT * FROM word_list", externalDBConnection).ExecuteReader()) - while (reader2.Read()) + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Loading external database: {dbFileName}", LOG_MODULE_NAME); + var externalDBConnection = new SqliteConnection("Data Source=" + dbFileName); + externalDBConnection.Open(); + + if (!CheckTable_Check("word_list")) { - string word = reader2["word"].ToString().Trim(); - bool isEndWord = Convert.ToBoolean(Convert.ToInt32(reader2["is_endword"])); - if (AddWord(word, isEndWord)) - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Imported word '{word}' {(isEndWord ? "(EndWord)" : "")}", LOG_MODULE_NAME); - else - ConsoleManager.Log(ConsoleManager.LogType.Warning, $"Word '{word}' is already existing in database.", LOG_MODULE_NAME); - WordCount++; + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Database doesn't contain table 'word_list'", LOG_MODULE_NAME); + return; } - - using (SqliteDataReader reader2 = new SqliteCommand("SELECT * FROM endword_list", externalDBConnection).ExecuteReader()) - while (reader2.Read()) + if (!CheckTable_Check("endword_list")) { - string endword = reader2["word_index"].ToString(); - if (AddEndWord(endword)) - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Added end-word '{endword}'", LOG_MODULE_NAME); - else - ConsoleManager.Log(ConsoleManager.LogType.Warning, $"End-word '{endword}' is already existing in database.", LOG_MODULE_NAME); - EndWordCount++; + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Database doesn't contain table 'endword_list'", LOG_MODULE_NAME); + return; } - ConsoleManager.Log(ConsoleManager.LogType.Info, $"DB Import Complete. ({WordCount} Words / {EndWordCount} EndWordNodes)", LOG_MODULE_NAME); - if (DBJobDone != null) - DBJobDone(null, new DBJobArgs("데이터베이스 불러오기", $"{WordCount} 개의 단어 / {EndWordCount} 개의 한방 노드")); - } - catch (Exception e) - { - ConsoleManager.Log(ConsoleManager.LogType.Error, "Failed to connect external DB : " + e.ToString(), LOG_MODULE_NAME); - } + int WordCount = 0; + int EndWordCount = 0; + using (SqliteDataReader reader2 = new SqliteCommand($"SELECT * FROM word_list", externalDBConnection).ExecuteReader()) + while (reader2.Read()) + { + string word = reader2["word"].ToString().Trim(); + bool isEndWord = Convert.ToBoolean(Convert.ToInt32(reader2["is_endword"])); + if (AddWord(word, isEndWord)) + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Imported word '{word}' {(isEndWord ? "(EndWord)" : "")}", LOG_MODULE_NAME); + else + ConsoleManager.Log(ConsoleManager.LogType.Warning, $"Word '{word}' is already existing in database.", LOG_MODULE_NAME); + WordCount++; + } + + using (SqliteDataReader reader2 = new SqliteCommand("SELECT * FROM endword_list", externalDBConnection).ExecuteReader()) + while (reader2.Read()) + { + string endword = reader2["word_index"].ToString(); + if (AddEndWord(endword)) + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Added end-word '{endword}'", LOG_MODULE_NAME); + else + ConsoleManager.Log(ConsoleManager.LogType.Warning, $"End-word '{endword}' is already existing in database.", LOG_MODULE_NAME); + EndWordCount++; + } + + ConsoleManager.Log(ConsoleManager.LogType.Info, $"DB Import Complete. ({WordCount} Words / {EndWordCount} EndWordNodes)", LOG_MODULE_NAME); + if (DBJobDone != null) + DBJobDone(null, new DBJobArgs("데이터베이스 불러오기", $"{WordCount} 개의 단어 / {EndWordCount} 개의 한방 노드")); + } + catch (Exception e) + { + ConsoleManager.Log(ConsoleManager.LogType.Error, "Failed to connect external DB : " + e.ToString(), LOG_MODULE_NAME); + } + }); } public static List GetEndWordList() @@ -161,7 +166,7 @@ public static bool AddEndWord(string node) return true; } - public static async void CheckDB(bool UseOnlineDB) + public static void CheckDB(bool UseOnlineDB) { if (UseOnlineDB && string.IsNullOrWhiteSpace(DatabaseManagement.EvaluateJS("document.getElementById('dict-output').style"))) { @@ -169,121 +174,125 @@ public static async void CheckDB(bool UseOnlineDB) return; } - try + if (DBJobStart != null) + DBJobStart(null, new DBJobArgs("데이터베이스 검증")); + + + Task.Run(() => { - if (DBJobStart != null) - DBJobStart(null, new DBJobArgs("데이터베이스 검증")); - - ConsoleManager.Log(ConsoleManager.LogType.Info, "Database Intergrity Check....\nIt will be very long task.", LOG_MODULE_NAME); - int dbTotalCount; - - int.TryParse(new SqliteCommand("SELECT COUNT(*) FROM word_list", _sqlLiteConnection).ExecuteScalar().ToString(), out dbTotalCount); - - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Database has Total {dbTotalCount} elements.", LOG_MODULE_NAME); - ConsoleManager.Log(ConsoleManager.LogType.Info, "Getting all elements from database..", LOG_MODULE_NAME); - int elementCount = 0; - int DeduplicatedCount = 0; - int RemovedCount = 0; - int FixedCount = 0; - var DeletionList = new List(); - var WordIndexCorrection = new List(); - var IsEndWordCorrection = new Dictionary(); - ConsoleManager.Log(ConsoleManager.LogType.Info, "Opening _ChecksqlLiteConnection.", LOG_MODULE_NAME); - - var _ChecksqlLiteConnection = new SqliteConnection("Data Source=" + _sqlLiteDBLocation); - _ChecksqlLiteConnection.Open(); - - // Deduplicate db - // https://wiki.postgresql.org/wiki/Deleting_duplicates try { - DeduplicatedCount = new SqliteCommand("DELETE FROM word_list WHERE _rowid_ IN (SELECT _rowid_ FROM (SELECT _rowid_, ROW_NUMBER() OVER w as rnum FROM word_list WINDOW w AS (PARTITION BY word ORDER BY _rowid_)) t WHERE t.rnum > 1);", _ChecksqlLiteConnection).ExecuteNonQuery(); - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Deduplicated {DeduplicatedCount} entries.", LOG_MODULE_NAME); - } - catch (Exception ex) - { - ConsoleManager.Log(ConsoleManager.LogType.Error, $"Word deduplication failed: {ex}", LOG_MODULE_NAME); - } - - // Check for errors - using (SqliteDataReader reader = new SqliteCommand("SELECT * FROM word_list ORDER BY(word) DESC", _ChecksqlLiteConnection).ExecuteReader()) - { - while (reader.Read()) + ConsoleManager.Log(ConsoleManager.LogType.Info, "Database Intergrity Check....\nIt will be very long task.", LOG_MODULE_NAME); + int dbTotalCount; + + int.TryParse(new SqliteCommand("SELECT COUNT(*) FROM word_list", _sqlLiteConnection).ExecuteScalar().ToString(), out dbTotalCount); + + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Database has Total {dbTotalCount} elements.", LOG_MODULE_NAME); + ConsoleManager.Log(ConsoleManager.LogType.Info, "Getting all elements from database..", LOG_MODULE_NAME); + int elementCount = 0; + int DeduplicatedCount = 0; + int RemovedCount = 0; + int FixedCount = 0; + var DeletionList = new List(); + var WordIndexCorrection = new List(); + var IsEndWordCorrection = new Dictionary(); + ConsoleManager.Log(ConsoleManager.LogType.Info, "Opening _ChecksqlLiteConnection.", LOG_MODULE_NAME); + + var _ChecksqlLiteConnection = new SqliteConnection("Data Source=" + _sqlLiteDBLocation); + _ChecksqlLiteConnection.Open(); + + // Deduplicate db + // https://wiki.postgresql.org/wiki/Deleting_duplicates + try + { + DeduplicatedCount = new SqliteCommand("DELETE FROM word_list WHERE _rowid_ IN (SELECT _rowid_ FROM (SELECT _rowid_, ROW_NUMBER() OVER w as rnum FROM word_list WINDOW w AS (PARTITION BY word ORDER BY _rowid_)) t WHERE t.rnum > 1);", _ChecksqlLiteConnection).ExecuteNonQuery(); + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Deduplicated {DeduplicatedCount} entries.", LOG_MODULE_NAME); + } + catch (Exception ex) { - elementCount++; - string content = reader["word"].ToString(); - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Total {dbTotalCount} of {elementCount} ({content})", LOG_MODULE_NAME); + ConsoleManager.Log(ConsoleManager.LogType.Error, $"Word deduplication failed: {ex}", LOG_MODULE_NAME); + } - // Check word validity - if (content.Length == 1 || int.TryParse(content[0].ToString(), out int _) || content[0] == '[' || content[0] == '-' || content[0] == '.' || content.Contains(" ")) + // Check for errors + using (SqliteDataReader reader = new SqliteCommand("SELECT * FROM word_list ORDER BY(word) DESC", _ChecksqlLiteConnection).ExecuteReader()) + { + while (reader.Read()) { - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Not a valid word; Will be removed.", LOG_MODULE_NAME); - DeletionList.Add(content); - continue; + elementCount++; + string content = reader["word"].ToString(); + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Total {dbTotalCount} of {elementCount} ({content})", LOG_MODULE_NAME); + + // Check word validity + if (content.Length == 1 || int.TryParse(content[0].ToString(), out int _) || content[0] == '[' || content[0] == '-' || content[0] == '.' || content.Contains(" ")) + { + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Not a valid word; Will be removed.", LOG_MODULE_NAME); + DeletionList.Add(content); + continue; + } + + // Online verify + if (UseOnlineDB && !CheckElementOnline(content)) + { + DeletionList.Add(content); + continue; + } + + // Check WordIndex tag + string correctWordIndex = content[0].ToString(); + if (correctWordIndex != reader["word_index"].ToString()) + { + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Invaild Word Index; Will be fixed to '{correctWordIndex}'.", LOG_MODULE_NAME); + WordIndexCorrection.Add(content); + } + + // Check IsEndWord tag + int CorrectIsEndWord = Convert.ToInt32(PathFinder.EndWordList.Contains(content.Last().ToString())); + if (CorrectIsEndWord != Convert.ToInt32(reader["is_endword"].ToString())) + { + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Invaild Is_EndWord Tag; Will be fixed to '{CorrectIsEndWord}'.", LOG_MODULE_NAME); + IsEndWordCorrection.Add(content, CorrectIsEndWord); + } } + } - // Online verify - if (UseOnlineDB && !CheckElementOnline(content)) - { - DeletionList.Add(content); - continue; - } + // Start fixing + foreach (string content in DeletionList) + { + RemovedCount++; + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Removed '{content}' from database.", LOG_MODULE_NAME); + ExecuteCommand("DELETE FROM word_list WHERE word = '" + content + "'"); + } - // Check WordIndex tag - string correctWordIndex = content[0].ToString(); - if (correctWordIndex != reader["word_index"].ToString()) - { - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Invaild Word Index; Will be fixed to '{correctWordIndex}'.", LOG_MODULE_NAME); - WordIndexCorrection.Add(content); - } + foreach (string content in WordIndexCorrection) + { + FixedCount++; - // Check IsEndWord tag - int CorrectIsEndWord = Convert.ToInt32(PathFinder.EndWordList.Contains(content.Last().ToString())); - if (CorrectIsEndWord != Convert.ToInt32(reader["is_endword"].ToString())) - { - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Invaild Is_EndWord Tag; Will be fixed to '{CorrectIsEndWord}'.", LOG_MODULE_NAME); - IsEndWordCorrection.Add(content, CorrectIsEndWord); - } + string correctWordIndex = content.First().ToString(); + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Fixed word_index of '{content}' to '{correctWordIndex}'.", LOG_MODULE_NAME); + ExecuteCommand($"UPDATE word_list SET word_index = '{correctWordIndex}' WHERE word = '{content}';"); } - } - // Start fixing - foreach (string content in DeletionList) - { - RemovedCount++; - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Removed '{content}' from database.", LOG_MODULE_NAME); - ExecuteCommand("DELETE FROM word_list WHERE word = '" + content + "'"); - } + foreach (var pair in IsEndWordCorrection) + { + FixedCount++; - foreach (string content in WordIndexCorrection) - { - FixedCount++; + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Fixed is_endword of '{pair.Key}' to '{pair.Value}'.", LOG_MODULE_NAME); + ExecuteCommand($"UPDATE word_list SET is_endword = '{pair.Value}' WHERE word = '{pair.Key}';"); + } - string correctWordIndex = content.First().ToString(); - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Fixed word_index of '{content}' to '{correctWordIndex}'.", LOG_MODULE_NAME); - ExecuteCommand($"UPDATE word_list SET word_index = '{correctWordIndex}' WHERE word = '{content}';"); - } + _ChecksqlLiteConnection.Close(); - foreach (var pair in IsEndWordCorrection) - { - FixedCount++; + ConsoleManager.Log(ConsoleManager.LogType.Info, $"Total {dbTotalCount} / Removed {RemovedCount} / Fixed {FixedCount}.", LOG_MODULE_NAME); + ConsoleManager.Log(ConsoleManager.LogType.Info, "Database Check Completed.", LOG_MODULE_NAME); - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Fixed is_endword of '{pair.Key}' to '{pair.Value}'.", LOG_MODULE_NAME); - ExecuteCommand($"UPDATE word_list SET is_endword = '{pair.Value}' WHERE word = '{pair.Key}';"); + if (DBJobDone != null) + DBJobDone(null, new DBJobArgs("데이터베이스 검증", $"{RemovedCount} 개 항목 제거됨 / {FixedCount} 개 항목 수정됨")); } - - _ChecksqlLiteConnection.Close(); - - ConsoleManager.Log(ConsoleManager.LogType.Info, $"Total {dbTotalCount} / Removed {RemovedCount} / Fixed {FixedCount}.", LOG_MODULE_NAME); - ConsoleManager.Log(ConsoleManager.LogType.Info, "Database Check Completed.", LOG_MODULE_NAME); - - if (DBJobDone != null) - DBJobDone(null, new DBJobArgs("데이터베이스 검증", $"{RemovedCount} 개 항목 제거됨 / {FixedCount} 개 항목 수정됨")); - } - catch (Exception ex) - { - ConsoleManager.Log(ConsoleManager.LogType.Error, $"Exception while checking database: {ex}", LOG_MODULE_NAME); - } + catch (Exception ex) + { + ConsoleManager.Log(ConsoleManager.LogType.Error, $"Exception while checking database: {ex}", LOG_MODULE_NAME); + } + }); } private static bool CheckElementOnline(string i)