fix bug of logger: would create multi-logger

haf/common/database.py

@@ -84,24 +84,24 @@ def connect_execute(self, sqlconfig:SQLConfig, sqlscript:list, **kwargs):
                 for ss in sqlscript:
                     if ss != None and ss != "None" and len(ss) > 5:
                         if not run_background:
-                            logger.info(f"{key} start {sqlconfig.host} execute {ss}")
+                            logger.info(f"{key} start {sqlconfig.host} execute {ss}", __name__)
                         cursor_m.execute(ss)
                         data.append(cursor_m.fetchall())
                         if not run_background:
-                            logger.info(f"{key} result {str(data)}")
+                            logger.info(f"{key} result {str(data)}", __name__)
             elif isinstance(sqlscript, str):
                 if sqlscript != None and sqlscript != "None" and "None" not in sqlscript and len(sqlscript) > 5:
                     if not run_background:
-                        logger.info(f"{key} start {sqlconfig.host} execute {sqlscript}")
+                        logger.info(f"{key} start {sqlconfig.host} execute {sqlscript}", __name__)
                     cursor_m.execute(sqlscript)
                     data.append(cursor_m.fetchall())
                     if not run_background:
-                        logger.info(f"{key} result {str(data)}")
+                        logger.info(f"{key} result {str(data)}", __name__)
             if commit:
                 self.connect_msql.commit()
             return data
         except Exception as e:
-            logger.error(str(e))
+            logger.error(str(e), __name__)
             if self.connect_msql.open:
                 self.connect_msql.close()
             return []
@@ -112,7 +112,7 @@ def close(self):
             if self.connect_msql is not None:
                 self.connect_msql.close()
         except Exception as e:
-            logger.error(e)
+            logger.error(e, __name__)
 
 
 class SqlServerTool(object):
@@ -147,20 +147,20 @@ def connect_execute(self, sqlconfig:SQLConfig, sqlscript:list, **kwargs):
             if isinstance(sqlscript, list):
                 for ss in sqlscript:
                     if ss != None and ss != "None" and "None" not in ss and len(ss) > 5:
-                        logger.info(f"{key} [{sqlconfig.host}] start execute {ss}")
+                        logger.info(f"{key} [{sqlconfig.host}] start execute {ss}", __name__)
                         cursor_m.execute(ss)
                         data.append(cursor_m.fetchall())
-                        logger.info(f"{key} result {str(data)}")
+                        logger.info(f"{key} result {str(data)}", __name__)
             elif isinstance(sqlscript, str):
                 if sqlscript != None and sqlscript != "None" and "None" not in sqlscript and len(sqlscript) > 5:
-                    logger.info(f"{key} start [{sqlconfig.host}] execute {sqlscript}")
+                    logger.info(f"{key} start [{sqlconfig.host}] execute {sqlscript}", __name__)
                     cursor_m.execute(sqlscript)
                     data.append(cursor_m.fetchall())
-                    logger.info(f"{key} result {str(data)}")
+                    logger.info(f"{key} result {str(data)}", __name__)
 
             return data
         except Exception as e:
-            logger.error(str(e))
+            logger.error(str(e), __name__)
             if self.connect_msql.open:
                 self.connect_msql.close()
             return []

haf/common/httprequest.py

@@ -54,7 +54,7 @@ def get(url, data=None, headers=None, **kwargs):
         key = kwargs.get("key")
         try:
             url = url + HttpController.getdata(data)
-            logger.info(f'{key} [url] {url}')
+            logger.info(f'{key} [url] {url}', __name__)
             request = ur.Request(url=url, headers=headers, method="GET")
             if headers is not None:
                 for key in headers.keys():
@@ -64,18 +64,18 @@ def get(url, data=None, headers=None, **kwargs):
             if response is None:
                 return {"result": "None"}
             else:
-                logger.info(f"{key} {str(response)}")
+                logger.info(f"{key} {str(response)}", __name__)
 
             return response
         except ur.URLError as e:
-            logger.info(f"{key}{str(e)}")
-            logger.info(f"{key}{traceback.format_exc()}")
+            logger.info(f"{key}{str(e)}", __name__)
+            logger.info(f"{key}{traceback.format_exc()}", __name__)
             traceback.print_exc()
             return e
         except Exception as ee:
             traceback.print_exc()
             logger.info(f"{key}{str(ee)}")
-            logger.info(f"{key}{traceback.format_exc()}")
+            logger.info(f"{key}{traceback.format_exc()}", __name__)
             return ee
 
     @staticmethod
@@ -93,19 +93,19 @@ def post(url, data=None, headers=None, **kwargs):
             if response is None:
                 return {"result": "None"}
             else:
-                logger.info(f"{key} {str(response)}")
+                logger.info(f"{key} {str(response)}", __name__)
 
             return response
         except ur.URLError as e:
-            logger.error(f"{key} {str(e)}")
+            logger.error(f"{key} {str(e)}", __name__)
             traceback.print_exc()
             return e
         except urlerror.HTTPError as httpe:
-            logger.error(f"{key} { str(httpe)}")
+            logger.error(f"{key} { str(httpe)}", __name__)
             traceback.print_exc()
             return httpe
         except Exception as ee:
-            logger.error(f"{key} {str(ee)}")
+            logger.error(f"{key} {str(ee)}", __name__)
             traceback.print_exc()
             return ee
 
@@ -120,10 +120,10 @@ def put(self, url, data=None, **kwargs):
             return result
         except ur.URLError as e:
             traceback.print_exc()
-            logger.error(f"{key} {str(e)}")
+            logger.error(f"{key} {str(e)}", __name__)
         except Exception as ee:
             traceback.print_exc()
-            logger.error(f"{key} {str(ee)}")
+            logger.error(f"{key} {str(ee)}", __name__)
 
     def delete(self, url, data=None):
         data = bytes(data, encoding='utf8') if data is not None else None
@@ -135,6 +135,6 @@ def delete(self, url, data=None):
             data = data.encode() if data is not None else None
             return result
         except ur.URLError as e:
-            logger.error(str(e))
+            logger.error(str(e), __name__)
         except Exception as ee:
-            logger.error(str(ee))
+            logger.error(str(ee), __name__)

haf/common/log.py

@@ -2,35 +2,41 @@
 
 import logging
 from haf.busclient import BusClient
+from haf.common.sigleton import SingletonType
 
 
-class BaseLogger(object):
+class BaseLogger(metaclass=SingletonType):
 
     def __init__(self, logger_name):
         self.logger_name = logger_name
         self.bus_client = None
-
+        self.process_id = 0
+
+    # here using this function to reconnect to the bus : loader, runner, recorder
     def bind_busclient(self, bus_client: BusClient):
         self.bus_client = BusClient(bus_client.domain, bus_client.port, bus_client.auth_key)
+
+    def bind_process(self, process_id):
+        self.process_id = process_id
 
-    def debug(self, msg):
-        msg = {"logger_name":self.logger_name, "level":"debug", "msg": msg}
+    def debug(self, msg, logger_name=None):
+        msg = {"process": self.process_id, "logger_name":self.logger_name if not logger_name else logger_name, "level":"debug", "msg": msg}
         self.msg_write(msg)
 
-    def info(self, msg):
-        msg = {"logger_name":self.logger_name, "level":"info", "msg": msg}
+    def info(self, msg, logger_name=None):
+        msg = {"process": self.process_id, "logger_name":self.logger_name if not logger_name else logger_name, "level":"info", "msg": msg}
         self.msg_write(msg)
 
-    def warning(self, msg):
-        msg = {"logger_name":self.logger_name, "level":"warning", "msg": msg}
+    def warning(self, msg, logger_name=None):
+        msg = {"process": self.process_id, "logger_name":self.logger_name if not logger_name else logger_name, "level":"warning", "msg": msg}
         self.msg_write(msg)
 
-    def error(self, msg):
-        msg = {"logger_name":self.logger_name, "level":"error", "msg": msg}
+    def error(self, msg, logger_name=None):
+        msg = {"process": self.process_id, "logger_name":self.logger_name if not logger_name else logger_name, "level":"error", "msg": msg}
         self.msg_write(msg)
 
-    def critical(self, msg):
-        msg = {"logger_name":self.logger_name, "level":"critical", "msg": msg}
+    def critical(self, msg, logger_name=None):
+        msg = {"process": self.process_id, "logger_name":self.logger_name if not logger_name else logger_name, "level":"critical", "msg": msg}
         self.msg_write(msg)
 
     def __new__(cls, *args, **kwargs):

haf/config.py

@@ -109,7 +109,7 @@
 
 MAIN_VERSION = 2
 SUB_VERSION = 6
-FIX_VERSION = 5
+FIX_VERSION = 6
 VERSION_TYPE = "haf"
 PLATFORM_VERSION = f"{VERSION_TYPE}-{MAIN_VERSION}.{SUB_VERSION}.{FIX_VERSION}"
 

haf/loader.py

@@ -15,7 +15,7 @@
 
 
 class Loader(Process):
-    def __init__(self, bus_client:BusClient=None):
+    def __init__(self, bus_client:BusClient=None, args:list=None):
         super().__init__()
         self.bus_client = bus_client
         self.daemon = True
@@ -24,13 +24,14 @@ def __init__(self, bus_client:BusClient=None):
 
     def run(self):
         try:
+            logger.bind_process(self.pid)
             self.key = f"{self.pid}$%loader$%"
             logger.bind_busclient(self.bus_client)
-            logger.info(f"{self.key} start loader")
+            logger.info(f"{self.key} start loader", __name__)
             self.case_queue = self.bus_client.get_case()
             while True:
                 if self.get_parameter() == SIGNAL_START:
-                    logger.info(f"{self.key} -- get start signal from main")
+                    logger.info(f"{self.key} -- get start signal from main", __name__)
                     break
                 time.sleep(0.01)
 
@@ -49,7 +50,7 @@ def run(self):
                 file_name = temp.get("file_name")
                 inputs = LoadFromConfig.load_from_file(file_name)
 
-                logger.info(f"{self.key} -- {inputs}")
+                # logger.info(f"{self.key} -- {inputs}", __name__)
                 input = inputs.get("config")[0]
                 bench_name = input.get("name")
                 module_name = input.get("module_name")
@@ -93,8 +94,8 @@ def run(self):
                 self.put_web_message("web")
                 time.sleep(0.01)
         except Exception as e:
-            logger.error(f"{self.key} {e}")
-            logger.error(f"{self.key} {FailLoaderException}")
+            logger.error(f"{self.key} {e}", __name__)
+            logger.error(f"{self.key} {FailLoaderException}", __name__)
             self.end_handler(e)
 
     def get_parameter(self, param_key=None):
@@ -121,18 +122,18 @@ def put_web_message(self, key:str):
 
     @locker
     def put_case(self, key:str, case):
-        logger.info(f"{self.key} -- put case {case.bench_name} - {case.ids.id}.{case.ids.subid}.{case.ids.name}")
+        logger.info(f"{self.key} -- put case {case.bench_name} - {case.ids.id}.{case.ids.subid}.{case.ids.name}", __name__)
         self.case_queue.put(case)
 
     def end_handler(self, error=None):
         try:
             if error:
-                logger.error(f"{self.key} end loader with Error - {error}")
+                logger.error(f"{self.key} end loader with Error - {error}", __name__)
             else:
-                logger.info(f"{self.key} end loader")
+                logger.info(f"{self.key} end loader", __name__)
             self.case_queue.put(SIGNAL_CASE_END)
         except Exception as e:
-            logger.error(f"{self.key} {e}")
+            logger.error(f"{self.key} {e}", __name__)
 
 
 class LoadFromConfig(object):

haf/logger.py

@@ -4,9 +4,8 @@
 from multiprocessing import Process
 from haf.busclient import BusClient
 from haf.config import *
-from haf.common.log import Log
 
-logger = Log.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class Logger(Process):
@@ -18,14 +17,16 @@ def __init__(self, case_name, log_dir, bus_client: BusClient):
         self.log_dir = log_dir
         self.loggers = {}
 
+    def reconnect_bus(self):
+        self.bus_client = BusClient(self.bus_client.domain, self.bus_client.port, self.bus_client.auth_key)
+
     def run(self):
-        logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s <%(process)d> [%(name)s] %(message)s')
-        logger.bind_busclient(self.bus_client)
-        logger.info("start system logger")
+        self.reconnect_bus()
+        logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
         log_home = f"{self.log_dir}/{self.case_name}"
         if not os.path.exists(log_home):
             os.makedirs(log_home)
-        # here delete BusClient(), using input bus_client
+        # here delete BusClient(), using input bus_clients
         # self.bus_client = BusClient()
         try:
             log_queue = self.bus_client.get_log()
@@ -47,11 +48,12 @@ def split_log(self, log):
     def log_print(self, log):
         logger_name = log.get("logger_name")
         level = log.get("level")
-        msg = log.get("msg")
-        if logger_name not in self.loggers:
-            self.loggers[logger_name] = logging.getLogger(logger_name)
-
-        logger = self.loggers.get(logger_name)
+        msg_origin = log.get("msg")
+        process = log.get("process")
+        if process not in self.loggers:
+            self.loggers[process] = logging.getLogger(logger_name)
+        logger = self.loggers.get(process)
+        msg = f"<{process}> [{logger_name}] {msg_origin}"
         if level=="debug":
             logger.debug(msg)
         elif level=="info":

haf/program.py

@@ -22,7 +22,7 @@
 from haf.runner import Runner
 from haf.utils import Utils
 
-logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s <%(process)d> [%(name)s] %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s [%(name)s] <%(process)d> %(message)s')
 logger = logging.getLogger(__name__)
 
 
@@ -54,7 +54,7 @@ def _start_loader(self, count: int, bus_client: BusClient):
         :return: None
         '''
         for x in range(count):
-            loader = Loader(bus_client)
+            loader = Loader(bus_client, self.args)
             loader.start()
             time.sleep(0.1)
 
@@ -76,7 +76,7 @@ def _start_recorder(self, bus_client: BusClient, sql_config: SQLConfig=None, sql
         time.sleep(0.1)
 
     def _init_logging_module(self, args):
-        logging.basicConfig(level=logging.INFO if not args.debug else logging.DEBUG, format='%(asctime)s %(levelname)s <%(process)d> [%(name)s] %(message)s')
+        logging.basicConfig(level=logging.INFO if not args.debug else logging.DEBUG, format='%(asctime)s %(levelname)s [%(name)s] %(message)s')
         pass
 
     def _init_system_logger(self, log_dir: str, bus_client: BusClient):