data_lake 완성

autosink_data_elt/log/filehandler.py

@@ -6,6 +6,7 @@
 import pytz
 from datetime import datetime
 from dataclasses import asdict
+from collections import deque
 
 # 프로젝트
 from autosink_data_elt.log.template.dishwashing import *
@@ -63,11 +64,14 @@ def read_file(self):
 
     def write_file(self, data):
         with open(self.filename, 'w') as file:
-            json.dump(asdict(data), file, indent=4)
+            json.dump(data.to_dict(), file, indent=4)
         self.image_counter = 0  # Reset the image counter after writing to file
         logger.info(f'Wrote data to {self.filename} and reset image counter')
 
-    def create_default_data(self, dishwashing_start=None, version=2):
+    def create_default_data(self,
+                            dishwashing_start=None,
+                            version=2,
+                            deque_size=10):
         if dishwashing_start is None:
             dishwashing_start = datetime.now(
                 self.timezone).isoformat(timespec='seconds')
@@ -83,27 +87,28 @@ def create_default_data(self, dishwashing_start=None, version=2):
             return DishwashingDataV2(version=2,
                                      user_id=self.user_id,
                                      dishwashing_id=self.dishwashing_id,
-                                     dishwashing_start=dishwashing_start)
+                                     dishwashing_start=dishwashing_start,
+                                     interaction=deque(maxlen=deque_size))
         else:
             logger.error(f'Unsupported version: {version}')
             raise ValueError(f'Unsupported version: {version}')
 
     def add_interaction(self, data, **kwargs):
-        timestamp = datetime.now(self.timezone).isoformat(timespec='seconds')
-        image = f'{self.image_counter}.png'
-        self.image_counter += 1
-        logger.info(f'Adding interaction with image {image}')
+        timestamp = datetime.now().isoformat(timespec='seconds')
+        # logger.info(f'Adding interaction with image from kwargs')
 
         if data.version == 1:
-            interaction = InteractionV1.create(timestamp, image, **kwargs)
+            interaction = InteractionV1.create(timestamp, kwargs.pop('image'),
+                                               **kwargs)
         elif data.version == 2:
-            interaction = InteractionV2.create(timestamp, image, **kwargs)
+            interaction = InteractionV2.create(timestamp, kwargs.pop('image'),
+                                               **kwargs)
         else:
             logger.error(f'Unsupported version: {data.version}')
             raise ValueError(f'Unsupported version: {data.version}')
 
         data.interaction.append(interaction)
-        logger.info(f'Added interaction: {interaction}')
+        # logger.info(f'Added interaction: {interaction}')
         return data
 
 

autosink_data_elt/log/template/dishwashing.py

@@ -1,6 +1,6 @@
 # 내장
 from typing import List, Deque
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, asdict
 from collections import deque
 
 # 프로젝트
@@ -15,11 +15,26 @@ class DishwashingDataV1:
     dishwashing_start: str
     interaction: List[InteractionV1] = field(default_factory=list)
 
+    def to_dict(self):
+        data_dict = asdict(self)
+        data_dict['interaction'] = [
+            interaction.to_dict() for interaction in self.interaction
+        ]
+        return data_dict
+
 
 @dataclass
 class DishwashingDataV2:
     version: int
     user_id: str
     dishwashing_id: str
     dishwashing_start: str
-    interaction: Deque[InteractionV1] = field(default_factory=deque)
+    interaction: Deque[InteractionV2] = field(
+        default_factory=lambda: deque(maxlen=10))
+
+    def to_dict(self):
+        data_dict = asdict(self)
+        data_dict['interaction'] = [
+            interaction.to_dict() for interaction in self.interaction
+        ]
+        return data_dict

autosink_data_elt/log/template/interaction.py

@@ -1,4 +1,3 @@
-# 내장
 from dataclasses import dataclass
 
 
@@ -11,12 +10,18 @@ class InteractionV1:
 
     @staticmethod
     def create(timestamp, image, **kwargs):
-        return InteractionV1(
-            timestamp=timestamp,
-            model_output=kwargs.get('model_output', 0),
-            magnetic=kwargs.get('magnetic', 0),
-            image=image
-        )
+        return InteractionV1(timestamp=timestamp,
+                             model_output=kwargs.get('model_output', 0),
+                             magnetic=kwargs.get('magnetic', 0),
+                             image=image)
+
+    def to_dict(self):
+        return {
+            "timestamp": self.timestamp,
+            "model_output": self.model_output,
+            "magnetic": self.magnetic,
+            "image": self.image
+        }
 
 
 @dataclass
@@ -29,10 +34,17 @@ class InteractionV2:
 
     @staticmethod
     def create(timestamp, image, **kwargs):
-        return InteractionV2(
-            timestamp=timestamp,
-            model_output=kwargs.get('model_output', 0),
-            arduino_output=kwargs.get('arduino_output', 0),
-            magnetic_status=kwargs.get('magnetic_status', 0),
-            image=image
-        )
+        return InteractionV2(timestamp=timestamp,
+                             model_output=kwargs.get('model_output', 0),
+                             arduino_output=kwargs.get('arduino_output', 0),
+                             magnetic_status=kwargs.get('magnetic_status', 0),
+                             image=image)
+
+    def to_dict(self):
+        return {
+            "timestamp": self.timestamp,
+            "model_output": self.model_output,
+            "arduino_output": self.arduino_output,
+            "magnetic_status": self.magnetic_status,
+            "image": self.image
+        }