Merge branch 'dev'

aggregation/Dockerfile

@@ -1,27 +1,27 @@
-FROM satantime/puppeteer-node:20.11-buster-slim As development
+FROM satantime/puppeteer-node:20.11-buster-slim AS development
 
 WORKDIR /usr/src/app
 COPY --chown=node:node package*.json ./
 RUN npm ci
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 . .
 USER node
 
 
-FROM satantime/puppeteer-node:20.11-buster-slim As build
+FROM satantime/puppeteer-node:20.11-buster-slim AS build
 
 WORKDIR /usr/src/app
-COPY --chown=node:node package*.json ./
-COPY --chown=node:node --from=development /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 package*.json ./
+COPY --chown=node:node --chmod=644 --from=development /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 . .
 RUN npm run build
 ENV NODE_ENV production
 RUN npm ci --only=production && npm cache clean --force
 USER node
 
-FROM satantime/puppeteer-node:20.11-buster-slim As production
+FROM satantime/puppeteer-node:20.11-buster-slim AS production
 
-COPY --chown=node:node --from=build /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node --from=build /usr/src/app/dist ./dist
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/dist ./dist
 RUN npx puppeteer browsers install chrome
 EXPOSE 3000
 

aggregation/src/data-fetchers/strategies/images/lexica-data-fetcher.ts

@@ -67,22 +67,26 @@ export class LexicaDataFetcher implements ImageFetcher {
 
   private async setMutationObserver(page: Page) {
     return page.evaluateHandle(() => {
+      const processNode = (node) => {
+        if (
+          node instanceof HTMLElement &&
+          node.matches('div[role="gridcell"]')
+        ) {
+          const url = node.querySelector('a')?.href;
+          if (url) {
+            (window as any).saveUrl(url);
+          }
+        }
+      };
+
+      const processMutation = (mutation) => {
+        mutation.addedNodes.forEach(processNode);
+      };
+
       const observer = new MutationObserver((mutations) => {
-        mutations.forEach((mutation) => {
-          mutation.addedNodes.forEach((node) => {
-            console.log(node);
-            if (
-              node instanceof HTMLElement &&
-              node.matches('div[role="gridcell"]')
-            ) {
-              const url = node.querySelector('a')?.href;
-              if (url) {
-                (window as any).saveUrl(url);
-              }
-            }
-          });
-        });
+        mutations.forEach(processMutation);
       });
+
       observer.observe(document.body, { childList: true, subtree: true });
       return observer;
     });

aggregation/src/data-fetchers/strategies/images/midjourney-data-fetcher.ts

@@ -58,35 +58,43 @@ export class MidjourneyDataFetcher implements ImageFetcher {
 
   private async monitorAndCaptureUrls(page: Page): Promise<string[]> {
     const urls = new Set<string>();
+
     await page.exposeFunction('saveUrl', (url: string) => {
       urls.add(url);
     });
 
-    const observerHandle = await page.evaluateHandle(() => {
-      const observer = new MutationObserver((mutations) => {
-        mutations.forEach((mutation) => {
-          const mutatedImageNodesCallback = (node: Node) => {
-            if (
-              node instanceof HTMLElement &&
-              node.matches('div:has(> a.block.bg-cover)')
-            ) {
-              const linkElement = node.querySelector('a.block.bg-cover');
-              const fullUrl = `https://www.midjourney.com${linkElement.getAttribute('href')}`;
-              (window as any).saveUrl(fullUrl);
-            }
-          };
-          mutation.addedNodes.forEach(mutatedImageNodesCallback);
-          mutation.removedNodes.forEach(mutatedImageNodesCallback);
-        });
+    const processNode = (node: Node) => {
+      if (
+        node instanceof HTMLElement &&
+        node.matches('div:has(> a.block.bg-cover)')
+      ) {
+        const linkElement = node.querySelector('a.block.bg-cover');
+        if (linkElement) {
+          const fullUrl = `https://www.midjourney.com${linkElement.getAttribute('href')}`;
+          (window as any).saveUrl(fullUrl);
+        }
+      }
+    };
+
+    const mutationCallback = (mutations: MutationRecord[]) => {
+      mutations.forEach((mutation) => {
+        mutation.addedNodes.forEach(processNode);
+        mutation.removedNodes.forEach(processNode);
       });
+    };
+
+    const observerHandle = await page.evaluateHandle(() => {
+      const observer = new MutationObserver((window as any).mutationCallback);
       observer.observe(document.body, { childList: true, subtree: true });
       return observer;
     });
 
+    await page.exposeFunction('mutationCallback', mutationCallback);
     await this.scrollGalleryToEnd(page, 'pageScroll');
     await observerHandle.evaluate((observer: MutationObserver) =>
       observer.disconnect(),
     );
+
     return Array.from(urls);
   }
 

ai-image-classifier/main.py

@@ -17,7 +17,7 @@
 models = {}
 
 
-def load_models():
+def load_genai_image_classifiers():
     model_dir = 'models'
     for model_file in os.listdir(model_dir):
         if model_file.endswith('.pkl'):
@@ -26,8 +26,8 @@ def load_models():
             models[model_name] = load_learner(os.path.join(model_dir, model_file))
 
 
-load_models()
-BEST_MODEL = 'resnet34_image_classifier_v4'
+load_genai_image_classifiers()
+BEST_CLASSIFIER = 'resnet34_image_classifier_v4'
 
 
 def get_prediction(model, request):
@@ -57,7 +57,7 @@ def get_prediction(model, request):
 
 @app.route('/prediction', methods=['POST'])
 def predict_using_best_model():
-    return get_prediction(BEST_MODEL, request)
+    return get_prediction(BEST_CLASSIFIER, request)
 
 
 @app.route('/<model>/prediction', methods=['POST'])

ai-image-classifier/model_training.py

@@ -1,8 +1,16 @@
-from fastai.vision.all import *
+import os
+
+import matplotlib.pyplot as plt
+from fastai.callback.tracker import EarlyStoppingCallback, SaveModelCallback
+from fastai.interpret import ClassificationInterpretation
+from fastai.metrics import accuracy
 from fastai.vision.augment import Resize
+from fastai.vision.data import ImageDataLoaders
+from fastai.vision.learner import vision_learner
+from fastai.vision.models import resnet34
 
 
-def setup_and_train():
+def setup_and_train_genai_classifier():
     dataset_path = 'dataset\\train'
     model_path = 'models'
 
@@ -14,37 +22,37 @@ def setup_and_train():
         item_tfms=Resize(224)
     )
 
-    learn = vision_learner(dls, resnet34, metrics=accuracy)
+    genai_classifier_learn = vision_learner(dls, resnet34, metrics=accuracy)
 
     early_stop_callback = EarlyStoppingCallback(monitor='valid_loss', min_delta=0.01, patience=3)
     save_model_callback = SaveModelCallback(fname='best_model', every_epoch=False, with_opt=True)
 
-    learn.lr_find()
-    learn.fit_one_cycle(20, cbs=[early_stop_callback, save_model_callback])
+    genai_classifier_learn.lr_find()
+    genai_classifier_learn.fit_one_cycle(20, cbs=[early_stop_callback, save_model_callback])
 
-    learn.save(f'resnet34_image_classifier_v4.pth')
-    learn.export(f'resnet34_image_classifier_v4.pkl')
-    print("Saved GenAI image classifier model to disk")
+    genai_classifier_learn.save('resnet34_image_classifier_v4.pth')
+    genai_classifier_learn.export('resnet34_image_classifier_v4.pkl')
+    print('Saved GenAI image classifier model to disk')
 
-    return learn, model_path
+    return genai_classifier_learn, model_path
 
 
-def plot_results(learn):
+def test_and_plot_results(genai_classifier_learn):
     plt.figure(figsize=(10, 4))
-    learn.recorder.plot_lr_find()
+    genai_classifier_learn.recorder.plot_lr_find()
     plt.title("Learning Rate Finder")
     plt.xlabel("Learning Rate")
     plt.ylabel("Loss")
     plt.savefig(f'learning_rate_finder.png')
 
     plt.figure(figsize=(10, 4))
-    learn.recorder.plot_loss()
+    genai_classifier_learn.recorder.plot_loss()
     plt.title("Training and Validation Loss")
     plt.xlabel("Iterations")
     plt.ylabel("Loss")
     plt.savefig(f'training_validation_loss.png')
 
-    interp = ClassificationInterpretation.from_learner(learn)
+    interp = ClassificationInterpretation.from_learner(genai_classifier_learn)
 
     interp.plot_top_losses(6, figsize=(15, 11))
     plt.savefig(f'top_losses.png')
@@ -54,5 +62,5 @@ def plot_results(learn):
 
 
 if __name__ == '__main__':
-    learn, model_path = setup_and_train()
-    plot_results(learn)
+    genai_classifier_learn, model_path = setup_and_train_genai_classifier()
+    test_and_plot_results(genai_classifier_learn)

content/Dockerfile

@@ -1,27 +1,27 @@
 FROM node:20-alpine AS development
 
 WORKDIR /usr/src/app
-COPY --chown=node:node package*.json ./
+COPY --chown=node:node --chmod=644 package*.json ./
 RUN npm ci
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 . .
 USER node
 
 
 FROM node:20-alpine AS build
 
 WORKDIR /usr/src/app
-COPY --chown=node:node package*.json ./
-COPY --chown=node:node --from=development /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 package*.json ./
+COPY --chown=node:node --chmod=644 --from=development /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 . .
 RUN npm run build
 ENV NODE_ENV production
 RUN npm ci --only=production && npm cache clean --force
 USER node
 
 FROM node:20-alpine AS production
 
-COPY --chown=node:node --from=build /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node --from=build /usr/src/app/dist ./dist
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/dist ./dist
 
 EXPOSE 3000
 

gateway/Dockerfile

@@ -1,27 +1,27 @@
-FROM node:20-alpine As development
+FROM node:20-alpine AS development
 
 WORKDIR /usr/src/app
-COPY --chown=node:node package*.json ./
+COPY --chown=node:node --chmod=644 package*.json ./
 RUN npm ci
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 . .
 USER node
 
 
-FROM node:20-alpine As build
+FROM node:20-alpine AS build
 
 WORKDIR /usr/src/app
-COPY --chown=node:node package*.json ./
-COPY --chown=node:node --from=development /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 package*.json ./
+COPY --chown=node:node --chmod=644 --from=development /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 . .
 RUN npm run build
 ENV NODE_ENV production
 RUN npm ci --only=production && npm cache clean --force
 USER node
 
-FROM node:20-alpine As production
+FROM node:20-alpine AS production
 
-COPY --chown=node:node --from=build /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node --from=build /usr/src/app/dist ./dist
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/dist ./dist
 
 EXPOSE 3000
 

gateway/src/classifiers/controllers/images.controller.ts

@@ -1,12 +1,4 @@
-import {
-  Controller,
-  Next,
-  Post,
-  Req,
-  Res,
-  UseGuards,
-  UseInterceptors,
-} from '@nestjs/common';
+import { Controller, Next, Post, Req, Res, UseGuards } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { RequestHandler } from 'http-proxy-middleware';
 import { IncomingMessage, ServerResponse } from 'http';
@@ -20,7 +12,6 @@ import {
   ApiTags,
 } from '@nestjs/swagger';
 import { JwtAuthGuard } from '../../shared/guards/jwt-auth.guard';
-import { FileInterceptor } from '@nestjs/platform-express';
 
 @ApiTags('image classifiers')
 @ApiBearerAuth('Access Token')

users/Dockerfile

@@ -1,26 +1,26 @@
-FROM node:20-alpine As development
+FROM node:20-alpine AS development
 
 WORKDIR /usr/src/app
-COPY --chown=node:node package*.json ./
+COPY --chown=node:node --chmod=644 package*.json ./
 RUN npm ci
-COPY --chown=node:node . .
+COPY --chown=node:node --chmod=644 . .
 USER node
 
 
-FROM node:20-alpine As build
+FROM node:20-alpine AS build
 
 WORKDIR /usr/src/app
 COPY --chown=node:node package*.json ./
-COPY --chown=node:node --from=development /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 --from=development /usr/src/app/node_modules ./node_modules
 COPY --chown=node:node . .
 RUN npm run build
 ENV NODE_ENV production
 RUN npm ci --only=production && npm cache clean --force
 USER node
 
-FROM node:20-alpine As production
+FROM node:20-alpine AS production
 
-COPY --chown=node:node --from=build /usr/src/app/node_modules ./node_modules
-COPY --chown=node:node --from=build /usr/src/app/dist ./dist
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/node_modules ./node_modules
+COPY --chown=node:node --chmod=644 --from=build /usr/src/app/dist ./dist
 
 CMD [ "node", "dist/main.js" ]

users/src/auth/auth.service.ts

@@ -104,10 +104,10 @@ export class AuthService {
     const userAuthData = await this.userAuthDataRepository.findOne({
       where: { id: decoded.sub },
     });
-    if (!userAuthData || !userAuthData.refreshToken) {
+    if (!userAuthData?.refreshToken) {
       throw new UnauthorizedException();
     }
-    const isRefreshTokenMatching = compare(token, userAuthData.refreshToken);
+    const isRefreshTokenMatching = await compare(token, userAuthData.refreshToken);
     if (!isRefreshTokenMatching) {
       throw new UnauthorizedException();
     }