Added Latex Extraction

.github/workflows/main.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04,ubuntu-18.04]
-        version: [3.6]
+        version: [3.6,3.7] 
     runs-on: ${{ matrix.os }}
 
     # service containers to run with `postgres-job`
@@ -24,7 +24,7 @@ jobs:
         # service environment variables
         # `POSTGRES_HOST` is `postgres`
         env:
-          # optional (defaults to `postgres`)
+          # optional (efaults to `postgres`)
           POSTGRES_DB: postgres_db
           # required
           POSTGRES_PASSWORD: postgres
@@ -83,12 +83,14 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3.1.2
         with:
-          python-version: 3.6
+          python-version: ${{ matrix.version }}
 
       # Dependency Installation 
 
       - name: Install Dependencies
         run: |
+          pip3 install flake8==3.7.7
+          flake8 --ignore=E501,F821,E265,E741,F823,F841 .
           echo python version = 
           python --version
           sudo apt-get install python3-dev
@@ -97,7 +99,6 @@ jobs:
           pip3 install tensorflow-serving-api==2.5.2
           pip3 install pytest
           echo installed dependencies
-          flake8 --ignore=E501,F821,E265,E741 .
       # Downloading Test Files
 
       - name: Download Files
@@ -119,10 +120,12 @@ jobs:
           wget https://www.dropbox.com/s/ij5hj4hznczvfcw/text.mp4
           wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1B-goSqkAqyq2dssvvpNy8vRhfxaZEMf5' -O caption1.jpg
           wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1B-goSqkAqyq2dssvvpNy8vRhfxaZEMf5' -O caption2.jpg
+          wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1Nl_sUHu6bQj4bxjVkqa9Wv_Xyj4BldQq' -O photo.jpg
           cd ../..
           cd media 
           mkdir object
           mkdir nsfw
+          mkdir latex
           cd ..
           cd corelib/model
           mkdir facenet
@@ -150,9 +153,10 @@ jobs:
          export DJANGO_SETTINGS_MODULE="Rekognition.settings"
          set -x
          docker version
-         docker pull tensorflow/serving:nightly-devel
+         docker pull tensorflow/serving:2.6.3-devel
+         docker pull lukasblecher/pix2tex:api
          echo $(pwd)
-         docker run -d -t -p 8500:8500 -p 8501:8501 -v /home/runner/work/Rekognition/Rekognition/corelib/model/tfs/model_volume:/home/ tensorflow/serving --model_config_file=/home/configs/models.conf
+         docker run -d -t -p 8500:8500 -p 8501:8501 -v /home/runner/work/Rekognition/Rekognition/corelib/model/tfs/model_volume:/home/ tensorflow/serving:2.6.3 --model_config_file=/home/configs/models.conf
          echo aagye
          python manage.py flush --no-input
          python manage.py migrate

coreapi/urls.py

@@ -19,4 +19,5 @@
     path('nsfwvideo/', views.NsfwVideo.as_view(), name='nsfw_video'),
     path('scenevideo/', views.SceneVideo.as_view(), name='scene_video'),
     path('caption/', views.CaptionGenerate.as_view(), name='caption_api'),
+    path('latex/', views.LatexGenerate.as_view(), name='latex_api'),
 ]

coreapi/views.py

@@ -9,7 +9,7 @@
                               createembedding, process_streaming_video,
                               nsfwclassifier, similarface, object_detect,
                               text_detect, object_detect_video, scene_detect,
-                              text_detect_video, scene_video, nsfw_video, generate_caption)
+                              text_detect_video, scene_video, nsfw_video, generate_caption, generate_latex)
 from .serializers import (EmbedSerializer, NameSuggestedSerializer,
                           SimilarFaceSerializer, ImageFrSerializers)
 from .models import InputEmbed, NameSuggested, SimilarFaceInImage
@@ -193,6 +193,60 @@ def post(self, request):
                 return Response(result, status=status.HTTP_400_BAD_REQUEST)
 
 
+class LatexGenerate(views.APIView):
+    """     To generate caption from an image
+    Workflow
+            *   if  POST method request is made, then initially a random
+                filename is generated and then caption_generate method is
+                called which process the image and outputs the result
+                containing the detected text as a string
+    Returns:
+            *   outputs the result
+                containing the detected latex code  as a string
+    """
+
+    def post(self, request):
+
+        tracemalloc.start()
+        start = time.time()
+        logger.info(msg="POST Request for Latex Generation made")
+        filename = getnewuniquefilename(request)
+        input_file = request.FILES['file']
+        result = generate_latex(input_file, filename)
+        if "Error" not in result:
+            logger.info(msg="Memory Used = " + str((tracemalloc.get_traced_memory()[1] - tracemalloc.get_traced_memory()[0]) * 0.001))
+            end = time.time()
+            logger.info(msg="Time For Prediction = " + str(int(end - start)))
+            result['Time'] = int(end - start)
+            result["Memory"] = (tracemalloc.get_traced_memory()[1] - tracemalloc.get_traced_memory()[0]) * 0.001
+            tracemalloc.stop()
+            return Response(result, status=status.HTTP_200_OK)
+
+        else:
+            if (result["Error"] == 'An HTTP error occurred.'):
+                return Response(result, status=status.HTTP_400_BAD_REQUEST)
+            elif (result["Error"] == 'A Connection error occurred.'):
+                return Response(result, status=status.HTTP_503_SERVICE_UNAVALIABLE)
+            elif (result["Error"] == 'The request timed out.'):
+                return Response(result, status=status.HTTP_408_REQUEST_TIMEOUT)
+            elif (result["Error"] == 'Bad URL'):
+                return Response(result, status=status.HTTP_400_BAD_REQUEST)
+            elif (result["Error"] == 'Text Detection Not Working'):
+                return Response(result, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+            elif (result["Error"] == 'The media format of the requested data is not supported by the server'):
+                return Response(result, status=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
+            elif (result["Error"] == 'A JSON error occurred.'):
+                return Response(result, status=status.HTTP_204_NO_CONTENT)
+            elif (result["Error"] == 'A proxy error occurred.'):
+                return Response(result, status=status.HTTP_407_PROXY_AUTHENTICATION_REQUIRED)
+            elif (result["Error"] == 'The header value provided was somehow invalid.'):
+                return Response(result, status=status.HTTP_411_LENGTH_REQUIRED)
+            elif (result["Error"] == 'The request timed out while trying to connect to the remote server.'):
+                return Response(result, status=status.HTTP_504_GATEWAY_TIMEOUT)
+            else:
+                return Response(result, status=status.HTTP_400_BAD_REQUEST)
+
+
 class NsfwRecognise(views.APIView):
     """     To recognise whether a image is nsfw or not
     Workflow

corelib/constant.py

@@ -43,3 +43,4 @@
 image_vectorization_url = 'models/xception:predict'
 dict_wordtoix_path = str(os.getcwd()) + '/data/caption_generator/wordtoix.pickle'
 dict_ixtoword_path = str(os.getcwd()) + '/data/caption_generator/ixtoword.pickle'
+latex_url = 'http://0.0.0.0:8502/predict/'

corelib/main_api.py

@@ -4,6 +4,7 @@
 import json
 import subprocess
 import shlex
+import requests
 import cv2
 import wordninja
 import urllib.parse
@@ -27,12 +28,11 @@
                               base_url, face_exp_url, nsfw_url, text_reco_url,
                               char_dict_path, ord_map_dict_path, text_detect_url,
                               coco_names_path, object_detect_url, scene_detect_url,
-                              scene_labels_path, image_vectorization_url)
+                              scene_labels_path, image_vectorization_url, latex_url)
 from corelib.utils import ImageFrNetworkChoices, get_class_names, bb_to_cv, get_classes
 from coreapi.models import InputImage, InputVideo, InputEmbed, SimilarFaceInImage
 from logger.logging import RekogntionLogger
 import numpy as np
-import requests
 from corelib.RetinaFace.retina_net import FaceDetectionRetina
 from django.db import IntegrityError, DatabaseError
 from corelib.CaptionGenerator.caption_generator_utils import greedyCaptionSearch, beam_search_predictions
@@ -185,6 +185,64 @@ def generate_caption(input_file, filename, method):
     return {"Texts": res}
 
 
+def generate_latex(input_file, filename):
+    """     Scene Text Detection
+    Args:
+            *   input_file: Contents of the input image file
+            *   filename: filename of the image
+    Workflow:
+            *
+    Returns:
+            *   A string containg the code of the given
+            latex expression
+    """
+
+    logger.info(msg="generate_latex called")
+    file_path = os.path.join(MEDIA_ROOT, 'latex', filename)
+    handle_uploaded_file(input_file, file_path)
+    try:
+
+        json_response = requests.post(latex_url, files={'file': open(file_path, 'rb')})
+    except requests.exceptions.HTTPError as errh:
+        logger.error(msg=errh)
+        return {"Error": "An HTTP error occurred."}
+    except requests.exceptions.ConnectTimeout as err:
+        logger.error(msg=err)
+        return {"Error": "The request timed out while trying to connect to the remote server."}
+    except requests.exceptions.ProxyError as err:
+        logger.error(msg=err)
+        return {"Error": "Scene Detect Not Working"}
+    except requests.exceptions.ConnectionError as errc:
+        logger.error(msg=errc)
+        return {"Error": "A Connection error occurred."}
+    except requests.exceptions.Timeout as errt:
+        logger.error(msg=errt)
+        return {"Error": "The request timed out."}
+    except requests.exceptions.InvalidURL as errm:
+        logger.error(msg=errm)
+        return {"Error": "Bad URL"}
+    except requests.exceptions.ContentDecodingError as err:
+        logger.error(msg=err)
+        return {"Error": "The media format of the requested data is not supported by the server"}
+    except requests.exceptions.InvalidJSONError as err:
+        logger.error(msg=err)
+        return {"Error": "A JSON error occurred."}
+    except requests.exceptions.InvalidHeader as err:
+        logger.error(msg=err)
+        return {"Error": "The header value provided was somehow invalid."}
+    except requests.exceptions.RequestException as err:
+        logger.error(msg=err)
+        return {"Error": "Latex Generator Not Working"}
+    except Exception as e:
+        logger.error(msg=e)
+        return {"Error": e}
+    predictions = json.loads(json_response.text)
+
+    raw_s = r'{}'.format(predictions)
+
+    return {"Result": raw_s}
+
+
 def text_detect(input_file, filename):
     """     Scene Text Detection
     Args:

manage.py

@@ -2,10 +2,17 @@
 """Django's command-line utility for administrative tasks."""
 import os
 import sys
+import docker
+
+container = None
 
 
 def main():
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Rekognition.settings')
+    try:
+        container.stop()
+    except BaseException:
+        pass
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
@@ -14,6 +21,15 @@ def main():
             "available on your PYTHONPATH environment variable? Did you "
             "forget to activate a virtual environment?"
         ) from exc
+    try:
+
+        client = docker.from_env()
+        container = client.containers.run("lukasblecher/pix2tex:api", detach=True,
+                                          ports={'8502/tcp': 8502})
+
+    except BaseException:
+        pass
+
     execute_from_command_line(sys.argv)
 
 

requirements.txt

@@ -20,6 +20,7 @@ dask==1.2.2
 decorator==4.4.0
 Django==2.2.1
 django-celery-beat==1.5.0
+docker
 django-celery-results==1.1.2
 django-cors-headers==3.0.2
 django-timezone-field==3.0

tests/test_views.py

@@ -69,6 +69,22 @@ def test_post(self):
         self.assertEqual(status.HTTP_200_OK, response2.status_code)
 
 
+class TestLatex(TestCase):
+
+    def setUp(self):
+
+        print("Testing Image Captioning")
+        super(TestLatex, self).setUp()
+        self.client = APIClient()
+        file1 = File(open('tests/testdata/photo.jpg', 'rb'))
+        self.uploaded_file1 = SimpleUploadedFile("temp1.jpg", file1.read(), content_type='multipart/form-data')
+
+    def test_post(self):
+
+        response1 = self.client.post('/api/latex/', {'file': self.uploaded_file1})
+        self.assertEqual(status.HTTP_200_OK, response1.status_code)
+
+
 class TestAsyncVideoFr(TestCase):
 
     def setUp(self):