init

Config.py

@@ -0,0 +1,4 @@
+"""
+    simplest xpath web scraper
+    @author: Asiri Hewage
+"""

Data.py

@@ -0,0 +1,27 @@
+"""
+    simplest xpath web scraper
+    @author: Asiri Hewage
+"""
+
+data = [
+    {
+        "url": "https://www.technology.pitt.edu/blog/zoom10faq",
+        "xpaths": [
+            {
+                "question": '//div[@class="field-item even"]/h2/text()',
+                "answers": '//div[@class="field-item even"]/p/text()',
+                "correct_answer": '//div[@class="field-item even"]/p[0]/text()'
+            }
+        ]
+    },
+    {
+        "url": "https://www.socialsciencespace.com/2020/03/16-answers-to-your-questions-about-teaching-online/",
+        "xpaths": [
+            {
+                "column_name1": "/img",
+                "column_name2": "/p",
+                "column_name3": "/a"
+            }
+        ]
+    }
+]

Database.py

@@ -0,0 +1,17 @@
+"""
+    simplest xpath web scraper
+    @author: Asiri Hewage
+"""
+
+import pymongo
+
+
+class Database:
+    def __init__(self):
+        self.myclient = pymongo.MongoClient("mongodb://localhost:27017/")
+        self.mydb = self.myclient["mydatabase"]
+        self.mycol = self.mydb["customers"]
+
+    def insert(self, data):
+        x = self.mycol.insert_one(data)
+        return x

Extractor.py

@@ -0,0 +1,37 @@
+"""
+    simplest xpath web scraper
+    @author: Asiri Hewage
+"""
+from lxml import html
+import requests
+
+
+class Extractor:
+    def __init__(self):
+        self.r = None
+        self.tree = None
+        self.xpaths = None
+
+    def extract(self, url, xpaths):
+        """
+        :return:
+        """
+        res = []
+        data = []
+        ret = {}
+
+        self.r = requests.get(url)
+        self.tree = html.fromstring(self.r.content)
+
+        for key, obj in enumerate(xpaths):
+            for column_name, xpath in obj.items():
+                value = self.tree.xpath(xpath)
+                res = {
+                    column_name: value
+                }
+                data.append(res)
+
+            ret["url"] = url
+            ret["data"] = data
+
+        return ret

README.md

@@ -0,0 +1,41 @@
+# Simplest xpath web scraper
+Simples web scraper created using Python3
+- extract data using multiple xpaths from multiple urls
+- save response in MongoDB
+- exceptions and error handling
+- only for basic web sraping work from static HTML web pages
+
+## setup Data.py for each url with xpath
+```json
+    {
+        "url": "https://www.technology.pitt.edu/blog/zoom10faq",
+        "xpaths": [
+            {
+                "questions": '//div[@class="field-item even"]/h2/text()',
+                "answers": '//div[@class="field-item even"]/p/text()',
+                "correct_answer": '//div[@class="field-item even"]/p[0]/text()'
+            }
+        ]
+    }
+```
+## setup mongodb database connection string
+```python
+myclient = pymongo.MongoClient("mongodb://host:port/") # or add the connection url
+mydb = myclient["database"]
+mycol = mydb["collection"]
+```
+
+## install python dependancies
+```commandline
+pip3 install -r requirements.txt
+```
+
+## run
+```commandline
+python3 main.py
+```
+
+## response
+ ![Simplest xpath web scraper](47dcf6e5-0d63-4824-9135-e2b4171a171f.jfif)
+
+### Author : Asiri Hewage

main.py

@@ -0,0 +1,23 @@
+"""
+    simplest xpath web scraper
+    @author: Asiri Hewage
+"""
+from Extractor import Extractor
+from Data import data
+# from Database import Database
+
+
+def run():
+    try:
+        extractor = Extractor()
+        # database = Database()
+        for obj in data:
+            res = extractor.extract(obj["url"], obj["xpaths"])
+            # database.insert(res)
+            print(res)
+    except Exception as er:
+        print(er)
+
+
+if __name__ == '__main__':
+    run()

requirements.txt

@@ -0,0 +1,3 @@
+requests~=2.26.0
+lxml~=4.6.3
+pymongo