Merge pull request #444 from pmeenan/wappalyzer-2021-10-19

Update Wappalyzer
catchpoint · Oct 20, 2021 · 51237a2 · 51237a2
2 parents 31e64fb + ccb502a
commit 51237a2
Show file tree

Hide file tree

Showing 35 changed files with 32,674 additions and 27,861 deletions.
diff --git a/internal/devtools.py b/internal/devtools.py
@@ -879,13 +879,15 @@ def send_command(self, method, params, wait=False, timeout=10, target_id=None):
             if wait:
                 self.pending_commands.append(command_id)
             end_time = monotonic() + timeout
-            self.send_command('Target.sendMessageToTarget',
+            target_response = self.send_command('Target.sendMessageToTarget',
                               {'targetId': target_id, 'message': json.dumps(msg)},
                               wait=wait, timeout=timeout)
             if wait:
                 if command_id in self.command_responses:
                     ret = self.command_responses[command_id]
                     del self.command_responses[command_id]
+                elif target_response is None or 'error' in target_response:
+                    ret = target_response
                 else:
                     while ret is None and monotonic() < end_time:
                         try:

diff --git a/internal/devtools_browser.py b/internal/devtools_browser.py
@@ -988,10 +988,14 @@ def wappalyzer_script(self, response_headers, cookies):
                 with open(os.path.join(self.support_path, 'Wappalyzer', 'wappalyzer.js')) as f_in:
                     wappalyzer = f_in.read()
                 if wappalyzer is not None:
-                    json_data = None
-                    with open(os.path.join(self.support_path, 'Wappalyzer', 'technologies.json')) as f_in:
-                        json_data = f_in.read()
-                    if json is not None:
+                    technologies = {}
+                    categories = {}
+                    with io.open(os.path.join(self.support_path, 'Wappalyzer', 'categories.json'), 'r', encoding='utf-8') as f_in:
+                        categories = json.load(f_in)
+                    for filename in sorted(glob.glob(os.path.join(self.support_path, 'Wappalyzer', 'technologies', '*.json'))):
+                        with io.open(filename, 'r', encoding='utf-8') as f_in:
+                            technologies.update(json.load(f_in))
+                    if technologies and categories:
                         # Format the headers as a dictionary of lists
                         headers = {}
                         if response_headers is not None:
@@ -1017,9 +1021,10 @@ def wappalyzer_script(self, response_headers, cookies):
                                             headers[key] = []
                                         headers[key].append(value)
                         script = script.replace('%WAPPALYZER%', wappalyzer)
-                        script = script.replace('%JSON%', json_data)
                         script = script.replace('%COOKIES%', json.dumps(cookies))
                         script = script.replace('%RESPONSE_HEADERS%', json.dumps(headers))
+                        script = script.replace('%CATEGORIES%', json.dumps(categories))
+                        script = script.replace('%TECHNOLOGIES%', json.dumps(technologies))
         except Exception:
             logging.exception('Error building wappalyzer script')
         return script