diff --git a/bazaar/contrib/sites/migrations/0004_alter_site_options.py b/bazaar/contrib/sites/migrations/0004_alter_site_options.py new file mode 100644 index 0000000..558ab06 --- /dev/null +++ b/bazaar/contrib/sites/migrations/0004_alter_site_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-11-24 15:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('sites', '0003_set_site_domain_and_name'), + ] + + operations = [ + migrations.AlterModelOptions( + name='site', + options={'ordering': ['domain'], 'verbose_name': 'site', 'verbose_name_plural': 'sites'}, + ), + ] diff --git a/bazaar/core/migrations/0005_auto_20211124_1534.py b/bazaar/core/migrations/0005_auto_20211124_1534.py new file mode 100644 index 0000000..68b3a29 --- /dev/null +++ b/bazaar/core/migrations/0005_auto_20211124_1534.py @@ -0,0 +1,36 @@ +# Generated by Django 3.2.4 on 2021-11-24 15:34 + +import bazaar.core.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('core', '0004_auto_20210305_1346'), + ] + + operations = [ + migrations.AlterField( + model_name='yara', + name='content', + field=models.TextField(help_text='Paste the content of your Yara rule', validators=[bazaar.core.models.check_yara_rule]), + ), + migrations.AlterField( + model_name='yara', + name='title', + field=models.CharField(help_text='The fancy name of your Yara rule', max_length=256), + ), + migrations.CreateModel( + name='Bookmark', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('sample', models.CharField(max_length=256)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/bazaar/core/migrations/0006_merge_0005_auto_20211124_1534_0005_auto_20220125_1346.py b/bazaar/core/migrations/0006_merge_0005_auto_20211124_1534_0005_auto_20220125_1346.py new file mode 100644 index 0000000..5087b67 --- /dev/null +++ b/bazaar/core/migrations/0006_merge_0005_auto_20211124_1534_0005_auto_20220125_1346.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.4 on 2022-04-29 16:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0005_auto_20211124_1534'), + ('core', '0005_auto_20220125_1346'), + ] + + operations = [ + ] diff --git a/bazaar/core/models.py b/bazaar/core/models.py index c48b864..7e4b4dc 100644 --- a/bazaar/core/models.py +++ b/bazaar/core/models.py @@ -1,12 +1,24 @@ +import uuid + import yara from django.conf import settings from django.core.exceptions import ValidationError from django.db import models -import uuid - +from django.db.models.expressions import F from elasticsearch import Elasticsearch +class Bookmark(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + sample = models.CharField(max_length=256) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + + @staticmethod + def get_es_index(user): + es_index = f'bookmark_user_{user.id}' + return es_index + + def check_yara_rule(rule): try: yara.compile(source=rule) diff --git a/bazaar/front/urls.py b/bazaar/front/urls.py index 57a67ef..423f9e4 100644 --- a/bazaar/front/urls.py +++ b/bazaar/front/urls.py @@ -1,27 +1,46 @@ from django.urls import path from django.views.generic.base import TemplateView -from bazaar.front.view import HomeView, ReportView, basic_upload_view, similarity_search_view, export_report_view, \ - download_sample_view, my_rules_view, my_rule_edit_view, my_rule_create_view, my_rule_delete_view, og_card_view, \ - my_retrohunt_view, get_andgrocfg_code, get_genom, basic_url_download_view +from bazaar.front.view import ( + HomeView, + ReportView, + add_bookmark_sample_view, + basic_upload_view, + basic_url_download_view, + download_sample_view, + export_report_view, + get_andgrocfg_code, + get_genom, + my_retrohunt_view, + my_rule_create_view, + my_rule_delete_view, + my_rule_edit_view, + og_card_view, + remove_bookmark_sample_view, + similarity_search_view, + workspace_view, +) app_name = "front" urlpatterns = [ path("", view=HomeView.as_view(), name="home"), path("robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain")), - path("report/", view=ReportView.as_view(), name="report"), - path("report//json", view=export_report_view, name="export_report"), - path("report//card", view=og_card_view, name="og_card"), + path("androcfg//", view=get_andgrocfg_code, name="get_andgrocfg_code"), + path("androcfg/all", view=get_genom, name="get_genom"), path("apk/", view=basic_upload_view, name="basic_upload"), - path("url/", view=basic_url_download_view, name="basic_url_download"), path("apk/", view=download_sample_view, name="download_sample"), - path("similar/", view=similarity_search_view, name="similarity_search"), - path("similar/", view=similarity_search_view, name="similarity_search"), - path("rules/", view=my_rules_view, name="my_rules"), - path("rules/new", view=my_rule_create_view, name="my_rule_create"), - path("rules//edit", view=my_rule_edit_view, name="my_rule_edit"), + path("apk//bookmark/add", view=add_bookmark_sample_view, name="add_bookmark_sample"), + path("apk//bookmark/remove", view=remove_bookmark_sample_view, + name="remove_bookmark_sample"), + path("report/", view=ReportView.as_view(), name="report"), + path("report//card", view=og_card_view, name="og_card"), + path("report//json", view=export_report_view, name="export_report"), path("rules//delete", view=my_rule_delete_view, name="my_rule_delete"), + path("rules//edit", view=my_rule_edit_view, name="my_rule_edit"), path("rules//retro", view=my_retrohunt_view, name="my_rule_retro"), - path("androcfg//", view=get_andgrocfg_code, name="get_andgrocfg_code"), - path("androcfg/all", view=get_genom, name="get_genom") + path("rules/new", view=my_rule_create_view, name="my_rule_create"), + path("similar/", view=similarity_search_view, name="similarity_search"), + path("similar/", view=similarity_search_view, name="similarity_search"), + path("url/", view=basic_url_download_view, name="basic_url_download"), + path("workspace/", view=workspace_view, name="workspace") ] diff --git a/bazaar/front/view.py b/bazaar/front/view.py index da619fe..5a5a944 100644 --- a/bazaar/front/view.py +++ b/bazaar/front/view.py @@ -1,16 +1,18 @@ +import hashlib import logging +from collections import Counter from tempfile import NamedTemporaryFile -import requests -import hashlib +import requests +from androcfg.code_style import U39bStyle from androguard.core.androconf import is_android from django.conf import settings from django.contrib import messages from django.core.cache import cache from django.core.files.storage import default_storage -from django.http import JsonResponse, HttpResponse +from django.http import HttpResponse, JsonResponse from django.http.response import HttpResponseBadRequest -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render from django.utils import timezone from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page @@ -19,21 +21,32 @@ from django_q.tasks import async_task from elasticsearch import Elasticsearch from elasticsearch.helpers.actions import scan -from rest_framework.authtoken.models import Token -from rest_framework.reverse import reverse_lazy from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers.jvm import JavaLexer -from androcfg.code_style import U39bStyle - +from rest_framework.authtoken.models import Token +from rest_framework.reverse import reverse_lazy -from bazaar.core.models import Yara +from bazaar.core.models import Bookmark, Yara from bazaar.core.tasks import analyze, retrohunt -from bazaar.core.utils import get_sha256_of_file, get_matching_items_by_dexofuzzy -from bazaar.front.forms import SearchForm, BasicUploadForm, SimilaritySearchForm, BasicUrlDownloadForm +from bazaar.core.utils import get_matching_items_by_dexofuzzy, get_sha256_of_file +from bazaar.front.forms import ( + BasicUploadForm, + BasicUrlDownloadForm, + SearchForm, + SimilaritySearchForm, +) from bazaar.front.og import generate_og_card -from bazaar.front.utils import transform_results, get_similarity_matrix, compute_status, generate_world_map, \ - transform_hl_results, get_sample_timeline, get_andro_cfg_storage_path +from bazaar.front.utils import ( + compute_status, + generate_world_map, + get_andro_cfg_storage_path, + get_sample_timeline, + get_similarity_matrix, + transform_hl_results, + transform_results, +) + from .forms import YaraCreateForm @@ -67,6 +80,7 @@ def get(self, request, *args, **kwargs): f = SearchForm(request.GET) form_to_show = f + if not request.GET: form_to_show = SearchForm() if f.is_valid(): @@ -86,7 +100,7 @@ def get(self, request, *args, **kwargs): 'list_results': list_results, 'report_example': report_example, 'q': q, 'matrix': matrix, - 'max_size': settings.MAX_APK_UPLOAD_SIZE + 'max_size': settings.MAX_APK_UPLOAD_SIZE, }) @@ -154,6 +168,12 @@ def get(self, request, *args, **kwargs): # Get timeline timeline = get_sample_timeline(sha) + # Check if the sample is already bookmarked + if request.user.is_authenticated: + is_bookmarked = get_user_bookmark_by_hash(request, sha) + else: + is_bookmarked = False + return render(request, 'front/report.html', { 'result': result, 'status': status, @@ -161,8 +181,9 @@ def get(self, request, *args, **kwargs): 'timeline': timeline, 'cache_key': f'{cache_key}_tpl', 'hunting_matches': hunting_matches, - 'similar_samples': similar_samples_extended, - 'cache_retention_time': cache_retention_time}) + 'similar_samples': similar_samples, + 'cache_retention_time': cache_retention_time, + 'is_bookmarked': is_bookmarked}) except Exception as e: logging.exception(e) return redirect(reverse_lazy('front:home')) @@ -290,18 +311,122 @@ def og_card_view(request, sha256): return HttpResponse(fp.read(), content_type="image/png") -def my_rules_view(request): +def get_user_bookmarks(request): + return Bookmark.objects.filter(owner=request.user) + + +def enrich_bookmarks(bookmarks): + res = [] + for sample in bookmarks: + apk = get_sample_light(sample.sample) + res.append(apk) + return res + + +def get_user_bookmark_by_hash(request, sha): + bookmark = Bookmark.objects.filter(owner=request.user, sample=sha) + return bookmark + + +def add_bookmark_sample_view(request, sha256): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method == 'GET': + user_bookmarks = get_user_bookmarks(request) + if not user_bookmarks.filter(sample=sha256): + new_bookmark = Bookmark.objects.create(sample=sha256, owner=request.user) + try: + new_bookmark.save() + user_bookmarks = get_user_bookmarks(request) + except Exception as e: + logging.exception(e) + + return redirect(reverse_lazy('front:report', [sha256])) + + +def remove_bookmark_sample_view(request, sha256): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method == 'GET': + bookmark = get_user_bookmark_by_hash(request, sha256) + try: + bookmark.delete() + except Exception as e: + logging.exception(e) + + return redirect(reverse_lazy('front:report', [sha256])) + + +def get_last_samples(): + query = { + "query": { + "query_string": { + "default_field": "sha256", + "query": "*" + } + }, + "highlight": { + "fields": { + "*": {"pre_tags": [""], "post_tags": [""]} + } + }, + "aggs": { + "permissions": { + "terms": {"field": "permissions.keyword"} + }, + "domains": { + "terms": {"field": "domains_analysis._name.keyword"} + }, + "android_features": { + "terms": {"field": "features.keyword"} + } + }, + "sort": {"analysis_date": "desc"}, + "_source": ["apk_hash", "sha256", "uploaded_at", "icon_base64", "handle", "app_name", + "version_code", "size", "dexofuzzy.apk", "quark.threat_level", "vt", "vt_report", "malware_bazaar", + "is_signed", "frosting_data.is_frosted", "features", "andro_cfg.genom"], + "size": 5, + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + raw_results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query) + results = transform_results(raw_results) + return results + except Exception as e: + return [] + + +def workspace_view(request): if not request.user.is_authenticated: return redirect(reverse_lazy('front:home')) my_rules = None + my_bookmarks = None if request.method == 'GET': + # TODO: refactor so there is only one call for all the rules, then filter by user and by score and output those in the tables + apk_count = get_count_apks() + malicious = get_malicious_samples_count() + count = { + "apks": apk_count, + "malicious": malicious + } my_rules = get_rules(request) + my_bookmarks = enrich_bookmarks(get_user_bookmarks(request)) + last_samples = get_last_samples() + highest_matching_rules = get_best_rules() + trending_malwares = get_trending_malware() + trends = { + 'samples': last_samples, 'rules': highest_matching_rules, 'trending_malwares': trending_malwares + } owner = request.user token, _ = Token.objects.get_or_create(user=owner) - - return render(request, 'front/yara_rules/my_rules.html', context={'my_rules': my_rules, 'my_token': token.key}) + return render(request, 'front/workspace/workspace_base.html', + context={'my_rules': my_rules, 'my_token': token, + 'bookmarked_samples': my_bookmarks, 'trends': trends, + 'count': count}) def my_rule_create_view(request): @@ -319,10 +444,10 @@ def my_rule_create_view(request): new_rule.save() messages.success(request, 'Your rule has been created!') except Exception: - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule}) - return redirect(reverse_lazy('front:my_rules')) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule}) + return redirect(reverse_lazy('front:workspace')) - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule}) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule}) def my_rule_edit_view(request, uuid): @@ -332,7 +457,7 @@ def my_rule_edit_view(request, uuid): if request.method == 'GET': rule = Yara.objects.get(id=uuid) new_rule = YaraCreateForm(instance=rule) - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule, 'edit': True}) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule, 'edit': True}) elif request.method == 'POST': rule = Yara.objects.get(id=uuid) @@ -345,8 +470,8 @@ def my_rule_edit_view(request, uuid): delete_es_matches(request, rule) messages.success(request, 'Your rule has been updated!') except Exception: - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule}) - return redirect(reverse_lazy('front:my_rules')) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule}) + return redirect(reverse_lazy('front:workspace')) else: return HttpResponseBadRequest() @@ -361,10 +486,10 @@ def my_rule_delete_view(request, uuid=None): delete_es_matches(request, rule) rule.delete() messages.success(request, 'Your rule has been deleted.') - return redirect(reverse_lazy('front:my_rules')) + return redirect(reverse_lazy('front:workspace')) except Exception as e: logging.exception(e) - return redirect(reverse_lazy('front:my_rules')) + return redirect(reverse_lazy('front:workspace')) def delete_es_matches(request, rule): @@ -391,6 +516,170 @@ def delete_es_matches(request, rule): return +def get_count_apks(): + query = { + "aggs": { + "type_value": { + "value_count": { + "field": "apk_hash.keyword" + } + } + } + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + + try: + count = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query)['aggregations']['type_value']['value'] + return count + except: + return -1 + + +def get_trending_malware(): + query = { + "query": { + "bool": { + "must": [], + "filter": [ + { + "bool": { + "should": [ + { + "range": { + "vt_report.attributes.popular_threat_classification.popular_threat_name.count": { + "gt": 1 + } + } + } + ], + "minimum_should_match": 1 + } + } + ], + "should": [], + "must_not": [] + } + }, + "size": 5000, + "_source": "vt_report.attributes.popular_threat_classification.popular_threat_name.value", + "sort": [ + { + "vt_report.attributes.popular_threat_classification.popular_threat_name.count": { + "order": "desc" + } + } + ] + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query)['hits']['hits'] + return Counter(value['_source']['vt_report']['attributes'] + ['popular_threat_classification']['popular_threat_name'][0]['value'] for value in results).most_common(5) + + except Exception: + return [] + + +def get_malicious_samples_count(): + query = { + "query": { + "bool": { + "must": [], + "filter": [ + { + "bool": { + "should": [ + { + "range": { + "vt.malicious": { + "gt": 1 + } + } + } + ], + "minimum_should_match": 1 + } + } + ], + "should": [], + "must_not": [] + } + }, + "size": 5000, + "_source": "vt", + "sort": [ + { + "vt.malicious": { + "order": "desc" + } + } + ] + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query)['hits']['hits'] + res = Counter(value['_source']['vt']['malicious'] for value in results) + # FIXME: total() doesn't seem to work, idk why + return sum(res.values()) + except: + return -1 + + +def get_best_rules(): + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + yara_rules = Yara.objects.all() + my_rules = [] + public_es_index, _ = Yara.get_es_index_names() + # TODO: note fo later: in an ideal world, it would be nice to have the matches.matching_files be set on keyword=true for Elastic search so we can retreive the result sorted instead of sorting them on our own. See: https://stackoverflow.com/questions/63784500/text-fields-are-not-optimised-for-operations-that-require-per-document-elastic + q = { + "query": { + "bool": { + "must": [], + "filter": [ + { + "bool": { + "should": [ + { + "range": { + "matches.matching_files": { + "gt": 1 + } + } + } + ], + "minimum_should_match": 1 + } + } + ], + "should": [], + "must_not": [] + } + }, + "size": 5000, + } + public_matches = None + try: + public_matches = es.search(index=public_es_index, body=q)['hits']['hits'] + for rule in yara_rules: + my_rule = { + 'rule': rule, + 'matching_date': '', + 'matches': [], + } + if public_matches: + for match in public_matches: + if match['_source']['rule'] == str(rule.id): + m = match['_source'] + m['sample'] = get_sample_light(match['_source']['matches']['apk_id']) + my_rule['matches'].append(m) + if len(my_rule['matches']) > 1: + my_rules.append(my_rule) + + return my_rules + except: + return [] + + def get_rules(request): es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) yara_rules = Yara.objects.filter(owner=request.user) @@ -403,7 +692,6 @@ def get_rules(request): }, 'size': 5000, } - public_matches, private_matches = None, None try: private_matches = es.search(index=private_es_index, body=q)['hits']['hits'] @@ -468,7 +756,7 @@ def my_retrohunt_view(request, uuid): except Exception as e: logging.exception(e) - return redirect(reverse_lazy('front:my_rules')) + return redirect(reverse_lazy('front:workspace')) def get_andgrocfg_code(request, sha256, foo): diff --git a/bazaar/templates/base.html b/bazaar/templates/base.html index ff46850..c414491 100644 --- a/bazaar/templates/base.html +++ b/bazaar/templates/base.html @@ -39,7 +39,7 @@ {% url 'front:home' as home_url %} {% url 'about' as about_url %} -{% url 'front:my_rules' as my_rules_url %} +{% url 'front:workspace' as workspace_url %} {% url 'account_logout' as account_logout_url %} {% url 'account_login' as account_login_url %} @@ -63,9 +63,9 @@ {% if request.user.is_authenticated %} -
  • - - {% trans "My hunting" %} +
  • + + {% trans "Workspace" %}