Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor and logging #540

Merged
merged 6 commits into from
Dec 13, 2023
Merged

Refactor and logging #540

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
707cc61
Refactor and logging
MarkDWilliams Dec 13, 2023
9083721
Merge branch 'master' into messageLogging
MarkDWilliams Dec 13, 2023
70a590f
adding check for [null]
MarkDWilliams Dec 13, 2023
baa2c9a
Merge branch 'messageLogging' of github.com:NCATSTranslator/Relay int…
MarkDWilliams Dec 13, 2023
d09c4f2
made some minor changes on the logs
Dec 13, 2023
8bf45d3
changed the annotator URL to point to the prod env
Dec 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 27 additions & 14 deletions tr_sys/tr_ars/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import logging
import traceback
import os
from datetime import time
from datetime import time, datetime
from django.db import transaction
import requests
import statistics
Expand Down Expand Up @@ -31,7 +31,6 @@
}

NORMALIZER_URL=os.getenv("TR_NORMALIZER") if os.getenv("TR_NORMALIZER") is not None else "https://nodenormalization-sri.renci.org/1.4/get_normalized_nodes"
#ANNOTATOR_URL=os.getenv("TR_ANNOTATOR") if os.getenv("TR_ANNOTATOR") is not None else "https://biothings.ci.transltr.io/annotator/"
ANNOTATOR_URL=os.getenv("TR_ANNOTATOR") if os.getenv("TR_ANNOTATOR") is not None else "https://biothings.ncats.io/annotator/"
APPRAISER_URL=os.getenv("TR_APPRAISE") if os.getenv("TR_APPRAISE") is not None else "http://localhost:9096/get_appraisal"

Expand Down Expand Up @@ -735,20 +734,40 @@ def scrub_null_attributes(data):
nodeAttributes.remove(None)

if edges is not None:
bad_sources = []
for edgeId, edgeStuff in edges.items():
edgeAttributes =get_safe(edgeStuff,"attributes")
if edgeAttributes is not None:
while None in edgeAttributes:
edgeAttributes.remove(None)

edgeSources=get_safe(edgeStuff, "sources")
sources_to_remove = {}
for edge_source in edgeSources:
if 'resource_id' in edge_source.keys() and edge_source["resource_id"] is None:
logging.info('found Null in resource_id : %s' % (edge_source["resource_id"]))
edgeSources.remove(edge_source)
elif 'upstream_resource_ids' in edge_source.keys() and edge_source["upstream_resource_ids"] is None:
logging.info('found Null in upstream_resource_ids : %s' % (edge_source["upstream_resource_ids"]))
edgeSources.remove(edge_source)
if 'resource_id' not in edge_source.keys() or edge_source["resource_id"] is None:
logging.info('found Null in resource_id : %s' % (edge_source))
if edgeId not in sources_to_remove.keys():
sources_to_remove[edgeId]=[edge_source]
else:
sources_to_remove[edgeId].append(edge_source)
elif 'upstream_resource_id' not in edge_source.keys() or ('upstream_resource_ids' in edge_source.keys() and edge_source["upstream_resource_ids"] is None)\
or edge_source["upstream_resource_ids"] is [None]:
logging.info('found Null in upstream_resource_ids : %s' % (edge_source))
if edgeId not in sources_to_remove.keys():
sources_to_remove[edgeId]=[edge_source]
else:
sources_to_remove[edgeId].append(edge_source)
if len(sources_to_remove)>0:
bad_sources.append(sources_to_remove)
for key, sources in sources_to_remove.items():
for source in sources:
edgeSources.remove(source)
log_tuple =[
"Removed the following bad sources: "+ str(bad_sources),
datetime.now().strftime('%H:%M:%S'),
"DEBUG"
]
add_log_entry(data,log_tuple)


def appraise(mesg,data, agent_name,retry_counter=0):
Expand Down Expand Up @@ -888,17 +907,11 @@ def decorate_edges_with_infores(data,inforesid):
else:
bad_sources=[]
for source in edge['sources']:
if 'resource_id' not in source.keys():
bad_sources.append(source)
#if the source is no good, we don't need to do anything else with it.
continue
if source['resource_id']==inforesid:
has_self=True

if source['resource_role']=="primary_knowledge_source":
has_primary=True
for source in bad_sources:
edge['sources'].remove(source)
if not has_self:
#if we already have a primary knowledge source but not our self, we add ourself as an aggregator
if has_primary:
Expand Down