from __future__ import division, print_function import time, re, sys, os, ast, collections import pandas as pd import pymongo from pymongo import MongoClient from bson.son import SON from bson.codec_options import CodecOptions from collections import OrderedDict import re, csv from add_sugar_protein_site_class import add_sugar_protein_site_class from os import listdir class update_entity_info: def __init__(self, dbF, dbT, entF, entT, textF, textT): self.dbF = dbF self.dbT = dbT self.entF = entF self.entT = entT self.textF = textF self.textT = textT # --- create database instances--- # Environment variables mongodb_host = os.environ.get("MONGODB_HOST", "0.0.0.0") # change to biotm2.cis.udel.edu before dockerizing mongodb_port = os.environ.get("MONGODB_PORT", "27017") db_name = os.environ.get("DBNAME", self.dbT) # change database name for your own dbName textCollectionName = os.environ.get("COLLECTION_TEXT", self.textT) entityCollectionName = os.environ.get("COLLECTION_ENTITY", self.entT) # Database URI MONGODB_URI = 'mongodb://' + mongodb_host + ':' + mongodb_port + '/' # Database object client = MongoClient(MONGODB_URI) opts = CodecOptions(document_class=SON) # Database dbName = client[db_name] # glyco/unicarb # Collection self.textCollection = dbName[textCollectionName].with_options(codec_options=opts) self.entityCollection = dbName[entityCollectionName].with_options(codec_options=opts) self.ASPSC = add_sugar_protein_site_class(dbF, dbT, entF, entT, textF, textT) def update_acronym_normalization(self, pmid): entityDoc = self.entityCollection.find_one({"docId": str(pmid)}) raw_doc = self.textCollection.find_one({"docId": str(pmid)}) protein_list, site_list, sugar_list = self.ASPSC.get_entity_list_from_mongoDB_after(pmid) found_acronym = {} if not raw_doc: return None if entityDoc: for iter_i in range(2): for duid, entity in entityDoc["entity"].items(): # skip the normalized ones if len(entity["entityId"]) > 0: continue if entity["entityType"] == "Protein" or entity["entityType"] == "Gene" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: #print('sentenceIndex Not Found!') entityDict["sentenceIndex"]=0 entityDict["entityText"] = entity["entityText"] # detect the un-normalized acronym here found_normalization = False for pi in protein_list: # bet_str=raw_doc["text"][pi[2]:entityDict["charStart"]].encode('utf-8') # print('Distance:',entityDict["charStart"]-pi[2],'Between String:',bet_str) if 0 <= entityDict["charStart"] - pi[2] <= 3 and '(' in raw_doc["text"][ pi[2]:entityDict["charStart"]] \ and (pi[3] == "PUBTATOR" or pi[3] == "PUBTATORP") and pi[4] != '': found_normalization = True entityDict["source"] = "PUBTATORP" entityDict["entityId"] = [{"idString": pi[4], "source": pi[3]}] break if found_normalization: entityKey = "entity." + entityDict["duid"] #print('Updated acronym entities: ', entityDict) found_acronym[entityDict["entityText"]] = entityDict["entityId"][0]["idString"] rr = self.entityCollection.update_one({"docId": pmid}, {"$set": {entityKey: entityDict}}) #print('Update String:', # 'update_one({\"docId:\": \"' + pmid + '\"} ,{\"$set\": {' + entityKey + ':') #print(entityDict) #print("Successd number:", rr.modified_count) # update all the possible ancronym across the abstract for duid, entity in entityDoc["entity"].items(): # skip the normalized ones if len(entity["entityId"]) > 0: continue if entity["entityType"] == "Protein" or entity["entityType"] == "Gene" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: #print('sentenceIndex Not Found!') entityDict["sentenceIndex"]=0 entityDict["entityText"] = entity["entityText"] if entityDict["entityText"] in found_acronym: entityDict["entityId"] = [ {"idString": found_acronym[entityDict["entityText"]], "source": "Entrez"}] entityKey = "entity." + entityDict["duid"] #print('Updated acronym entities: ', entityDict) found_acronym[entityDict["entityText"]] = entityDict["entityId"][0]["idString"] self.entityCollection.update_one({"docId": pmid}, {"$set": {entityKey: entityDict}}) return None def extend_protein_normalization(self, pmid): entityDoc = self.entityCollection.find_one({"docId": str(pmid)}) raw_doc = self.textCollection.find_one({"docId": str(pmid)}) protein_list, site_list, sugar_list = self.ASPSC.get_entity_list_from_mongoDB_after(pmid) if not raw_doc: return None if entityDoc: for duid, entity in entityDoc["entity"].items(): # skip the normalized ones if len(entity["entityId"]) > 0: continue if entity["entityType"] == "Protein" or entity["entityType"] == "Gene" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: #print('sentenceIndex Not Found!') entityDict["sentenceIndex"]=0 entityDict["entityText"] = entity["entityText"] # detect the un-normalized protein here found_normalization = False for pi in protein_list: if entityDict["entityText"].find(pi[0]) >= 0 \ and pi[3] == "PUBTATOR" and pi[4] != '': found_normalization = True entityDict["entityId"] = [{"idString": pi[4], "source": pi[3]}] break if found_normalization: entityKey = "entity." + entityDict["duid"] #print('Updated protein entities: ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$set": {entityKey: entityDict}}) #print(rr) return None def remove_protein_is_part_of_sugar(self, pmid): entityDoc = self.entityCollection.find_one({"docId": str(pmid)}) raw_doc = self.textCollection.find_one({"docId": str(pmid)}) protein_list, site_list, sugar_list = self.ASPSC.get_entity_list_from_mongoDB_after(pmid) # print("protein_list:",protein_list) # print("sugar list:",sugar_list) if not raw_doc: return None found_sugar_overlap = {} if entityDoc: for duid, entity in entityDoc["entity"].items(): if entity["entityType"] == "Protein" or entity["entityType"] == "Gene" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: #print('sentenceIndex Not Found!') entityDict["sentenceIndex"]=0 entityDict["entityText"] = entity["entityText"] # detect the un-normalized protein here found_part_of_sugar = False for si in sugar_list: if (entityDict["charStart"] >= si[1] and entityDict["charStart"] <= si[2]) or \ (si[1] >= entityDict["charStart"] and si[1] <= entityDict["charEnd"]): found_part_of_sugar = True entityDict["entityId"] = [] entityDict["entityType"] = 'Sugar' found_sugar_overlap[entityDict["entityText"]] = 'Sugar' break if found_part_of_sugar: entityKey = "entity." + entityDict["duid"] #print('Removed entities(remove_protein_is_part_of_sugar): ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$unset": {entityKey: 1}}) # print(rr) # update other cases in the abstract ''' for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Protein" or entity["entityType"] == "Gene": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"]+len(entity["entityText"])-1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: print('sentenceIndex Not Found!') #entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] if entityDict["entityText"] in found_sugar_overlap: entityDict["entityId"]=[] entityDict["entityType"]='Sugar' entityKey = "entity." + entityDict["duid"] print('Removed entities (remove_protein_is_part_of_sugar): ',entityDict) rr=self.entityCollection.update_one({"docId":pmid},{ "$unset": { entityKey: 1}}) print(rr) ''' return None def remove_site_is_part_of_sugar(self, pmid): entityDoc = self.entityCollection.find_one({"docId": str(pmid)}) raw_doc = self.textCollection.find_one({"docId": str(pmid)}) protein_list, site_list, sugar_list = self.ASPSC.get_entity_list_from_mongoDB_after(pmid) # print("protein_list:",protein_list) # print("sugar list:",sugar_list) if not raw_doc: return None found_sugar_overlap = {} if entityDoc: for duid, entity in entityDoc["entity"].items(): if entity["entityType"] == "SiteSequence" or entity["entityType"] == "SpecificSite" or entity[ "entityType"] == "Site": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: #print('sentenceIndex Not Found!') entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] # detect the un-normalized protein here found_part_of_sugar = False for si in sugar_list: if (entityDict["charStart"] >= si[1] and entityDict["charStart"] <= si[2]) or \ (si[1] >= entityDict["charStart"] and si[1] <= entityDict["charEnd"]): found_part_of_sugar = True entityDict["entityId"] = [] entityDict["entityType"] = 'Sugar' found_sugar_overlap[entityDict["entityText"]] = 'Sugar' break if found_part_of_sugar: entityKey = "entity." + entityDict["duid"] #print('Removed entities: ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$unset": {entityKey: 1}}) #print(rr) # update other cases in the abstract ''' for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "SiteSequence" or entity["entityType"] == "SpecificSite" or entity["entityType"] == "Site": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"]+len(entity["entityText"])-1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: print('sentenceIndex Not Found!') #entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] if entityDict["entityText"] in found_sugar_overlap: entityDict["entityId"]=[] entityDict["entityType"]='Sugar' entityKey = "entity." + entityDict["duid"] print('Removed entities: ',entityDict) rr=self.entityCollection.update_one({"docId":pmid},{ "$unset": { entityKey: 1}}) print(rr) ''' return None def remove_protein_is_part_of_site(self, pmid): entityDoc = self.entityCollection.find_one({"docId": str(pmid)}) raw_doc = self.textCollection.find_one({"docId": str(pmid)}) protein_list, site_list, sugar_list = self.ASPSC.get_entity_list_from_mongoDB_after(pmid) # print("protein_list:",protein_list) # print("sugar list:",sugar_list) if not raw_doc: return None found_site_overlap = {} if entityDoc: for duid, entity in entityDoc["entity"].items(): if entity["entityType"] == "Protein" or entity["entityType"] == "Gene" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: #print('sentenceIndex Not Found!') entityDict["sentenceIndex"]=0 entityDict["entityText"] = entity["entityText"] # detect the un-normalized protein here found_part_of_site = False for si in site_list: if (entityDict["charStart"] >= si[1] and entityDict["charStart"] <= si[2]) or \ (si[1] >= entityDict["charStart"] and si[1] <= entityDict["charEnd"]): found_part_of_site = True entityDict["entityId"] = [] entityDict["entityType"] = 'Sugar' found_site_overlap[entityDict["entityText"]] = 'Site' break if found_part_of_site: entityKey = "entity." + entityDict["duid"] #print('Removed entities: ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$unset": {entityKey: 1}}) #print(rr) # update other cases in the abstract ''' for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Protein" or entity["entityType"] == "Gene": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"]+len(entity["entityText"])-1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: print('sentenceIndex Not Found!') #entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] if entityDict["entityText"] in found_site_overlap: entityDict["entityId"]=[] entityDict["entityType"]='Site' entityKey = "entity." + entityDict["duid"] print('Removed entities: ',entityDict) rr=self.entityCollection.update_one({"docId":pmid},{ "$unset": { entityKey: 1}}) print(rr) ''' return None def update_specific_protein_names(self, pmid): # there are several strange protein names # case in 9118998 # like 1 (mFA1, 1 (SCP-1 # case in 8349699 # like prostaglandin endoperoxide synthases-1 and -2 entityDoc = self.entityCollection.find_one({"docId": str(pmid)}) raw_doc = self.textCollection.find_one({"docId": str(pmid)}) protein_list, site_list, sugar_list = self.ASPSC.get_entity_list_from_mongoDB_after(pmid) # print("protein_list:",protein_list) # print("sugar list:",sugar_list) if not raw_doc: return None to_delete_entity = [] if entityDoc: for duid, entity in entityDoc["entity"].items(): if entity["entityType"] == "Gene" or entity["entityType"] == "Protein" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: entityDict["sentenceIndex"] = 0 #print('sentenceIndex Not Found!') # entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] if entityDict["entityText"].startswith('1 ('): entityKey = "entity." + entityDict["duid"] entityDict["charStart"] = entity["charStart"] + 3 entityDict["entityText"] = entity["entityText"][3:] #print('Updated entities: ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$set": {entityKey: entityDict}}) elif entityDict["entityText"].endswith(' and -2'): entityKey = "entity." + entityDict["duid"] entityDict["charEnd"] = entityDict["charStart"] - 7 entityDict["entityText"] = entity["entityText"][:-7] #print('Updated entities: ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$set": {entityKey: entityDict}}) if entityDict["source"] == 'PUBTATOR' or entityDict["source"] == 'OGER' or \ len(entityDict["entityId"]) > 0: for pi in protein_list: if (pi[1] - entityDict["charEnd"]) == 2 and raw_doc["text"][pi[1] - 1] == ' ': entityDict["charEnd"] = pi[2] entityDict["entityText"] = entityDict["entityText"] + ' ' + pi[0] #print('Updated entities: ', entityDict) entityKey = "entity." + entityDict["duid"] rr = self.entityCollection.update_one({"docId": pmid}, {"$set": {entityKey: entityDict}}) to_delete_entity.append(pi) break for duid, entity in entityDoc["entity"].items(): if entity["entityType"] == "Gene" or entity["entityType"] == "Protein" \ or entity["entityType"] == "Protein-Complex": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"] + len(entity["entityText"]) - 1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] entityDict["entityId"] = entity["entityId"] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: entityDict["sentenceIndex"] = 0 #print('sentenceIndex Not Found!') # entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] delete_sign = False for di in to_delete_entity: if di[1] == entityDict["charStart"] and di[2] == entityDict["charEnd"]: delete_sign = True break if delete_sign: entityKey = "entity." + entityDict["duid"] #print('Removed entities: ', entityDict) rr = self.entityCollection.update_one({"docId": pmid}, {"$unset": {entityKey: 1}}) return None if __name__ == '__main__': # test pmidFile = sys.argv[1] dbF = sys.argv[2] dbT = sys.argv[3] colF = sys.argv[4] colT = sys.argv[5] textT = sys.argv[6] edg = sys.argv[7] ''' pmidFile='test.txt' dbF='pubtator' dbT='glygen_test_new' colF='medline.aligned' colT='entities' textT='text' edg='glygen_testset_091520.tsv' ''' # ASPSC=add_sugar_protein_site_class(dbF,dbT,colF,colT,textT,textT) UEI = update_entity_info(dbF, dbT, colF, colT, textT, textT) # UEI=update_entity_info("pubtator","unicarb","medline.aligned","entities","text","text") # pmidFile='unicarb.txt' pmidList = pd.read_csv(pmidFile, header=None).iloc[:, 0].tolist() # edg_file='unicarb.tsv' pmidList = list(set(pmidList)) # pmidList=['19864504'] count = 0 for pi in pmidList: print(pi) UEI.update_acronym_normalization(str(pi)) for pi in pmidList: print(pi) UEI.extend_protein_normalization(str(pi)) for pi in pmidList: print(pi) UEI.remove_protein_is_part_of_sugar(str(pi)) for pi in pmidList: UEI.remove_site_is_part_of_sugar(str(pi)) for pi in pmidList: UEI.remove_protein_is_part_of_site(str(pi)) for pi in pmidList: UEI.update_specific_protein_names(str(pi))