from __future__ import division,print_function import time,re,sys,os,ast,json import pandas as pd from os import path import pymongo from collections import OrderedDict from pymongo import MongoClient from bson.son import SON from bson.codec_options import CodecOptions import re,csv from detect_polysachharide import detect_polysachharide import codecs from nltk.tokenize import word_tokenize from nltk.corpus import words from np_generator import np_generator_for_abstrat class add_sugar_protein_site_class: def __init__(self,dbF,dbT,entF,entT,textF,textT): self.dbF=dbF self.dbT=dbT self.entF=entF self.entT=entT self.textF=textF self.textT=textT #for checking c-term self.dict= words.words() #for np dic if path.exists('glygen_set_abstract_np.json'): self.abstract_np_file='glygen_set_abstract_np.json' with open(self.abstract_np_file) as jfile: self.abstract_np_dic=json.load(jfile) else: self.abstract_np_dic={} #for the OGER detected entities self.oger_json_file='' self.oger_json_file_cp='' self.oger_json_file_complex='' if path.exists('glygen_set_oger_entity.json'): self.oger_json_file='glygen_set_oger_entity.json' if path.exists('glygen_set_oger_entity_cp.json'): self.oger_json_file_cp='glygen_set_oger_entity_cp.json' if path.exists('glygen_set_oger_entity_complex.json'): self.oger_json_file_complex='glygen_set_oger_entity_complex.json' #--- create database instances--- # Environment variables mongodb_host = os.environ.get("MONGODB_HOST","0.0.0.0") # change to biotm2.cis.udel.edu before dockerizing mongodb_port = os.environ.get("MONGODB_PORT","27017") db_name = os.environ.get("DBNAME_TO",self.dbT) # change database name for your own dbName textCollectionName = os.environ.get("COLLECTION_TO_TEXT",self.textT) entityCollectionName = os.environ.get("COLLECTION_TO_ENTITY",self.entT) # Database URI MONGODB_URI = 'mongodb://'+mongodb_host+':'+mongodb_port+'/' # Database object client = MongoClient(MONGODB_URI) opts = CodecOptions(document_class=SON) # Database dbName = client[db_name] # glyco/unicarb # Collection self.textCollection = dbName[textCollectionName].with_options(codec_options=opts) self.entityCollection = dbName[entityCollectionName].with_options(codec_options=opts) self.protein_f_term=['proteins?','enzymes?','receptors?', \ 'channels?','polypeptides?','isoforms?', \ 'precursors?','dimers?','trimers?','plasminogens?',\ 'fragments?','variants?','factors?','subunits?'] #output_file='GDS_Detection.csv' self.gds_file_column=['pmid','sent_index','gds','head_word','extension_word',\ 'adj_ylated','adj_type','adj_antenna','adj_link','adj_motif',\ 'is_conjunction','is_preposition','is_ING','is_in_html'] ''' with open(output_file, 'a') as csvfile: spamwriter = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) title_row=[ri.encode('utf-8') for ri in self.gds_file_column ] spamwriter.writerow(title_row) ''' def get_entity_list_from_mongoDB_before(self,pmid): protein_list=[] site_list=[] sugar_list=[] entityDoc = self.entityCollection.find_one({"docId":str(pmid)}) if entityDoc: for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Gene": protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"])) if entity["entityType"] == "MPS": sugar_list.append((entity["entityText"],entity["charStart"],entity["charEnd"])) if entity["entityType"] == "AminoAcid": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"])) if entity["entityType"] == "SiteSequence": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"])) if entity["entityType"] == "SpecificSite": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"])) return protein_list,site_list,sugar_list def get_entity_list_from_mongoDB_after(self,pmid): protein_list=[] site_list=[] sugar_list=[] entityDoc = self.entityCollection.find_one({"docId":str(pmid)}) if entityDoc: for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Gene": if len(entity["entityId"])>0: protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],entity["entityId"][0]["idString"],entity["duid"])) else: protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],'',entity["duid"])) if entity["entityType"] == "Protein" or entity["entityType"] == "Protein-Complex" \ or entity["entityType"] == "Complex": if len(entity["entityId"])>0: protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],entity["entityId"][0]["idString"],entity["duid"])) else: protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],'',entity["duid"])) if entity["entityType"] == "Sugar": sugar_list.append((entity["entityText"],entity["charStart"],entity["charEnd"])) if entity["entityType"] == "AminoAcid": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["entityType"])) if entity["entityType"] == "SiteSequence": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["entityType"])) if entity["entityType"] == "SpecificSite": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["entityType"])) if entity["entityType"] == "Site": site_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["entityType"])) return protein_list,site_list,sugar_list def get_complex_protein_list_from_mongoDB(self,pmid): complex_protein_list=[] entityDoc = self.entityCollection.find_one({"docId":str(pmid)}) if entityDoc: for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Protein-Complex" \ or entity["entityType"] == "Complex": if len(entity["entityId"])>0: complex_protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],entity["entityId"][0]["idString"],entity["entityType"])) else: complex_protein_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],'',entity["entityType"])) return complex_protein_list def get_species_list_from_mongoDB(self,pmid): species_list=[] entityDoc = self.entityCollection.find_one({"docId":str(pmid)}) if entityDoc: for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Species": if len(entity["entityId"])>0: species_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],entity["entityId"][0]["idString"])) else: species_list.append((entity["entityText"],entity["charStart"],entity["charEnd"],entity["source"],'')) return species_list def get_abstract_text(self,pmid): raw_doc = self.textCollection.find_one({"docId":str(pmid)}) return raw_doc def extend_protein_names(self,pmid): entityDoc = self.entityCollection.find_one({"docId":str(pmid)}) raw_doc = self.textCollection.find_one({"docId":str(pmid)}) if not raw_doc: return None if entityDoc: for duid,entity in entityDoc["entity"].items(): if entity["entityType"] == "Protein" or entity["entityType"] == "Gene": entityDict = OrderedDict() entityDict["duid"] = duid entityDict["entityType"] = entity["entityType"] entityDict["charEnd"] = entity["charStart"]+len(entity["entityText"])-1 entityDict["source"] = entity["source"] entityDict["charStart"] = entity["charStart"] try: entityDict["entityId"] = entity["entityId"] except: entityDict["entityId"] = [] try: entityDict["sentenceIndex"] = entity["sentenceIndex"] except: entityDict["sentenceIndex"]=100 entityDict["entityText"] = entity["entityText"] #do the extension here left_extend_sign=False right_extend_sign=False text = raw_doc["text"] left_extension=['example of','structure of','sample of','isoform of','form of','variant of', \ 'example of the','structure of the','sample of the','isoform of the','form of the','variant of the'] text_before_protein=text[:entityDict["charStart"]] text_before_protein_split=text_before_protein.split(' ') if '' in text_before_protein_split: text_before_protein_split.remove('') if ' ' in text_before_protein_split: text_before_protein_split.remove(' ') if len(text_before_protein_split)>=2: left_extension_str=text_before_protein_split[-2].strip()+' '+text_before_protein_split[-1].strip() left_extension_str=left_extension_str.lower() if left_extension_str in left_extension: left_extend_sign=True entityDict["charStart"]=entityDict["charStart"]-len(left_extension_str)-1 entityDict["entityText"]=left_extension_str+' '+entityDict["entityText"] if len(text_before_protein_split)>=3: left_extension_str=text_before_protein_split[-3].strip()+' '+text_before_protein_split[-2].strip()+' '+text_before_protein_split[-1].strip() left_extension_str=left_extension_str.lower() if left_extension_str in left_extension: left_extend_sign=True entityDict["charStart"]=entityDict["charStart"]-len(left_extension_str)-1 entityDict["entityText"]=left_extension_str+' '+entityDict["entityText"] #right extension text_after_protein=text[entityDict["charEnd"]+1:] right_extend_pattern='^\s?(alpha|beta|\w|\d)(\s|,|\.)' com_pattern_right_extend=re.compile(right_extend_pattern) sr_right_extend=com_pattern_right_extend.search(text_after_protein) if sr_right_extend: right_extend_sign=True entityDict["charEnd"]=entityDict["charEnd"]+sr_right_extend.span()[1]-1 entityDict["entityText"]=entityDict["entityText"]+sr_right_extend.group(0)[:-1] if left_extend_sign or right_extend_sign: #extend the protein at the left side entityKey = "entity." + entityDict["duid"] #print('Extended entities: ',entityDict) self.entityCollection.update_one({"docId":pmid},{ "$set": { entityKey: entityDict}}) return None def add_begin_end_string(self,text, span_tuple): new_tuple=list(span_tuple) while new_tuple[0]>0 and text[new_tuple[0]-1]!=' ': new_tuple[0]=new_tuple[0]-1 while new_tuple[1]=new_span_list[j][1] and new_span_list[i][1]<=new_span_list[j][2]+1) \ or (new_span_list[j][1]>=new_span_list[i][1] and new_span_list[j][1]<=new_span_list[i][2]+1): delete_index.append(j) delete_index.append(i) new_add.append((min(new_span_list[i][1],new_span_list[j][1]),max(new_span_list[i][2],new_span_list[j][2]))) delete_index=list(set(delete_index)) delete_index.sort(reverse=True) for di in delete_index: del new_span_list[di] for ni in new_add: new_span_list.append((text[ni[0]:ni[1]],ni[0],ni[1])) merge_happed-=1 new_span_list=list(set(new_span_list)) #print("After merge:",new_span_list) span_list=new_span_list #merge the entities if they overlap or next to each other for si in range(len(span_list)): oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if span_list[si][1]>=int(senInfo["charStart"]) and span_list[si][2]<=int(senInfo["charEnd"])+2: oneEntity["charEnd"]=span_list[si][2]-1 oneEntity["charStart"]=span_list[si][1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: continue oneEntity["entityType"]="Sugar" oneEntity["source"]="SugarDetector" oneEntity["entityText"]=span_list[si][0] entityList.append(oneEntity) #print('Sugar entities: ',entityList) return entityList def generate_sugar_entity_list_from_mongoDB_v2(self,pmid): raw_doc = self.textCollection.find_one({"docId":str(pmid)}) sugar_extension='.*(chains?|structures?|units?|groups?|residues?|types?|contents?|forms?|alditols?|fractions?|entities|entity)$' sugar_headword='.*(\w*saccharides?|[Gg]lycans?|[Ss]ugars?|[Cc]arbohydrates?|[Aa]ntenna|[Aa]ntennae|[Dd]eterminants?|[Rr]epeats?|[Ff]ucosylations?|[Mm]annosylations?|[Ss]ialylations?|[Gg]lcnacylations?|[Gg]alnacylations?)$' sugar_headword_middle=sugar_headword[:-1]+'\W+' modifier_pattern1= '.*(\w*-?[Ss]ialylated|\w*-?[Ff]ucosylated|\w*-?[Ss]ialyl).*' modifier_pattern2='.*([NnOo]\s?-\s?glycans?|[Hh]igh mannose|[Hh]igh-mannose|[Oo]ligomannose|[Pp]aucimannose|[Hh]ybrid|[Cc]omplex|[Cc]ore \d{1}|T antigen|Sialy-Tn antigen).*' modifier_pattern3='.*(\w*antennary).*' modifier_pattern4='.*(\w*-linked).*' left_extension_pattern='([OoNn]-\s?|[Mm]ono-(\s?\(\d+\.?\d*%[a-zA-Z\s]*\))?,?\s?|'+\ '[Dd]i-(\s?\(\d+\.?\d*%[a-zA-Z\s]*\))?,?\s?|'+\ '[Bb]i-(\s?\(\d+\.?\d*%[a-zA-Z\s]*\))?,?\s?|'+\ '[Tt]ri-(\s?\(\d+\.?\d*%[a-zA-Z\s]*\))?,?\s?|'+\ 'and\s?|\w*-?mannosylated\s?|\w*-?fucosylated\s?|\w*-?sialylated\s?|[Ll]ewis\s?){1,5}\s?$' #print(left_extension_pattern) com_left_extension_pattern=re.compile(left_extension_pattern) motifFile='motif.txt' motif_list=pd.read_csv(motifFile,header=None,delimiter='\t').iloc[:,0].tolist() core_motif_name_list=list(set(motif_list)) #sort based on the length of the string core_motif_name_list.sort(key=lambda x:len(x),reverse=True) modifier_pattern5='(\W+.*|^)('+'|'.join(core_motif_name_list)+')(\W+.*|$)' #compile the patterns com_sugar_extension=re.compile(sugar_extension) com_sugar_headword=re.compile(sugar_headword) com_sugar_headword_middle=re.compile(sugar_headword_middle) com_modifier_pattern1=re.compile(modifier_pattern1) com_modifier_pattern2=re.compile(modifier_pattern2) com_modifier_pattern3=re.compile(modifier_pattern3) com_modifier_pattern4=re.compile(modifier_pattern4) com_modifier_pattern5=re.compile(modifier_pattern5) monosaccharide_name_list=['[Ee]pilegionaminic acid', '[Aa]cinetaminic acid', \ '[Aa]lluronic acid', '[Aa]ltruronic acid', \ '[Dd]eoxy lyxo-heptulosaric acid', '[Gg]alacturonic acid', \ '[Gg]lucuronic acid', '[Gg]uluronic acid', \ '[Ii]duronic acid', '[Dd]eoxy glycero galacto-nonulosonic acid', \ '[Dd]eoxy manno-octulosonic acid', '[Ll]egionaminic acid', \ '[Mm]annuronic acid', '[Mm]uramic acid', \ '[Aa]cetylmuramic acid', '[Gg]lycolylmuramic acid', \ '[Nn]euraminic acid', '[Aa]cetylneuraminic acid', \ '[Gg]lycolylneuraminic acid', '[Pp]seudaminic acid', \ '[Ss]ialic acid', '[Tt]aluronic acid','[Dd]eoxy-D-glycero-D-galactononulosonic acid', \ 'galactosamine',] monosaccharide_pattern='.*('+'|'.join(monosaccharide_name_list)+').*' com_monosaccharide_pattern=re.compile(monosaccharide_pattern) span_list=[] entityList=[] if raw_doc: text = raw_doc["text"] #detect mps monosaccharide_name_tsv='dictionaries - sugars.tsv' monosaccharide_short_name_txt='monosaccharide_short_name.txt' found_poly_l,span_list_mps=detect_polysachharide(monosaccharide_name_tsv,monosaccharide_short_name_txt,text) for pi in range(len(found_poly_l)): if '(' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('(','\(') if ')' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace(')','\)') if '[' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('[','\[') if ']' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace(']','\]') if '+' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('+','\+') if len(found_poly_l)==0: mps_pattern='-----' else: mps_pattern='.*('+'|'.join(found_poly_l)+').*' #print('mps_pattern:',mps_pattern) com_mps=re.compile(mps_pattern) for pi in range(len(found_poly_l)): if '\(' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('\(','(') if '\)' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('\)',')') if '\[' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('\[','[') if '\]' in found_poly_l[pi]: found_poly_l[pi]=found_poly_l[pi].replace('\]',']') #skip the nps that contain .*ase word ase_pattern='.*ase(\W|$)' com_ase_pattern=re.compile(ase_pattern) if self.abstract_np_dic and str(pmid) in self.abstract_np_dic: #print('Load the preprocessed NP!') nps=self.abstract_np_dic[str(pmid)] else: nps=[] output_dic={} for npi in nps: np_offset=npi[3] np_offset=np_offset.split(':') np_start=int(np_offset[0]) np_end=int(np_offset[1]) np_text=npi[2] sr_head=com_sugar_headword.search(np_text) sr_head_middle=com_sugar_headword_middle.search(np_text) sr_extension=com_sugar_extension.search(np_text) sr_pattern1=com_modifier_pattern1.search(np_text) sr_pattern2=com_modifier_pattern2.search(np_text) sr_pattern3=com_modifier_pattern3.search(np_text) sr_pattern4=com_modifier_pattern4.search(np_text) sr_pattern5=com_modifier_pattern5.search(np_text) sr_com_monosaccharide_pattern=com_monosaccharide_pattern.search(np_text) sr_com_mps=com_mps.search(np_text) sr_com_ase=com_ase_pattern.search(np_text) if sr_com_ase: continue if sr_head or sr_head_middle or sr_com_monosaccharide_pattern or sr_com_mps or sr_extension\ or sr_pattern1 or sr_pattern2 or sr_pattern3 or sr_pattern4 or sr_pattern5 or len(np_text)==1: #consider the left extension here if np_start>0: left_text=text[:np_start] sr_left_text=com_left_extension_pattern.search(left_text) if sr_left_text: np_start=sr_left_text.span()[0] np_text=text[np_start:np_end+1] if np_text.startswith('and '): np_start+=4 np_text=text[np_start:np_end+1] #print("Changed np:",np_text) sr_pattern5=com_modifier_pattern5.search(np_text) ''' print(np_text) print('sr_head: ',sr_head) print('sr_head_middle: ',sr_head_middle) print('sr_extension: ',sr_extension) print('sr_pattern1: ',sr_pattern1) print('sr_pattern2: ',sr_pattern2) print('sr_pattern3: ',sr_pattern3) print('sr_pattern4: ',sr_pattern4) print('sr_pattern5: ',sr_pattern5) print('sr_com_monosaccharide_pattern: ',sr_com_monosaccharide_pattern) print('sr_com_mps: ',sr_com_mps) ''' matched_part=0 #['pmid','sent_index','gds','head_word','extension_word','adj_ylated','adj_type','adj_antenna','adj_link','adj_motif'] #get the sentence index gds_row=['']*len(self.gds_file_column) sentence = raw_doc["sentence"] for senInfo in sentence: senIndex = senInfo["index"] if np_start>=int(senInfo["charStart"]) and np_end<=int(senInfo["charEnd"])+2: break gds_row[self.gds_file_column.index('pmid')]=str(pmid) gds_row[self.gds_file_column.index('sent_index')]=str(senIndex) gds_row[self.gds_file_column.index('gds')]=np_text gds_row[self.gds_file_column.index('head_word')]='No' gds_row[self.gds_file_column.index('extension_word')]='No' gds_row[self.gds_file_column.index('adj_ylated')]='No' gds_row[self.gds_file_column.index('adj_type')]='No' gds_row[self.gds_file_column.index('adj_antenna')]='No' gds_row[self.gds_file_column.index('adj_link')]='No' gds_row[self.gds_file_column.index('adj_motif')]='No' gds_row[self.gds_file_column.index('is_in_html')]='No' gds_row[self.gds_file_column.index('is_conjunction')]='No' gds_row[self.gds_file_column.index('is_preposition')]='No' gds_row[self.gds_file_column.index('is_ING')]='No' if sr_head or sr_head_middle or sr_com_monosaccharide_pattern or sr_com_mps: matched_part+=1 gds_row[self.gds_file_column.index('head_word')]='Yes' if sr_extension: matched_part+=1 gds_row[self.gds_file_column.index('extension_word')]='Yes' if sr_pattern1: matched_part+=1 gds_row[self.gds_file_column.index('adj_ylated')]='Yes' if sr_pattern2: matched_part+=1 gds_row[self.gds_file_column.index('adj_type')]='Yes' if sr_pattern3: matched_part+=1 gds_row[self.gds_file_column.index('adj_antenna')]='Yes' if sr_pattern4: matched_part+=1 gds_row[self.gds_file_column.index('adj_link')]='Yes' if sr_pattern5: matched_part+=1 gds_row[self.gds_file_column.index('adj_motif')]='Yes' if sr_head or sr_head_middle: span_list.append((np_text,np_start,np_end)) gds_row[self.gds_file_column.index('is_in_html')]='Yes' elif sr_com_monosaccharide_pattern: span_list.append((np_text,np_start,np_end)) gds_row[self.gds_file_column.index('is_in_html')]='Yes' elif sr_com_mps: span_list.append((np_text,np_start,np_end)) gds_row[self.gds_file_column.index('is_in_html')]='Yes' elif sr_pattern5: span_list.append((np_text,np_start,np_end)) gds_row[self.gds_file_column.index('is_in_html')]='Yes' elif sr_extension: if sr_head_middle or sr_pattern1 or sr_pattern2 or sr_pattern3 or sr_pattern4: span_list.append((np_text,np_start,np_end)) gds_row[self.gds_file_column.index('is_in_html')]='Yes' #if some matched part found and skip the case of only containing extension word if matched_part>0 and not (matched_part==1 and sr_extension): output_dic[(np_text,np_start,np_end)]=gds_row #gds_row=[ri.encode('utf-8') for ri in gds_row ] #spamwriter.writerow(gds_row) for si in range(len(span_list)): entity_text=span_list[si][0] old_span=(span_list[si][1],span_list[si][2]) #print('text:',text) #print('old_span:',old_span) new_tuple=self.add_begin_end_string(text, old_span) ''' try: new_tuple=self.add_begin_end_string(text, old_span) except: new_tuple=old_span ''' sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if new_tuple[0]>=int(senInfo["charStart"]) and new_tuple[1]<=int(senInfo["charEnd"])+2: break if text[new_tuple[0]:new_tuple[1]]!=span_list[si][0] or new_tuple[0]!=span_list[si][0] or \ new_tuple[1]!=span_list[si][1]: output_dic[(text[new_tuple[0]:new_tuple[1]],new_tuple[0],new_tuple[1])]=\ output_dic[(span_list[si][0],span_list[si][1],span_list[si][2])] output_dic.pop((span_list[si][0],span_list[si][1],span_list[si][2]),None) span_list[si]=(text[new_tuple[0]:new_tuple[1]],new_tuple[0],new_tuple[1],senIndex) #merge the entities in span_list and detecion of polysacharides mps_span_list=[(found_poly_l[i],span_list_mps[i][0],span_list_mps[i][1]) for i in range(len(found_poly_l))] for mi in range(len(mps_span_list)): sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if mps_span_list[mi][1]>=int(senInfo["charStart"]) and mps_span_list[mi][2]<=int(senInfo["charEnd"])+2: break mps_span_list[mi]=(mps_span_list[mi][0],mps_span_list[mi][1],mps_span_list[mi][2],senIndex) delete_span_list=[] for merge_i in range(30): span_list=[i for i in span_list if i not in delete_span_list ] mps_span_list=[i for i in mps_span_list if i not in delete_span_list ] delete_span_list=[] for mi in range(len(mps_span_list)): found_overlap=False for si in range(len(span_list)): if (span_list[si][1]>=mps_span_list[mi][1] and span_list[si][1]<=mps_span_list[mi][2]) or \ (mps_span_list[mi][1]>=span_list[si][1] and mps_span_list[mi][1]<=span_list[si][2]): delete_span_list.append(span_list[si]) delete_span_list.append(mps_span_list[mi]) new_start=min(span_list[si][1],mps_span_list[mi][1]) new_end=max(span_list[si][2],mps_span_list[mi][2]) merged_text=text[new_start:new_end+1] merged_gds_row=output_dic[(span_list[si][0],span_list[si][1],span_list[si][2])] merged_gds_row[self.gds_file_column.index('gds')]=merged_text output_dic[(merged_text,new_start,new_end)]=merged_gds_row output_dic.pop((span_list[si][0],span_list[si][1],span_list[si][2]),None) span_list.append((merged_text,new_start,new_end,span_list[si][3])) #print("Before GDS:",span_list[si][0]) #print("Before MPS:",mps_span_list[mi][0]) #print('Merged GDS:',(merged_text,new_start,new_end,span_list[si][3])) found_overlap=True break if found_overlap: break else: mps_row=['']*len(self.gds_file_column) mps_row[self.gds_file_column.index('pmid')]=str(pmid) mps_row[self.gds_file_column.index('sent_index')]=str(mps_span_list[mi][3]) mps_row[self.gds_file_column.index('gds')]=mps_span_list[mi][0] #sssssssssdelete_span_list.append(mps_span_list[mi]) output_dic[(mps_span_list[mi][0],mps_span_list[mi][1],mps_span_list[mi][2])]=mps_row span_list.append(mps_span_list[mi]) if len(delete_span_list)==0: break #merge the entities if they are connected by conjunction, preposition, or ING words span_list.sort(key=lambda x: x[1]) #print('span_list before merege:',span_list) delete_span_list=[] conj_wl=['and', ', and','or', ''] prep_wl=['at','in','on','as','off','between','by','through','via','toward','with','within','of'] ing_word_pattern='(\w+ing)\s?('+'|'.join(prep_wl)+')?$' com_ing_word_pattern=re.compile(ing_word_pattern) for merge_i in range(10): span_list=[i for i in span_list if i not in delete_span_list ] span_list.sort(key=lambda x: x[1]) delete_span_list=[] for si in range(len(span_list)-1): #if these two entities are not in the same sentence, then skip if span_list[si][3]!=span_list[si+1][3]: continue if span_list[si][1]>span_list[si+1][2]: text_between=text[span_list[si+1][2]+1:span_list[si][1]] else: text_between=text[span_list[si][2]+1:span_list[si+1][1]] #print('span_list_si:',span_list[si]) #print('span_list_si+1:',span_list[si+1]) #print('text_between:',text_between) new_start=min(span_list[si][1],span_list[si+1][1]) new_end=max(span_list[si][2],span_list[si+1][2]) merged_text=text[new_start:new_end+1] text_between=text_between.strip() #text_between=str(text_between) sr_ing_word=com_ing_word_pattern.search(text_between) if text_between in conj_wl: delete_span_list.append(span_list[si]) delete_span_list.append(span_list[si+1]) merged_gds_row=self.merge_gds_row(output_dic[(span_list[si][0],span_list[si][1],span_list[si][2])],\ output_dic[(span_list[si+1][0],span_list[si+1][1],span_list[si+1][2])]) merged_gds_row[self.gds_file_column.index('gds')]=merged_text merged_gds_row[self.gds_file_column.index('is_conjunction')]='Yes' output_dic[(merged_text,new_start,new_end)]=merged_gds_row output_dic.pop((span_list[si][0],span_list[si][1],span_list[si][2]),None) output_dic.pop((span_list[si+1][0],span_list[si+1][1],span_list[si+1][2]),None) span_list.append((merged_text,new_start,new_end,span_list[si][3])) #print("Before GDS1:",span_list[si][0]) #print("Before GDS2:",span_list[si+1][0]) #print('Merged GDS:',(merged_text,new_start,new_end,span_list[si][3])) break elif text_between in prep_wl: delete_span_list.append(span_list[si]) delete_span_list.append(span_list[si+1]) merged_gds_row=self.merge_gds_row(output_dic[(span_list[si][0],span_list[si][1],span_list[si][2])], \ output_dic[(span_list[si+1][0],span_list[si+1][1],span_list[si+1][2])]) merged_gds_row[self.gds_file_column.index('gds')]=merged_text merged_gds_row[self.gds_file_column.index('is_preposition')]='Yes' output_dic[(merged_text,new_start,new_end)]=merged_gds_row output_dic.pop((span_list[si][0],span_list[si][1],span_list[si][2]),None) output_dic.pop((span_list[si+1][0],span_list[si+1][1],span_list[si+1][2]),None) span_list.append((merged_text,new_start,new_end,span_list[si][3])) #print("Before GDS1:",span_list[si][0]) #print("Before GDS2:",span_list[si+1][0]) #print('Merged GDS:',(merged_text,new_start,new_end,span_list[si][3])) break elif sr_ing_word and len(text_between.split())<=4: delete_span_list.append(span_list[si]) delete_span_list.append(span_list[si+1]) merged_gds_row=self.merge_gds_row(output_dic[(span_list[si][0],span_list[si][1],span_list[si][2])], \ output_dic[(span_list[si+1][0],span_list[si+1][1],span_list[si+1][2])]) merged_gds_row[self.gds_file_column.index('gds')]=merged_text merged_gds_row[self.gds_file_column.index('is_ING')]='Yes' output_dic[(merged_text,new_start,new_end)]=merged_gds_row output_dic.pop((span_list[si][0],span_list[si][1],span_list[si][2]),None) output_dic.pop((span_list[si+1][0],span_list[si+1][1],span_list[si+1][2]),None) span_list.append((merged_text,new_start,new_end,span_list[si][3])) #print("Before GDS1:",span_list[si][0]) #print("Before GDS2:",span_list[si+1][0]) #print('Merged GDS:',(merged_text,new_start,new_end,span_list[si][3])) break if len(delete_span_list)==0: break #print('output_dic',output_dic) #in some cases that the np is not GDS, we need to delete then #for example, np ending with 'analyses','antigens' delete_head_word_list=['analyses','analysis','mapping','bond',\ 'specificities','fragmentation','activities','heavy chain'] span_list_after_del=[] for si in span_list: del_sign=False for di in delete_head_word_list: if si[0].endswith(di): del_sign=True break if not del_sign: span_list_after_del.append(si) else: output_dic.pop(si,None) span_list=span_list_after_del #write the rows into file output_file='GDS_Detection.csv' with open(output_file, 'a') as csvfile: spamwriter = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) for ki in output_dic: gds_row=output_dic[ki] gds_row=[ri.encode('utf-8') for ri in gds_row ] spamwriter.writerow(gds_row) for si in range(len(span_list)): oneEntity={} oneEntity["charEnd"]=span_list[si][2] oneEntity["charStart"]=span_list[si][1] oneEntity["sentenceIndex"]=span_list[si][3] if "charEnd" not in oneEntity: continue oneEntity["entityType"]="Sugar" oneEntity["source"]="SugarDetector" oneEntity["entityText"]=span_list[si][0] entityList.append(oneEntity) #print('Sugar entities: ',entityList) return entityList def merge_gds_row(self,row1,row2): for i in range(len(row1)): if row1[i] =='Yes' or row2[i] =='Yes': row1[i]='Yes' return row1 def generate_protein_entity_list_from_EDG_file(self,pmid,edg_file): raw_doc = self.textCollection.find_one({"docId":str(pmid)}) protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) #regular expression to match protein protein_name_pattern='(.*)('+'|'.join(self.protein_f_term)+')' protein_name_more='(.){4,30}(in)$' protein_name_more_one='.*(in) \w$' protein_name_ase='(.){3,30}(ase)$' protein_name_ase_one='.*(ase) \w$' protein_name_p='(p\d{2,3}[A-Z]*)' span_list=[] with open(edg_file) as csvfile: spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') for row in spamreader: col_title=row break pmid_index=col_title.index('doc_id') arg_head_index=col_title.index('arg_head') arg_head_offset_index=col_title.index('arg_head_offset') arg_base_np_index=col_title.index('arg_base_np') arg_base_np_offset_index=col_title.index('arg_base_np_offset') for row in spamreader: pmid_=row[pmid_index] arg_head=row[arg_head_index] arg_head_offset=row[arg_head_offset_index] #parse the offset to get the real offset tuple arg_head_offset_split=arg_head_offset.split(':') arg_head_offset=(int(arg_head_offset_split[0]),int(arg_head_offset_split[1])) arg_base_np=row[arg_base_np_index] arg_base_np_offset=row[arg_base_np_offset_index] arg_base_np_offset_split=arg_base_np_offset.split(':') arg_base_np_offset=(int(arg_base_np_offset_split[0]),int(arg_base_np_offset_split[1])) if str(pmid_)!=str(pmid): continue #first consider the case of only one letter if len(arg_head)==1 and 97<=ord(arg_head.lower())<=122: #match pattern that ending with one letter m_one1=self.match_pattern(protein_name_more_one,arg_base_np) m_one2=self.match_pattern(protein_name_ase_one,arg_base_np) if m_one1: span_list.append((m_one1[0][0],m_one1[0][1]+arg_base_np_offset[0],m_one1[0][2]+arg_base_np_offset[0],'Cterm')) if m_one2: span_list.append((m_one2[0][0],m_one2[0][1]+arg_base_np_offset[0],m_one2[0][2]+arg_base_np_offset[0],'Cterm')) else: #match first rule m1=re.findall(protein_name_pattern,arg_head) if m1: span_list.append((arg_head,arg_head_offset[0],arg_head_offset[1],'Fterm')) #this part is to check the token before fterm and see if it is a Cterm arg_base_np_split=arg_base_np.split(' ') entity_list_before_fterm=[] if arg_head in arg_base_np_split: arg_head_list_index=arg_base_np_split.index(arg_head) c_term_list=[i[0] for i in self.c_term_detector(arg_base_np)] c_term_del_list=['C-terminal','O-glycosylated'] for cdi in c_term_del_list: #delete the proteins that have the length of 2 if cdi in c_term_list or len(cdi)==2: c_term_list.remove(cdi) #print(c_term_list) if arg_head_list_index>0 and arg_base_np_split[arg_head_list_index-1] in c_term_list: entity_list_before_fterm=self.match_pattern(arg_base_np_split[arg_head_list_index-1],arg_base_np) for ei in entity_list_before_fterm: span_list.append((ei[0],ei[1]+arg_base_np_offset[0],ei[2]+arg_base_np_offset[0],'Cterm')) #match third rule #print('Matching m3',arg_head,'--',protein_name_more) m3=re.finditer(protein_name_more,arg_head) count3=[1 for i in m3] skip_protein_name_more=False skip_protein_name_more_list=['protein','domain'] for si in skip_protein_name_more_list: if arg_head.endswith(si): skip_protein_name_more=True if sum(count3)>0 and not skip_protein_name_more: span_list.append((arg_head,arg_head_offset[0],arg_head_offset[1],'Fterm')) m31=re.finditer(protein_name_ase,arg_head) count31=[1 for i in m31] skip_protein_name_ase=False skip_protein_name_ase_list=['decrease','increase','disease','release'] for si in skip_protein_name_ase_list: if arg_head.endswith(si): skip_protein_name_ase=True if sum(count31)>0 and not skip_protein_name_ase: span_list.append((arg_head,arg_head_offset[0],arg_head_offset[1],'Fterm')) #match fourth rule m4=re.finditer(protein_name_p,arg_head) count4=[1 for i in m4] if sum(count4)>0: span_list.append((arg_head,arg_head_offset[0],arg_head_offset[1],'Cterm')) span_list=list(set(span_list)) entityList=[] #print('span_list:',span_list) if not raw_doc: return entityList for si in range(len(span_list)): skip_sign=False #already detected by previous tools for ei in protein_list: if (int(ei[1])>=span_list[si][1] and int(ei[1])<=span_list[si][2]) \ or (span_list[si][1]>=int(ei[1]) and span_list[si][1]<=int(ei[2])): skip_sign=True break if skip_sign: continue oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if span_list[si][1]>=int(senInfo["charStart"]) and span_list[si][2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=span_list[si][2] oneEntity["charStart"]=span_list[si][1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: continue oneEntity["entityType"]="Protein" oneEntity["source"]=span_list[si][3] oneEntity["entityText"]=span_list[si][0] if 'immunoglobulin M' in span_list[si][0] or 'Immunoglobulin M' in span_list[si][0]: oneEntity["entityId"] = [{"idString" : 'P01871',"source" : "OGER"}] elif 'immunoglobulin E' in span_list[si][0] or 'Immunoglobulin E' in span_list[si][0]: oneEntity["entityId"] = [{"idString" : 'P01854',"source" : "OGER"}] entityList.append(oneEntity) #print('Protein entities: ',entityList) del protein_list,site_list,sugar_list return entityList def match_pattern(self,pattern,text): search_list=[] com_pattern=re.compile(pattern) sr_pattern=com_pattern.search(text) while sr_pattern: search_list.append((sr_pattern.group(),sr_pattern.span()[0],sr_pattern.span()[1])) sr_pattern=com_pattern.search(text,pos=sr_pattern.span()[1]) return search_list def generate_site_entity_list_from_EDG_file(self,pmid,edg_file): raw_doc = self.textCollection.find_one({"docId":str(pmid)}) if not raw_doc: return [] protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_before(pmid) span_list=[] site_name=['site','sites'] extra_pattern_for_site='(.*sites?|.*residues?)' #this is to detect the position \d position_pattern='(positions?\s?(and| |,|\d+)*)' com_position_pattern=re.compile(position_pattern) ab_text=raw_doc['text'] sr=com_position_pattern.search(ab_text) while sr: position_str=ab_text[sr.span()[0]:sr.span()[1]] position_str_start=sr.span()[0] position_str_end=sr.span()[1] while True: if position_str.endswith(' '): position_str=position_str[:-1] position_str_end-=1 elif position_str.endswith('and'): position_str=position_str[:-3] position_str_end-=3 else: break span_list.append((position_str,position_str_start,position_str_end-1)) sr=com_position_pattern.search(ab_text,pos=sr.span()[1]) domain_name=['sequon','face','head','tail','fragment','lobe','peptide','faceside','domain','motif','region','terminus','epitope','sequence', 'arm'\ 'sequons','faces','heads','tails','fragments','lobes','peptides','facesides','domains','motifs','regions','termini','epitopes','sequences','arms'] extra_pattern_for_domain='(.*peptides?|.*domains?|.*motifs?|.*lobes?)' extra_pattern_for_chain_domain='(.*)(light|heavy|alpha|beta|) (chains?)' amino_acid_shrot=['Ala', 'Arg', 'Asn', 'Asp', 'Cys', 'Glu', 'Gln', 'Gly', 'His', 'Hyp', 'Ile', 'Leu', 'Lys', 'Met', 'Phe', 'Pro', 'Glp', 'Ser', 'Thr', 'Trp', 'Tyr', 'Val',] amino_acid_pattern='('+'|'.join(amino_acid_shrot)+')(\W\d*)' with open(edg_file) as csvfile: spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') for row in spamreader: col_title=row break pmid_index=col_title.index('doc_id') arg_head_index=col_title.index('arg_head') arg_head_offset_index=col_title.index('arg_head_offset') for row in spamreader: pmid_=row[pmid_index] arg_head=row[arg_head_index] arg_head_offset=row[arg_head_offset_index] #parse the offset to get the real offset tuple arg_head_offset_split=arg_head_offset.split(':') arg_head_offset=(int(arg_head_offset_split[0]),int(arg_head_offset_split[1])) if str(pmid_)!=str(pmid): continue #match rule mme=re.findall(extra_pattern_for_domain,arg_head) mms=re.findall(extra_pattern_for_site,arg_head) mm_chain=re.findall(extra_pattern_for_chain_domain,arg_head) if arg_head in site_name or arg_head in domain_name or mme or mms or mm_chain: span_list.append((arg_head,arg_head_offset[0],arg_head_offset[1])) m1=re.findall(amino_acid_pattern,arg_head) if m1: span_list.append((arg_head,arg_head_offset[0],arg_head_offset[1])) span_list=list(set(span_list)) entityList=[] for si in range(len(span_list)): skip_sign=False #already detected by previous tools try: for ei in site_list: if (int(ei[1])>=span_list[si][1] and int(ei[1])<=span_list[si][2]) \ or (span_list[si][1]>=int(ei[1]) and span_list[si][1]<=int(ei[2])): skip_sign=True break except: print("ERROR!!!") #print(site_list) if skip_sign: continue oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if span_list[si][1]>=int(senInfo["charStart"]) and span_list[si][2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=span_list[si][2] oneEntity["charStart"]=span_list[si][1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: continue oneEntity["entityType"]="Site" oneEntity["source"]="SiteDetector" oneEntity["entityText"]=span_list[si][0] entityList.append(oneEntity) #print('Site entities: ',entityList) del protein_list,site_list,sugar_list return entityList def add_new_protein_name(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] raw_doc = self.textCollection.find_one({"docId":str(pmid)}) new_protein_pattern1='(((Ig[ADEGM])(\d|a|b|c)?)|([Ii]mmunoglobulin\s[ADEGM](\d|a|b|c)?))' #immunoglobulin new_protein_pattern2='(([Ff]actor|F)\s[VIX]{1,5})' new_protein_pattern3='\W([Aa]lbumin)\W' # for the recombinant part, we have one case in the acronym function new_protein_pattern4='([Rr]ecombinant\s(human |mouse |murine |mice |rat )(\S+\s?){1,4}\((\S+)\))' new_protein_pattern5='([Rr]ecombinant |[Hh]uman |[Mm]ouse] )?([Ff]ollicle stimulating hormone)' com_pattern_1=re.compile(new_protein_pattern1) com_pattern_2=re.compile(new_protein_pattern2) com_pattern_3=re.compile(new_protein_pattern3) com_pattern_4=re.compile(new_protein_pattern4) com_pattern_5=re.compile(new_protein_pattern5) sub_pattern='([Rr]ecombinant\s(human |mouse |murine |mice |rat ))' com_sub_pattern_4=re.compile(sub_pattern) entityList=[] if raw_doc: text = raw_doc["text"] new_protein_list=[] sr1=com_pattern_1.search(text) sr2=com_pattern_2.search(text) sr3=com_pattern_3.search(text) sr4=com_pattern_4.search(text) sr5=com_pattern_5.search(text) while sr1: #print(sr1.span(),'-','--'+sr1.group(0)+'--') #print(sr.span()[1],'-',text[sr.span()[1]]) new_protein_list.append((text[sr1.span()[0]:sr1.span()[1]],sr1.span()[0],sr1.span()[1]-1)) sr1=com_pattern_1.search(text,pos=sr1.span()[1]) while sr2: #print(sr2.span(),'-','--'+sr2.group(0)+'--') #print(sr.span()[1],'-',text[sr.span()[1]]) new_protein_list.append((text[sr2.span()[0]:sr2.span()[1]],sr2.span()[0],sr2.span()[1]-1)) sr2=com_pattern_2.search(text,pos=sr2.span()[1]) while sr3: #print(sr.span(),'-','--'+sr.group(0)+'--') #print(sr.span()[1],'-',text[sr.span()[1]]) new_protein_list.append((text[sr3.span()[0]+1:sr3.span()[1]-1],sr3.span()[0]+1,sr3.span()[1]-1)) sr3=com_pattern_3.search(text,pos=sr3.span()[1]) while sr4: #print(sr4.span(),'-','--'+sr4.group(0)+'--') full_string=sr4.group(0) #detect the prefix sr_sub=com_sub_pattern_4.search(full_string) protein_name=text[sr4.span()[0]+sr_sub.span()[1]:sr4.span()[0]+full_string.index('(')] protein_name=protein_name.strip() new_protein_list.append((protein_name,sr4.span()[0]+sr_sub.span()[1],sr4.span()[0]+sr_sub.span()[1]+len(protein_name))) sr4=com_pattern_4.search(text,pos=sr4.span()[1]) while sr5: #print(sr.span(),'-','--'+sr.group(0)+'--') #print(sr.span()[1],'-',text[sr.span()[1]]) new_protein_list.append((text[sr5.span()[0]:sr5.span()[1]],sr5.span()[0],sr5.span()[1])) sr5=com_pattern_5.search(text,pos=sr5.span()[1]) for si in range(len(new_protein_list)): skip_sign=False #already detected by previous tools for ei in protein_list: if (int(ei[1])>=new_protein_list[si][1] and int(ei[1])<=new_protein_list[si][2]) \ or (new_protein_list[si][1]>=int(ei[1]) and new_protein_list[si][1]<=int(ei[2])): skip_sign=True break if skip_sign: continue oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if new_protein_list[si][1]>=int(senInfo["charStart"]) and new_protein_list[si][2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=new_protein_list[si][2]-1 oneEntity["charStart"]=new_protein_list[si][1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]="Protein" oneEntity["source"]="Cterm" oneEntity["entityText"]=new_protein_list[si][0] if 'immunoglobulin M' in new_protein_list[si][0] or 'Immunoglobulin M' in new_protein_list[si][0]: oneEntity["entityId"] = [{"idString" : 'P01871',"source" : "OGER"}] elif 'immunoglobulin E' in new_protein_list[si][0] or 'Immunoglobulin E' in new_protein_list[si][0]: oneEntity["entityId"] = [{"idString" : 'P01854',"source" : "OGER"}] entityList.append(oneEntity) else: print(str(pmid),' No text found!') #print('New protein found: ',entityList) del protein_list,site_list,sugar_list return entityList def add_acronym_protein(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] protein_str_list=[p[0] for p in protein_list] #this is used to propagate the id of full name to ancronym ancronym_id_doc={} raw_doc = self.textCollection.find_one({"docId":str(pmid)}) acronym_pattern='(^\s?\(\w*-?[A-Z0-9]+-?\w*\))' acronym_pattern_and='(^\s?\(\w*-?[A-Z0-9]+-?\w*) and (\w*-?[A-Z0-9]+-?\w*\))' acronym_pattern_plus_one_word='(\(\w*-?[A-Z0-9]+-?\w*\))\s?\s?(\w*-?\w*)' com_pattern_acronym=re.compile(acronym_pattern) com_pattern_acronym_and=re.compile(acronym_pattern_and) com_pattern_acronym_plus_one_word=re.compile(acronym_pattern_plus_one_word) entityList=[] if raw_doc: text = raw_doc["text"] ancronym_list=[] sr_acronym_plus_one_word=com_pattern_acronym_plus_one_word.search(text) while sr_acronym_plus_one_word: ancronym_str=text[sr_acronym_plus_one_word.span()[0]:sr_acronym_plus_one_word.span()[1]] ancronym_str_np=ancronym_str.replace('(','') ancronym_str_np=ancronym_str_np.replace(')','') if ancronym_str_np in protein_str_list: ancronym_str=ancronym_str.replace('(','\(') ancronym_str=ancronym_str.replace(')','\)') ancronym_list.append(ancronym_str) sr_acronym_plus_one_word=com_pattern_acronym_plus_one_word.search(text,pos=sr_acronym_plus_one_word.span()[1]) for pi in protein_list: #if the protein is not normalized, just skip it if pi[4]=='': continue sr_ancronym=com_pattern_acronym.search(text[pi[2]+1:]) sr_ancronym_and=com_pattern_acronym_and.search(text[pi[2]+1:]) if sr_ancronym: start_index=sr_ancronym.span()[0]+pi[2]+1 #print(sr_ancronym.span(),'-',sr_ancronym.group(0)) while text[start_index]!='(': start_index+=1 ancronym_str=text[start_index+1:sr_ancronym.span()[1]+pi[2]] ancronym_list.append(ancronym_str) if pi[4]!='': ancronym_id_doc[ancronym_str]=pi[4] elif sr_ancronym_and: start_index=sr_ancronym_and.span()[0]+pi[2]+1 #print(sr_ancronym.span(),'-',sr_ancronym.group(0)) while text[start_index]!='(': start_index+=1 ancronym_str=text[start_index+1:sr_ancronym_and.span()[1]+pi[2]] and_index=ancronym_str.index('and') ancronym_list.append(ancronym_str[:and_index-1]) ancronym_list.append(ancronym_str[and_index+4:]) #add recombinant ancronym recombinant_pattern='([Rr]ecombinant\s(\S+\s?){1,4}\((\S+)\))' com_pattern_recombinant=re.compile(recombinant_pattern) sr_recombinant=com_pattern_recombinant.search(text) while sr_recombinant: start_index=sr_recombinant.span()[0] #print(sr_ancronym.span(),'-',sr_ancronym.group(0)) while text[start_index]!='(': start_index+=1 ancronym_str=text[start_index+1:sr_recombinant.span()[1]-1] ancronym_list.append(ancronym_str) sr_recombinant=com_pattern_recombinant.search(text,pos=sr_recombinant.span()[1]) #detect all the ancronym in the abstract ancronym_list=list(set(ancronym_list)) new_protein_list=[] for ai in ancronym_list: one_acronym_pattern='(\W'+ai+'\W)' try: com_pattern_one=re.compile(one_acronym_pattern) except: print('ERROR') #print(ancronym_list) continue sr=com_pattern_one.search(text) while sr: #print(sr.span(),'-','--'+sr.group(0)+'--') #print(sr.span()[1],'-',text[sr.span()[1]]) new_protein_list.append((text[sr.span()[0]+1:sr.span()[1]-1],sr.span()[0]+1,sr.span()[1]-1)) sr=com_pattern_one.search(text,pos=sr.span()[1]) for si in range(len(new_protein_list)): skip_sign=False #already detected by previous tools for ei in protein_list: if (int(ei[1])>=new_protein_list[si][1] and int(ei[1])<=new_protein_list[si][2]) \ or (new_protein_list[si][1]>=int(ei[1]) and new_protein_list[si][1]<=int(ei[2])): skip_sign=True break if skip_sign: continue oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if new_protein_list[si][1]>=int(senInfo["charStart"]) and new_protein_list[si][2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=new_protein_list[si][2]-1 oneEntity["charStart"]=new_protein_list[si][1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]="Protein" oneEntity["source"]="Cterm" oneEntity["entityText"]=new_protein_list[si][0] if new_protein_list[si][0] in ancronym_id_doc: oneEntity["entityId"] =[{"idString" : ancronym_id_doc[new_protein_list[si][0]],"source" : 'OGER'}] entityList.append(oneEntity) else: print(str(pmid),' No text found!') del protein_list,site_list,sugar_list return entityList def add_extra_protein_entity(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] if len(protein_list)==0: return [] for pii in range(len(protein_list)): pi=list(protein_list[pii]) if '(' in pi[0]: pi[0]=pi[0].replace('(','\(') if ')' in pi[0]: pi[0]=pi[0].replace(')','\)') if '[' in pi[0]: pi[0]=pi[0].replace('[','\[') if ']' in pi[0]: pi[0]=pi[0].replace(']','\]') protein_list[pii]=tuple(pi) raw_doc = self.textCollection.find_one({"docId":str(pmid)}) protein_dic={} for pi in protein_list: protein_dic[pi[0]]=pi[3] protein_list_pattern=[pi[0] for pi in protein_list if pi[0]!=''] extra_pattern='('+'|'.join(protein_list_pattern)+')\W' #print(extra_pattern) try: com_pattern_extra=re.compile(extra_pattern) except: print("ERROR!") extra_pattern=extra_pattern.encode('utf-8') com_pattern_extra=re.compile("(test)") entityList=[] if raw_doc: text = raw_doc["text"] new_protein_list=[] sr=com_pattern_extra.search(text) while sr: #print(sr.span(),'-','--'+sr.group(0)+'--') #print(sr.span()[1],'-',text[sr.span()[1]]) new_protein_list.append((text[sr.span()[0]:sr.span()[1]-1],sr.span()[0],sr.span()[1]-1)) sr=com_pattern_extra.search(text,pos=sr.span()[1]) #print(new_protein_list) for si in range(len(new_protein_list)): skip_sign=False protein_name_contain_parenthese=new_protein_list[si][0] if '(' in protein_name_contain_parenthese: protein_name_contain_parenthese=protein_name_contain_parenthese.replace('(','\(') if ')' in protein_name_contain_parenthese: protein_name_contain_parenthese=protein_name_contain_parenthese.replace(')','\)') if '[' in protein_name_contain_parenthese: protein_name_contain_parenthese=protein_name_contain_parenthese.replace('[','\[') if ']' in protein_name_contain_parenthese: protein_name_contain_parenthese=protein_name_contain_parenthese.replace(']','\]') #already detected by previous tools for ei in protein_list: if (int(ei[1])>=new_protein_list[si][1] and int(ei[1])<=new_protein_list[si][2]) \ or (new_protein_list[si][1]>=int(ei[1]) and new_protein_list[si][1]<=int(ei[2])): skip_sign=True break if skip_sign: continue oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if new_protein_list[si][1]>=int(senInfo["charStart"]) and new_protein_list[si][2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=new_protein_list[si][2]-1 oneEntity["charStart"]=new_protein_list[si][1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]="Protein" oneEntity["source"]=protein_dic[protein_name_contain_parenthese] oneEntity["entityText"]=new_protein_list[si][0] entityList.append(oneEntity) else: print(str(pmid),' No text found!') del protein_list,site_list,sugar_list return entityList def c_term_detector(self,text): list_of_word=word_tokenize(text) word_extra=[] for wii in word_extra: self.dict.append(wii) #if there is a capital letter or a digit in the word, we will see it as c-term pattern_for_capital_letter='[A-Z]+' pattern_for_digit='\d+' com_pattern_capital=re.compile(pattern_for_capital_letter) com_pattern_digit=re.compile(pattern_for_digit) sub_string_list=['Man','kDa','C-terminal','Different','This',\ 'BACKGROUND','Various','RESULTS','Although',\ 'CONCLUSION','Natural','O-glycosylated'] c_term_set=set() for wi in list_of_word: wi=wi.replace('+','') wi=wi.replace('*','') if len(wi)==0: continue skip_sign=False for si in sub_string_list: if wi.find(si)>=0: skip_sign=True if skip_sign: continue if wi not in self.dict: sr_capital=com_pattern_capital.search(wi) sr_digit=com_pattern_digit.search(wi) if sr_capital or sr_digit: c_term_set.add(wi) pattern_for_c_term='\W('+'|'.join(list(c_term_set))+')\W' try: com_pattern_c_term=re.compile(pattern_for_c_term) except: print(pattern_for_c_term) sr_c_term=com_pattern_c_term.search(text) c_term_span_list=[] while sr_c_term: #print(sr_c_term.span(),'-',sr_c_term.group(1)) c_term_span_list.append((sr_c_term.group(1),sr_c_term.span()[0]+1,sr_c_term.span()[1]-1)) sr_c_term=com_pattern_c_term.search(text,pos=sr_c_term.span()[1]) #print('c-term',c_term_span_list) return c_term_span_list def add_c_term_protein(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] entity_list=protein_list+site_list+sugar_list raw_doc = self.textCollection.find_one({"docId":str(pmid)}) entityList=[] c_term_list=[] if raw_doc: text = raw_doc["text"] c_term_list=self.c_term_detector(text) if c_term_list: for ci in c_term_list: skip_sign=False add_sign=False related_protein='' #detect overlap or not for ei in entity_list: if (int(ei[1])<=ci[1] and int(ei[2])>=ci[2]) \ or (ci[1]<=int(ei[1]) and ci[2]>int(ei[1])) or (ci[1]=int(ei[2])): skip_sign=True break if skip_sign: #will not consider this c-term continue #consider the case that c-term appear at the left/right side for ei in protein_list: if abs(ci[2]-int(ei[1]))<=2 or abs(ci[1]-int(ei[2]))<=2: add_sign=True related_protein=ei #print('Related f term:',ei) #print('Found c term:',ci) break if add_sign and related_protein[3]=='Fterm': #create one entity here oneEntity={} sentence = raw_doc["sentence"] for senInfo in sentence: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if ci[1]>=int(senInfo["charStart"]) and ci[2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=ci[2]-1 oneEntity["charStart"]=ci[1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]="Protein" oneEntity["source"]="Cterm" oneEntity["entityText"]=ci[0] entityList.append(oneEntity) elif add_sign: self.update_entity_info(pmid,related_protein,ci) del protein_list,site_list,sugar_list return entityList def detect_entity_from_json_file(self,pmid,json_file,entity_type,text,sentences,entity_list): dic_file='./entrez_reviewed_ac_mapping_new.txt' reviewed = {} with codecs.open(dic_file, 'r', encoding='utf8') as f: for line in f: line = line.strip() line_split=line.split('\t') reviewed[line_split[0]]=line_split[1] entityList=[] update_entityList=[] if json_file: with open(json_file) as bjfile: json_entity_dic=json.load(bjfile) else: json_entity_dic={} entity_original_list=[] if str(pmid) in json_entity_dic: entity_original_list=json_entity_dic[str(pmid)] detected_entity_list=[] detected_entity_str_list=[] detected_entity_id_dic={} for ei in entity_original_list: detected_entity_str_list.append(ei[0]) if ei[0] not in detected_entity_id_dic: detected_entity_id_dic[ei[0]]=ei[1] detected_entity_str_list=list(set(detected_entity_str_list)) for pii in range(len(detected_entity_str_list)): if '(' in detected_entity_str_list[pii]: detected_entity_str_list[pii]=detected_entity_str_list[pii].replace('(','\(') if ')' in detected_entity_str_list[pii]: detected_entity_str_list[pii]=detected_entity_str_list[pii].replace(')','\)') if '[' in detected_entity_str_list[pii]: detected_entity_str_list[pii]=detected_entity_str_list[pii].replace('[','\[') if ']' in detected_entity_str_list[pii]: detected_entity_str_list[pii]=detected_entity_str_list[pii].replace(']','\]') for entity_i in detected_entity_str_list: if '.' in entity_i: continue protein_pattern='(\W|^)('+entity_i+')(\W|$)' com_protein_pattern=re.compile(protein_pattern) sr=com_protein_pattern.search(text) if '\(' in entity_i: entity_i=entity_i.replace('\(','(') if '\)' in entity_i: entity_i=entity_i.replace('\)',')') if '\[' in entity_i: entity_i=entity_i.replace('\[','[') if '\]' in entity_i: entity_i=entity_i.replace('\]',']') while sr: str_start=sr.span()[0] str_end=sr.span()[1] entity_text=text[sr.span()[0]:sr.span()[1]] count_to_stop=0 while True: if not entity_text.endswith(entity_i): entity_text=entity_text[:-1] str_end-=1 elif not entity_text.startswith(entity_i): entity_text=entity_text[1:] str_start+=1 else: break count_to_stop+=1 if count_to_stop>100: break sr=com_protein_pattern.search(text,pos=sr.span()[1]) if entity_text not in detected_entity_id_dic: continue detected_entity_list.append((entity_text,str_start,str_end,'OGER',detected_entity_id_dic[entity_text])) #detect overlap or not #in 17015441, the protein "TRPM8" is normalized to different ids in different sentences #in sentence 1, it is normalized to a mouse id (this is correct) #but in other cases, it is normalized to human id (this is wrong) #In order to make our species assignment work, we need to use all mentions of the protein #so I use the need_to_normalize_protein_name to store all the protein names that need to be normalized #Even though in some cases, pubtator normalize it correctly, we still need it for the normalization of other mentions need_to_normalize_protein_name=[] for bi in detected_entity_list: #in some cases, the "Gene" type is normalized by Pubtator, and it is also normalized by OGER with Complex-protein #So we need to change the "Gene" type to "Complex-protein" to make is usable for subunit normalization change_from_gene_to_complex_protein=False skip_sign=False duid='' for ei in entity_list: if (bi[1]<=int(ei[1]) and bi[2]>=int(ei[1])) or \ (bi[1]<=int(ei[2]) and bi[1]>=int(ei[1])): skip_sign=True if len(ei)>4: if ei[4]=='' or ei[4] not in reviewed \ or ei[0] in need_to_normalize_protein_name: update_entityList.append([ei[5],[{"idString" : bi[4],"source" : "OGER"}]]) duid=ei[5] skip_sign=False if entity_type=='Protein-Complex': change_from_gene_to_complex_protein=True duid=ei[5] skip_sign=False keep_pubtator_source=ei[3] keep_pubtator_id=ei[4] break if skip_sign: #will not consider this protein continue need_to_normalize_protein_name.append(bi[0]) #create one entity here oneEntity={} for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if bi[1]>=int(senInfo["charStart"]) and bi[2]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=bi[2]-1 oneEntity["charStart"]=bi[1] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]=entity_type oneEntity["source"]="OGER" oneEntity["entityText"]=bi[0] oneEntity["entityId"]=[{"idString" : bi[4],"source" : "OGER"}] if change_from_gene_to_complex_protein: oneEntity["source"]=keep_pubtator_source oneEntity["entityId"]=[{"idString" : keep_pubtator_id,"source" : "Entrez"}] if duid: oneEntity["duid"] = duid entityList.append(oneEntity) #print('OGER added:',entityList) return entityList,update_entityList def add_oger_protein_complex(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] entity_list=protein_list+site_list+sugar_list #print('protein_list (inside becas function)',protein_list) #print('entity_list (inside becas function)',entity_list) raw_doc = self.textCollection.find_one({"docId":str(pmid)}) if not raw_doc: return [] if raw_doc: text = raw_doc["text"] sentences = raw_doc["sentence"] #oger_json_file='glygen_large_oger_entity_cp.json' entityList,update_entityList=self.detect_entity_from_json_file(pmid,self.oger_json_file_cp,'Protein-Complex',text,sentences,entity_list) #print('entityList before species assignment:',entityList) entityList_after_species_assignment=self.species_assignment_for_protein(pmid,entityList) return entityList_after_species_assignment def add_oger_protein(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] entity_list=protein_list+site_list+sugar_list #print('protein_list (inside becas function)',protein_list) #print('entity_list (inside becas function)',entity_list) raw_doc = self.textCollection.find_one({"docId":str(pmid)}) if not raw_doc: return [],[] if raw_doc: text = raw_doc["text"] sentences = raw_doc["sentence"] #oger_json_file='glygen_large_oger_entity.json' entityList,update_entityList=self.detect_entity_from_json_file(pmid,self.oger_json_file,'Protein',text,sentences,entity_list) #print('entityList before species assignment:',entityList) entityList_dedup=[] for ei in entityList: already_in_db=False for pi in protein_list: if (ei["charStart"]>= pi[1] and ei["charStart"]<=pi[2]) or \ (pi[1]>=ei["charStart"] and pi[1]<=ei["charEnd"]): already_in_db=True break if not already_in_db: entityList_dedup.append(ei) entityList_after_species_assignment=self.species_assignment_for_protein(pmid,entityList_dedup) return entityList_after_species_assignment,update_entityList def add_oger_complex(self,pmid): protein_list,site_list,sugar_list=self.get_entity_list_from_mongoDB_after(pmid) protein_list=[pi for pi in protein_list if pi[0]!='.'] entity_list=protein_list+site_list+sugar_list #print('protein_list (inside becas function)',protein_list) #print('entity_list (inside becas function)',entity_list) raw_doc = self.textCollection.find_one({"docId":str(pmid)}) if not raw_doc: return [] if raw_doc: text = raw_doc["text"] sentences = raw_doc["sentence"] #oger_json_file='glygen_large_oger_entity_complex.json' entityList,update_entityList=self.detect_entity_from_json_file(pmid,self.oger_json_file_complex,'Complex',text,sentences,entity_list) #print('entityList before species assignment:',entityList) entityList_after_species_assignment=self.species_assignment_for_protein(pmid,entityList) return entityList_after_species_assignment def normalize_subunit_chain_complex(self,pmid): text_entity=self.get_abstract_text(str(pmid)) if not text_entity: return [] abstract_text=text_entity["text"] sentences=text_entity["sentence"] entityList=[] #first detect the position of subunit/chain subunit_chain_pattern='([Ss]ubunits?|[Cc]hains?)' com_subunit_chain=re.compile(subunit_chain_pattern) sr_subunit_chain=com_subunit_chain.search(abstract_text) while sr_subunit_chain: possible_multiple_subunit_sign=False subunit_span=sr_subunit_chain.span() matched_subunit_chain_text=abstract_text[sr_subunit_chain.span()[0]:sr_subunit_chain.span()[1]] matched_subunit_chain_text=matched_subunit_chain_text.strip() if matched_subunit_chain_text.endswith('s'): #for the subunits or chains, we need to consider three words around #for example: "alpha and beta subunits" or "subunits alpha and beta" words_around_start=sr_subunit_chain.span()[0] words_around_end=sr_subunit_chain.span()[1] space_left=5 while space_left>0: words_around_start-=1 if words_around_start<=0: break if abstract_text[words_around_start]==' ': space_left-=1 space_right=3 while space_right>0: words_around_end+=1 if words_around_end>=len(abstract_text): break if abstract_text[words_around_end]==' ': space_right-=1 words_around=abstract_text[words_around_start+1:words_around_end] possible_multiple_subunit_sign=True else: #for subunit or chain, we only consider one word around it #for example alpha subunit words_around_start=sr_subunit_chain.span()[0] words_around_end=sr_subunit_chain.span()[1] space_left=2 while space_left>0: words_around_start-=1 if words_around_start<=0: break if abstract_text[words_around_start]==' ': space_left-=1 space_right=1 while space_right>0: words_around_end+=1 if words_around_end>=len(abstract_text): break if abstract_text[words_around_end]==' ': space_right-=1 words_around=abstract_text[words_around_start+1:words_around_end] #print('matched_subunit_chain_text:',matched_subunit_chain_text) #print('words_around:',words_around) #for the next search results sr_subunit_chain=com_subunit_chain.search(abstract_text,pos=sr_subunit_chain.span()[1]) #then detect the subunit type like alpha, beta, 1, 2, A, I, etc #for the information after subunit greek_list=['[Aa]lpha','[Bb]eta','[Gg]amma','[Dd]elta','[Ss]igma'] greek_list_short=[gi[1] for gi in greek_list] subunit_greek='('+'|'.join(greek_list)+')' subunit_greek_digit='('+'|'.join(greek_list)+')-?\s?(\d+)([A-Z]?)' subunit_digit='\d+' subunit_cap_letter='\W[A-Z]{1,3}\W' #comple the patterns complie_subunit_greek=re.compile(subunit_greek) complie_subunit_greek_digit=re.compile(subunit_greek_digit) complie_subunit_single_digit=re.compile(subunit_digit) complie_subunit_single_letter=re.compile(subunit_cap_letter) #search the patterns in the string around the subunit/chain #case 1: match the case of greek letter + digit greek_digit_matched_type=[] sr_greek_digit=complie_subunit_greek_digit.search(words_around) if possible_multiple_subunit_sign: while sr_greek_digit: greek_digit_matched_type.append(sr_greek_digit.group(0)) sr_greek_digit=complie_subunit_greek_digit.search(words_around,pos=sr_greek_digit.span()[1]) else: if sr_greek_digit: greek_digit_matched_type.append(sr_greek_digit.group(0)) #case 2: only greek greek_matched_type=[] sr_greek=complie_subunit_greek.search(words_around) if possible_multiple_subunit_sign: while sr_greek: greek_matched_type.append(sr_greek.group(0)) sr_greek=complie_subunit_greek.search(words_around,pos=sr_greek.span()[1]) else: if sr_greek: greek_matched_type.append(sr_greek.group(0)) #case 3: only digit digit_matched_type=[] sr_digit=complie_subunit_single_digit.search(words_around) if possible_multiple_subunit_sign: while sr_digit: digit_matched_type.append(sr_digit.group(0)) sr_digit=complie_subunit_single_digit.search(words_around,pos=sr_digit.span()[1]) else: if sr_digit: digit_matched_type.append(sr_digit.group(0)) #case 4: only capital letters cap_letter_matched_type=[] sr_cap_letter=complie_subunit_single_letter.search(words_around) if possible_multiple_subunit_sign: while sr_cap_letter: cap_letter_matched_type.append(sr_cap_letter.group(0)[1:-1]) sr_cap_letter=complie_subunit_single_letter.search(words_around,pos=sr_cap_letter.span()[1]) else: if sr_cap_letter: cap_letter_matched_type.append(sr_cap_letter.group(0)[1:-1]) #print('greek_digit_matched_type:',greek_digit_matched_type) #print('greek_matched_type:',greek_matched_type) #print('digit_matched_type:',digit_matched_type) #print('cap_letter_matched_type:',cap_letter_matched_type) #if there are mutiple matches, skip it if len(greek_digit_matched_type)>=2 or len(greek_matched_type)>=2 or \ len(digit_matched_type)>=2 or len(cap_letter_matched_type)>=2: continue #if there is no match, also skip it if len(greek_digit_matched_type)==0 and len(greek_matched_type)==0 and \ len(digit_matched_type)==0 and len(cap_letter_matched_type)==0: continue #figure out the matched type here final_matched_type='' if greek_digit_matched_type: final_matched_type=greek_digit_matched_type[0] elif greek_matched_type: final_matched_type=greek_matched_type[0] elif digit_matched_type: final_matched_type=digit_matched_type[0] elif cap_letter_matched_type: final_matched_type=cap_letter_matched_type[0] #print('final_matched_type:',final_matched_type) #load the complex dictionary complex_dic_file='complex_dic.json' with open(complex_dic_file) as bjfile: complex_dic=json.load(bjfile) #then find out the complex name and normalize the subunit complex_protein_list=self.get_complex_protein_list_from_mongoDB(pmid) complex_list=[cpi for cpi in complex_protein_list if cpi[-1]=='Complex'] complex_protein_list=[cpi for cpi in complex_protein_list if cpi[-1]!='Complex'] between_of_and_complex='^\s?(the\s?)?\w*\s?$' com_between_of_and_complex=re.compile(between_of_and_complex) final_matched_type_pattern='(^|\W)'+final_matched_type.lower()+'(\W|$)' com_final_matched_type=re.compile(final_matched_type_pattern) oneEntity={} #case 1: subunit of X (X is the complex) str_after_subunit=abstract_text[subunit_span[1]+1:] if str_after_subunit.startswith('of'): for cpi in complex_list: str_between_of_and_complex=abstract_text[subunit_span[1]+3:cpi[1]] sr_between_of_and_complex=com_between_of_and_complex.search(str_between_of_and_complex) if sr_between_of_and_complex and cpi[0].lower() in complex_dic: for complex_type_i in complex_dic[cpi[0].lower()]: if com_final_matched_type.search(complex_type_i[0].lower()): for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if subunit_span[0]>=int(senInfo["charStart"]) and subunit_span[1]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=subunit_span[1]-1 oneEntity["charStart"]=subunit_span[0] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]='Protein' oneEntity["source"]="OGER" oneEntity["entityText"]=matched_subunit_chain_text oneEntity["entityId"]=[{"idString" : complex_type_i[1],"source" : "OGER"}] entityList.append(oneEntity) break break if len(oneEntity)>0: #print('Rule 1 mathched for complex:',oneEntity) continue #case 2: X subunit for cpi in complex_list: if (subunit_span[0]-cpi[2]==1 or subunit_span[0]-cpi[2]==2) and cpi[0].lower() in complex_dic: for complex_type_i in complex_dic[cpi[0].lower()]: if com_final_matched_type.search(complex_type_i[0].lower()): for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if subunit_span[0]>=int(senInfo["charStart"]) and subunit_span[1]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=subunit_span[1]-1 oneEntity["charStart"]=subunit_span[0] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]='Protein' oneEntity["source"]="OGER" oneEntity["entityText"]=matched_subunit_chain_text oneEntity["entityId"]=[{"idString" : complex_type_i[1],"source" : "OGER"}] entityList.append(oneEntity) break break if len(oneEntity)>0: #print('Rule 2 mathched for complex:',oneEntity) continue #case 3: complex in the title complex_in_title_list=[] for cpi in complex_list: if self.get_sentence_index((cpi[1],cpi[2]),sentences)==0: complex_in_title_list.append(cpi) #print('complex_in_title_list:',complex_in_title_list) if len(complex_in_title_list)==1 and complex_in_title_list[0][0].lower() in complex_dic: for complex_type_i in complex_dic[complex_in_title_list[0][0].lower()]: #print('complex_type_i:',complex_type_i) if com_final_matched_type.search(complex_type_i[0].lower()): for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if subunit_span[0]>=int(senInfo["charStart"]) and subunit_span[1]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=subunit_span[1]-1 oneEntity["charStart"]=subunit_span[0] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]='Protein' oneEntity["source"]="OGER" oneEntity["entityText"]=matched_subunit_chain_text oneEntity["entityId"]=[{"idString" : complex_type_i[1],"source" : "OGER"}] entityList.append(oneEntity) break #print('normalize_subunit_chain_complex:',entityList) entityList_after_species_assignment=self.species_assignment_for_protein(pmid,entityList) return entityList_after_species_assignment def normalize_subunit_chain_complex_protein(self,pmid): text_entity=self.get_abstract_text(str(pmid)) if not text_entity: return [] abstract_text=text_entity["text"] sentences=text_entity["sentence"] entityList=[] #first detect the position of subunit/chain subunit_chain_pattern='([Ss]ubunits?|[Cc]hains?)' com_subunit_chain=re.compile(subunit_chain_pattern) sr_subunit_chain=com_subunit_chain.search(abstract_text) while sr_subunit_chain: subunit_span=sr_subunit_chain.span() matched_subunit_chain_text=abstract_text[sr_subunit_chain.span()[0]:sr_subunit_chain.span()[1]] matched_subunit_chain_text=matched_subunit_chain_text.strip() #for the next search results sr_subunit_chain=com_subunit_chain.search(abstract_text,pos=sr_subunit_chain.span()[1]) #then find out the complex-protein name and normalize the subunit complex_protein_list=self.get_complex_protein_list_from_mongoDB(pmid) complex_protein_list=[cpi for cpi in complex_protein_list if cpi[-1]!='Complex'] between_of_and_complex='^\s?(the\s?)?\w*\s?$' com_between_of_and_complex=re.compile(between_of_and_complex) oneEntity={} #case 1: subunit of X (X is the complex) str_after_subunit=abstract_text[subunit_span[1]+1:] if str_after_subunit.startswith('of'): for cpi in complex_protein_list: str_between_of_and_complex=abstract_text[subunit_span[1]+3:cpi[1]] sr_between_of_and_complex=com_between_of_and_complex.search(str_between_of_and_complex) if sr_between_of_and_complex: for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if subunit_span[0]>=int(senInfo["charStart"]) and subunit_span[1]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=subunit_span[1]-1 oneEntity["charStart"]=subunit_span[0] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]='Protein' oneEntity["source"]="OGER" oneEntity["entityText"]=matched_subunit_chain_text oneEntity["entityId"]=[{"idString" : cpi[4],"source" : "OGER"}] entityList.append(oneEntity) break if len(oneEntity)>0: #print('Rule 1 mathched for complex/protein:',oneEntity) continue #case 2: X subunit for cpi in complex_protein_list: if subunit_span[0]-cpi[2]==1 or subunit_span[0]-cpi[2]==2: for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if subunit_span[0]>=int(senInfo["charStart"]) and subunit_span[1]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=subunit_span[1]-1 oneEntity["charStart"]=subunit_span[0] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]='Protein' oneEntity["source"]="OGER" oneEntity["entityText"]=matched_subunit_chain_text oneEntity["entityId"]=[{"idString" : cpi[4],"source" : "OGER"}] entityList.append(oneEntity) break if len(oneEntity)>0: #print('Rule 2 mathched for complex/protein:',oneEntity) continue #case 3: complex in the title complex_in_title_list=[] for cpi in complex_protein_list: if self.get_sentence_index((cpi[1],cpi[2]),sentences)==0: complex_in_title_list.append(cpi) #print('complex_in_title_list:',complex_in_title_list) if len(complex_in_title_list)==1: for senInfo in sentences: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if subunit_span[0]>=int(senInfo["charStart"]) and subunit_span[1]<=int(senInfo["charEnd"]): oneEntity["charEnd"]=subunit_span[1]-1 oneEntity["charStart"]=subunit_span[0] oneEntity["sentenceIndex"]=senIndex break if "charEnd" not in oneEntity: oneEntity["charEnd"]=0 oneEntity["charStart"]=0 oneEntity["sentenceIndex"]=0 oneEntity["entityType"]='Protein' oneEntity["source"]="OGER" oneEntity["entityText"]=matched_subunit_chain_text oneEntity["entityId"]=[{"idString" : complex_in_title_list[0][4],"source" : "OGER"}] entityList.append(oneEntity) #print('normalize_subunit_chain_complex_protein:',entityList) return entityList def species_assignment_for_protein(self,pmid,entity_list): uniprot_kb_ac_to_id_file='uniprot_kb_ac_to_id.json' uniprot_kb_id_to_ac_file='uniprot_kb_id_to_ac.json' with open(uniprot_kb_ac_to_id_file) as bjfile: uniprot_kb_ac_to_id_dic=json.load(bjfile) with open(uniprot_kb_id_to_ac_file) as bjfile: uniprot_kb_id_to_ac_dic=json.load(bjfile) #define the species set that we need to consider species_list=['human','rat','mouse'] #build the dictionary for the NCBI ID species_ncbi_id_dic={'9606':'human','10116':'rat','10090':'mouse'} #build the dictionary for the species prefix species_prefix_dic={} for si in species_list: if len(si.split(' '))==1: species_prefix_dic[si[0].lower()]=si else: species_prefix_str=''.join([i[0] for i in si.split(' ')]) species_prefix_str=species_prefix_str[0].upper()+species_prefix_str[1:] species_prefix_dic[species_prefix_str]=si #build the dictionary for the species entry name in the uniprot database species_entry_name_dic={} for si in species_list: species_entry_name_dic[si]=si.upper() #we only assignment species for the entities from Uniprot, not from Pubtator pubtator_entity_list=[ei for ei in entity_list if ei["entityId"][0]["idString"] not in uniprot_kb_ac_to_id_dic] entity_list=[ei for ei in entity_list if ei["entityId"][0]["idString"] in uniprot_kb_ac_to_id_dic] updated_entity_list=[] #rule 1: species prefix for ei in entity_list: entity_str=ei["entityText"] found_species='' if len(entity_str.split(' '))==1: for prei in species_prefix_dic.keys(): if entity_str.startswith(prei): found_species=species_prefix_dic[prei] break if found_species: if found_species=='human': if ei not in updated_entity_list: updated_entity_list.append(ei) #entity_list.remove(ei) else: #map the id for human to other species #first map the id for human to the entry name protein_ac_for_other_species=self.map_protein_ac_to_other_species(uniprot_kb_ac_to_id_dic, \ uniprot_kb_id_to_ac_dic, \ species_entry_name_dic, \ found_species, \ ei) ei["entityId"]=[{"idString" : protein_ac_for_other_species,"source" : "OGER"}] if ei not in updated_entity_list: updated_entity_list.append(ei) #entity_list.remove(ei) #print("Update the protein id of species based on rule 1 !!",ei) #update the entity_list for ei in updated_entity_list: if ei in entity_list: entity_list.remove(ei) #print('entity_list after rule 1: ',entity_list) #print('updated_entity_list after rule 1: ',updated_entity_list) #rule 2: search in the noun phrase if self.abstract_np_dic and str(pmid) in self.abstract_np_dic: #print('Load the preprocessed NP!') nps=self.abstract_np_dic[str(pmid)] else: nps=[] species_list=self.get_species_list_from_mongoDB(pmid) if species_list: for npi in nps: np_offset=npi[3] np_offset=np_offset.split(':') np_start=int(np_offset[0]) np_end=int(np_offset[1]) np_text=npi[2] #print('Current NP:',npi) #check if the species and protein are in the same np species_in_this_np=None protein_in_this_np=None for si in species_list: if si[1]>=np_start and si[2]<=np_end: #(entity text, entity id) species_in_this_np=(si[0],si[4]) break for pi in entity_list: if pi["charStart"]>=np_start and pi["charEnd"]<=np_end: #(entity text, entity id) protein_in_this_np=(pi["entityText"],pi["entityId"][0]["idString"]) break if species_in_this_np and protein_in_this_np: #only consider specific species here #print("Found species and protein in the same np:",species_in_this_np,protein_in_this_np) if species_in_this_np[1] not in species_ncbi_id_dic: continue species_str=species_ncbi_id_dic[species_in_this_np[1]] #update the all the protein id here for ei in entity_list: if ei["entityId"][0]["idString"]==protein_in_this_np[1]: if species_str=='human': if ei not in updated_entity_list: updated_entity_list.append(ei) #entity_list.remove(ei) else: #map the id for human to other species #first map the id for human to the entry name protein_ac_for_other_species=self.map_protein_ac_to_other_species(uniprot_kb_ac_to_id_dic, \ uniprot_kb_id_to_ac_dic, \ species_entry_name_dic, \ species_str, \ ei) ei["entityId"]=[{"idString" : protein_ac_for_other_species,"source" : "OGER"}] if ei not in updated_entity_list: updated_entity_list.append(ei) #entity_list.remove(ei) #print("Update the protein id of species based on rule 2 !!",ei) #update the entity_list for ei in updated_entity_list: if ei in entity_list: entity_list.remove(ei) #print('entity_list after rule 2: ',entity_list) #print('updated_entity_list after rule 2: ',updated_entity_list) #case 3: single species #print('species_list:',species_list) if len(species_list)==1: if species_list[0][4] in species_ncbi_id_dic: species_str=species_ncbi_id_dic[species_list[0][4]] #update the all the protein id here for ei in entity_list: #print("deal with ei",ei) if species_str=='human': if ei not in updated_entity_list: updated_entity_list.append(ei) #entity_list.remove(ei) else: #map the id for human to other species #first map the id for human to the entry name protein_ac_for_other_species=self.map_protein_ac_to_other_species(uniprot_kb_ac_to_id_dic, \ uniprot_kb_id_to_ac_dic, \ species_entry_name_dic, \ species_str, \ ei) ei["entityId"]=[{"idString" : protein_ac_for_other_species,"source" : "OGER"}] if ei not in updated_entity_list: updated_entity_list.append(ei) #entity_list.remove(ei) #print("Update the protein id of species based on rule 3 !!",ei) #update the entity_list for ei in updated_entity_list: if ei in entity_list: entity_list.remove(ei) #print('entity_list after rule 3: ',entity_list) #print('updated_entity_list after rule 3: ',updated_entity_list) #case 4: species in the same sentence if species_list and entity_list: text_entity=self.get_abstract_text(str(pmid)) for si in species_list: for ei in entity_list: species_sent_index=self.get_sentence_index((si[1],si[2]),text_entity['sentence']) pritein_sent_index=self.get_sentence_index((ei["charStart"],ei["charEnd"]),text_entity['sentence']) if species_sent_index==pritein_sent_index and pritein_sent_index!=-1: if si[4] in species_ncbi_id_dic: species_str=species_ncbi_id_dic[si[4]] #update the all the protein id here for eii in entity_list: if eii["entityId"][0]["idString"]==ei["entityId"][0]["idString"]: if species_str=='human': if eii not in updated_entity_list: updated_entity_list.append(eii) #entity_list.remove(eii) else: #map the id for human to other species #first map the id for human to the entry name protein_ac_for_other_species=self.map_protein_ac_to_other_species(uniprot_kb_ac_to_id_dic, \ uniprot_kb_id_to_ac_dic, \ species_entry_name_dic, \ species_str, \ eii) eii["entityId"]=[{"idString" : protein_ac_for_other_species,"source" : "OGER"}] if eii not in updated_entity_list: updated_entity_list.append(eii) #entity_list.remove(eii) #print("Update the protein id of species based on rule 4 !!",eii) #update the entity_list for ei in updated_entity_list: if ei in entity_list: entity_list.remove(ei) #case 5: species in the previous sentence if species_list and entity_list: text_entity=self.get_abstract_text(str(pmid)) for si in species_list: for ei in entity_list: species_sent_index=self.get_sentence_index((si[1],si[2]),text_entity['sentence']) pritein_sent_index=self.get_sentence_index((ei["charStart"],ei["charEnd"]),text_entity['sentence']) if species_sent_index-pritein_sent_index==-1 and pritein_sent_index!=-1 and species_sent_index!=-1: if si[4] in species_ncbi_id_dic: species_str=species_ncbi_id_dic[si[4]] #update the all the protein id here for eii in entity_list: if eii["entityId"][0]["idString"]==ei["entityId"][0]["idString"]: if species_str=='human': if eii not in updated_entity_list: updated_entity_list.append(eii) #entity_list.remove(eii) else: #map the id for human to other species #first map the id for human to the entry name protein_ac_for_other_species=self.map_protein_ac_to_other_species(uniprot_kb_ac_to_id_dic, \ uniprot_kb_id_to_ac_dic, \ species_entry_name_dic, \ species_str, \ eii) eii["entityId"]=[{"idString" : protein_ac_for_other_species,"source" : "OGER"}] if eii not in updated_entity_list: updated_entity_list.append(eii) #entity_list.remove(eii) #print("Update the protein id of species based on rule 5 !!",eii) #update the entity_list for ei in updated_entity_list: if ei in entity_list: entity_list.remove(ei) #case 6: major species: species in the title or in the MeSH terms if species_list and entity_list: text_entity=self.get_abstract_text(str(pmid)) for si in species_list: for ei in entity_list: species_sent_index=self.get_sentence_index((si[1],si[2]),text_entity['sentence']) pritein_sent_index=self.get_sentence_index((ei["charStart"],ei["charEnd"]),text_entity['sentence']) if species_sent_index-pritein_sent_index==0: if si[4] in species_ncbi_id_dic: species_str=species_ncbi_id_dic[si[4]] #update the all the protein id here for eii in entity_list: if eii["entityId"][0]["idString"]==ei["entityId"][0]["idString"]: if species_str=='human': if eii not in updated_entity_list: updated_entity_list.append(eii) #entity_list.remove(eii) else: #map the id for human to other species #first map the id for human to the entry name protein_ac_for_other_species=self.map_protein_ac_to_other_species(uniprot_kb_ac_to_id_dic,\ uniprot_kb_id_to_ac_dic,\ species_entry_name_dic,\ species_str,\ eii) eii["entityId"]=[{"idString" : protein_ac_for_other_species,"source" : "OGER"}] if eii not in updated_entity_list: updated_entity_list.append(eii) #entity_list.remove(eii) #print("Update the protein id of species based on rule 6 !!",eii) #update the entity_list for ei in updated_entity_list: if ei in entity_list: entity_list.remove(ei) return updated_entity_list+entity_list+pubtator_entity_list def map_protein_ac_to_other_species(self,uniprot_kb_ac_to_id_dic,uniprot_kb_id_to_ac_dic,species_entry_name_dic,species_str,entity): protein_id_human=uniprot_kb_ac_to_id_dic[entity["entityId"][0]["idString"]] protein_official_name=protein_id_human.split('_')[0] protein_id_for_other_species=protein_official_name+'_'+species_entry_name_dic[species_str] if protein_id_for_other_species in uniprot_kb_id_to_ac_dic: protein_ac_for_other_species=uniprot_kb_id_to_ac_dic[protein_id_for_other_species] else: protein_ac_for_other_species=entity["entityId"][0]["idString"] return protein_ac_for_other_species def get_sentence_index(self,entity_offset,sentence_entity): #sentence = raw_doc["sentence"] for senInfo in sentence_entity: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if entity_offset[0]>=int(senInfo["charStart"]) and entity_offset[1]<=int(senInfo["charEnd"]): return senIndex return -1 def update_entity_info(self,pmid,ei,ci): #decide it is left extension or right extension left=False right=False if ci[1]