from __future__ import division,print_function import time,re,sys,os,ast,collections import pandas as pd import re,csv,codecs,json from os import path from add_sugar_protein_site_class import add_sugar_protein_site_class from collections import OrderedDict, defaultdict from convert_uniprot_site import split_amino_acid_position from f_terms_EDG import extrac_certain_columns_from_postprocessed_file from np_generator import np_generator_for_abstrat def site_substrate_fusion(file_input1,file_input2,output_file,db_name,pmidFile): #for logging the pmids that have multiple protein in the title pmid_multiple_protein_in_title={} site_dic={} site_same_trigger_dic={} site_previous_trigger_dic={} site_column_name='arg_site' protein_column_name='arg_protein' relation_column_name='relation' dic_id_column_name='doc_id' #this is for detect if the two proteins are the same: #one from pubtator, one from OGER dic_file='./entrez_reviewed_ac_mapping_new.txt' reviewed = {} with codecs.open(dic_file, 'r', encoding='utf8') as f: for line in f: line = line.strip() line_split=line.split('\t') reviewed[line_split[0]]=line_split[1] with open(file_input1) as csvfile: spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') for row in spamreader: col_title=row break site_index=col_title.index(site_column_name) protein_index=col_title.index(protein_column_name) relation_index=col_title.index(relation_column_name) pmid_index=col_title.index(dic_id_column_name) Source_index=col_title.index('PSource') SiteSource_index=col_title.index('SiteSource') relation_id_index= col_title.index('relation_id') NProtein_index=col_title.index('NProtein') NID_index=col_title.index('NID') SiteName_index=col_title.index('SiteName') sent_index_index=col_title.index('sent_index') trigger_offset_index=col_title.index('trigger_offset') for row in spamreader: row=[ri.decode('utf-8') for ri in row ] site_name=row[SiteName_index] protein_name=row[protein_index] relation_name=row[relation_index] relation_id=row[relation_id_index] NProtein=row[NProtein_index] NID=row[NID_index] pmid=row[pmid_index] sent_index=row[sent_index_index] trigger_offset=row[trigger_offset_index] trigger_offset=int(trigger_offset.split(':')[0]) if len(site_name)>0 and len(protein_name)>0: site_dic[str(pmid)+site_name]=(protein_name,NProtein,NID) #print("Site information",str(pmid)+site_name,'With relation :',relation_name) if len(protein_name)>0: site_same_trigger_dic[str(pmid)+'-'+str(relation_id)]=(protein_name,NProtein,NID) if str(pmid)+'-'+str(sent_index) not in site_previous_trigger_dic: site_previous_trigger_dic[str(pmid)+'-'+str(sent_index)]=[(protein_name,NProtein,NID,trigger_offset)] else: site_previous_trigger_dic[str(pmid)+'-'+str(sent_index)].append((protein_name,NProtein,NID,trigger_offset)) with open(output_file, 'w') as csvfile: spamwriter_o = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter_o.writerow(col_title) with open(file_input2) as csvfile: spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') for row in spamreader: col_title=row break site_index=col_title.index(site_column_name) protein_index=col_title.index(protein_column_name) relation_index=col_title.index(relation_column_name) pmid_index=col_title.index(dic_id_column_name) Source_index=col_title.index('PSource') SiteSource_index=col_title.index('SiteSource') NProtein_index=col_title.index('NProtein') NID_index=col_title.index('NID') SiteName_index=col_title.index('SiteName') relation_id_index= col_title.index('relation_id') sent_index_index=col_title.index('sent_index') trigger_offset_index=col_title.index('trigger_offset') for row in spamreader: row=[ri.decode('utf-8') for ri in row ] site_name=row[SiteName_index] protein_name=row[NProtein_index] relation_name=row[relation_index] pmid=row[pmid_index] sent_index=int(row[sent_index_index]) relation_id=row[relation_id_index] trigger_offset=row[trigger_offset_index] trigger_offset=int(trigger_offset.split(':')[0]) if pmidFile: pmidList = pd.read_csv(pmidFile,header=None).iloc[:,0].tolist() pmidList=[str(i) for i in pmidList] if str(pmid) not in pmidList: continue #skip part_of if relation_name=='part_of': continue #only consider the site that contain some position information res_site=split_amino_acid_position(site_name) if len(res_site)==0 or len(protein_name)>0: row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) continue #step 0: if str(pmid)+'-'+str(relation_id) in site_same_trigger_dic and len(protein_name)==0 and len(site_name)>0: #print('Row before update:',row) row[protein_index]=site_same_trigger_dic[str(pmid)+'-'+str(relation_id)][0] row[NProtein_index]=site_same_trigger_dic[str(pmid)+'-'+str(relation_id)][1] row[NID_index]=site_same_trigger_dic[str(pmid)+'-'+str(relation_id)][2] #print('Row after update',row) #print('Site information:',str(pmid)+site_name) row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #skip the the following steps continue #site_dic contain the information for step 1 if str(pmid)+site_name in site_dic and len(protein_name)==0: #print('Row before update:',row) row[protein_index]=site_dic[str(pmid)+site_name][0] row[NProtein_index]=site_dic[str(pmid)+site_name][1] row[NID_index]=site_dic[str(pmid)+site_name][2] #print('Row after update',row) #print('Site information:',str(pmid)+site_name) row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #skip the step 3 continue #this for step 2 if str(pmid)+'-'+str(sent_index) in site_previous_trigger_dic and len(protein_name)==0 and len(site_name)>0: #print('Row before update:',row) site_previous_trigger_dic[str(pmid)+'-'+str(sent_index)].sort(key=lambda s: s[3],reverse=True) found_previous_trigger=False for previous_trigger in site_previous_trigger_dic[str(pmid)+'-'+str(sent_index)]: if trigger_offset>previous_trigger[3]: row[protein_index]=previous_trigger[0] row[NProtein_index]=previous_trigger[1] row[NID_index]=previous_trigger[2] found_previous_trigger=True #print('Found trigger in the same sentence:',trigger_offset) break if found_previous_trigger: row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #skip the the following steps continue #this is for step 3 if len(site_name)>0 and relation_name=='gly' and len(protein_name)==0: #continue ASPSC=add_sugar_protein_site_class("pubtator",db_name,"medline.aligned","entities","text","text") protein_list,site_list,sugar_list=ASPSC.get_entity_list_from_mongoDB_after(str(pmid)) text=ASPSC.get_abstract_text(str(pmid)) if text is None: continue #first check if there is protein name in the title gly_protein_in_title=protein_glycosylated_in_sent(pmid,protein_list,file_input2,0) protein_in_title=protein_in_sent(protein_list,text['sentence'],0) head_word_protein_in_title=protein_is_head_word_of_np_in_sent(pmid,protein_list,text['sentence'],0) normalized_protein_in_title=list(set(list(tuple(protein_in_title[2])))) #check if protein from pubtator and OGER are the same? normalized_protein_in_title_new=[] for npi in normalized_protein_in_title: if npi in reviewed: normalized_protein_in_title_new.append(reviewed[npi]) else: normalized_protein_in_title_new.append(npi) normalized_protein_in_title=list(set(normalized_protein_in_title_new)) while '' in normalized_protein_in_title: normalized_protein_in_title.remove('') #print('protein_in_title:',protein_in_title) #print('normalized_protein_in_title:',normalized_protein_in_title) protein_name_pairs=find_full_name_acronym_pair(protein_list,text) if protein_in_title[0]: if len(protein_in_title[1])==1: row[NProtein_index]=protein_in_title[1][0] row[NID_index]=protein_in_title[2][0] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("Title sentence:",row) continue #even though the protein mighe normalized, but we did not update the NID here elif len(protein_in_title[1])>1: #pmid_multiple_protein_in_title[pmid]=protein_in_title[1] interection_gly=set(protein_in_title[1]).intersection(set(gly_protein_in_title[1])) for (p1,p2) in protein_name_pairs: if p1 in interection_gly and p2 in interection_gly: interection_gly.discard(p1) if len(interection_gly)==1: row[NProtein_index]=list(interection_gly)[0] row[NID_index]=protein_in_title[2][protein_in_title[1].index(list(interection_gly)[0])] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("Title sentence:",row) continue elif len(normalized_protein_in_title)==1: row[NProtein_index]=protein_in_title[1][0] row[NID_index]=protein_in_title[2][0] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("Title sentence:",row) continue elif len(head_word_protein_in_title[1])==1: row[NProtein_index]=head_word_protein_in_title[1][0] row[NID_index]=head_word_protein_in_title[2][0] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("Title sentence:",row) continue #if there are multiple proteins in the title, let other rule to cover this case ''' elif len(interection_gly)>1: row[NProtein_index]='?' row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) print("Title sentence:",row) continue ''' #last check if protein is in the first sentence protein_in_first_sent=protein_in_sent(protein_list,text['sentence'],1) protein_first_np_first_sent=protein_in_first_np_of_first_sent(pmid,protein_list) normalized_protein_in_first_sent=list(set(list(tuple(protein_in_first_sent[2])))) while '' in normalized_protein_in_first_sent: normalized_protein_in_first_sent.remove('') if protein_in_first_sent[0]: if len(protein_in_first_sent[1])==1: row[NProtein_index]=protein_in_first_sent[1][0] row[NID_index]=protein_in_first_sent[2][0] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First sentence:",row) continue #even though the protein mighe normalized, but we did not update the NID here elif len(protein_first_np_first_sent)==1: row[NProtein_index]=protein_first_np_first_sent[0][0] row[NID_index]=protein_first_np_first_sent[0][4] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First sentence:",row) continue elif len(protein_in_first_sent[1])>1 and gly_protein_in_title[0]: interection_gly=set(protein_in_first_sent[1]).intersection(set(gly_protein_in_title[1])) for (p1,p2) in protein_name_pairs: if p1 in interection_gly and p2 in interection_gly: interection_gly.discard(p1) if len(interection_gly)==1: row[NProtein_index]=list(interection_gly)[0] row[NID_index]=protein_in_first_sent[2][protein_in_first_sent[1].index(list(interection_gly)[0])] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First sentence:",row) continue elif len(normalized_protein_in_first_sent)==1: row[NProtein_index]=protein_in_first_sent[1][0] row[NID_index]=protein_in_first_sent[2][0] row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First sentence:",row) continue elif len(interection_gly)>1: row[NProtein_index]='?' row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First sentence:",row) continue #then check if analysis of protein in previous 2 sentences gly_sent_index_list,protein_pattern_dic=detect_pattern_in_sent(protein_list,text) if sent_index-1 in protein_pattern_dic: row[NProtein_index]=', '.join([i[0] for i in protein_pattern_dic[sent_index-1]]) row[NID_index]=', '.join([i[1] for i in protein_pattern_dic[sent_index-1]]) row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First analysis:",row) continue if sent_index-2 in protein_pattern_dic: row[NProtein_index]=', '.join([i[0] for i in protein_pattern_dic[sent_index-2]]) row[NID_index]=', '.join([i[1] for i in protein_pattern_dic[sent_index-2]]) row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("Second analysis:",row) continue #then check if preotein is substrate in the previous 2 sentences substrate_sent_ind_list,protein_substrate_dic=extract_substrate_from_post_processing_file(file_input2,pmid) if sent_index-1 in protein_substrate_dic: row[NProtein_index]=', '.join([i[0] for i in protein_substrate_dic[sent_index-1]]) row[NID_index]=', '.join([i[1] for i in protein_substrate_dic[sent_index-1]]) row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("First substrate:",row) continue if sent_index-2 in protein_substrate_dic: row[NProtein_index]=', '.join([i[0] for i in protein_substrate_dic[sent_index-2]]) row[NID_index]=', '.join([i[1] for i in protein_substrate_dic[sent_index-2]]) row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) #print("Second substrate:",row) continue row=[ri.encode('utf-8') for ri in row ] spamwriter_o.writerow(row) if len(protein_in_title[1])>1: pmid_multiple_protein_in_title[pmid]=protein_in_title[1] #for logging the pimds that have multiple proteins in title with open('pmid_multiple_protein_in_title_unresolved.csv', 'w') as csvfile: spamwriter_mpt = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter_mpt.writerow(['pmid','proteins']) for ki in pmid_multiple_protein_in_title: mpt_row=[ki,','.join(pmid_multiple_protein_in_title[ki])] mpt_row=[ri.encode('utf-8') for ri in mpt_row ] spamwriter_mpt.writerow(mpt_row) def find_full_name_acronym_pair(protein_list,text): pair_list=[] abstract_text=text['text'] if protein_list and abstract_text: for si in range(len(protein_list)): for sii in range(si+1,len(protein_list)): if protein_list[si][1]>protein_list[sii][2]: text_between=abstract_text[protein_list[sii][2]+1:protein_list[si][1]] else: text_between=abstract_text[protein_list[si][2]+1:protein_list[sii][1]] if text_between.strip()=='(': pair_list.append((protein_list[si][0],protein_list[sii][0])) return pair_list def extract_substrate_from_post_processing_file(file_name,pmid): protein_dic=defaultdict(list) sent_index_list=[] with open(file_name) as csvfile: spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') for row in spamreader: col_title=row break for row in spamreader: row=[ri.decode('utf-8') for ri in row ] if str(row[0])==str(pmid) and len(row[col_title.index('arg_protein')])>0 and len(row[col_title.index('NProtein')])>0: sent_index_list.append(row[col_title.index('sent_index')]) protein_dic[str(row[col_title.index('sent_index')])].append((row[col_title.index('NProtein')],row[col_title.index('NID')])) return list(set(sent_index_list)), protein_dic def detect_pattern_in_sent(protein_list,text_entity): sent_index_list=[] protein_pattern_dic={} key_word_noun=['[Aa]nalysis','[Ii]nvestigation','[Pp]urification'] key_word_verb=['[Cc]onfirmed','[Ii]nvestigated','[Dd]igested','[Pp]urified'] key_word_noun_pattern='('+'|'.join(key_word_noun)+')' key_word_verb_pattern= '('+'|'.join(key_word_verb)+')' #case 1 noun key word + protein result_noun=re.finditer(key_word_noun_pattern,text_entity['text']) if result_noun: #print(result) for ri in result_noun: key_word_sent_index=get_sentence_index(ri.span(),text_entity['sentence']) for pi in protein_list: if get_sentence_index((pi[1],pi[2]),text_entity['sentence'])==key_word_sent_index \ and int(pi[1])>=int(ri.span()[0]) and pi[3]!='Fterm': sent_index_list.append(key_word_sent_index) if key_word_sent_index in protein_pattern_dic: if (pi[0],pi[4]) not in protein_pattern_dic[key_word_sent_index]: protein_pattern_dic[key_word_sent_index].append((pi[0],pi[4])) else: protein_pattern_dic[key_word_sent_index]=[(pi[0],pi[4])] #case 2 protein + verb result_verb=re.finditer(key_word_verb_pattern,text_entity['text']) if result_verb: #print(result) for ri in result_verb: key_word_sent_index=get_sentence_index(ri.span(),text_entity['sentence']) for pi in protein_list: if get_sentence_index((pi[1],pi[2]),text_entity['sentence'])==key_word_sent_index \ and int(pi[1])<=int(ri.span()[0]) and pi[3]!='Fterm': sent_index_list.append(key_word_sent_index) if key_word_sent_index in protein_pattern_dic: if (pi[0],pi[4]) not in protein_pattern_dic[key_word_sent_index]: protein_pattern_dic[key_word_sent_index].append((pi[0],pi[4])) else: protein_pattern_dic[key_word_sent_index]=[(pi[0],pi[4])] return list(set(sent_index_list)), protein_pattern_dic def get_sentence_index(entity_offset,sentence_entity): #sentence = raw_doc["sentence"] for senInfo in sentence_entity: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] senIndex = senInfo["index"] if entity_offset[0]>=int(senInfo["charStart"]) and entity_offset[1]<=int(senInfo["charEnd"]): return senIndex return -1 def protein_in_sent(protein_list,sentence_entity,sent_index): protein_info=[False,[],[]] for senInfo in sentence_entity: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] if senInfo["index"]==sent_index: for pi in protein_list: if pi[1]>=int(senInfo["charStart"]) and pi[2]<=int(senInfo["charEnd"]) and pi[3]!='Fterm': protein_info[0]=True protein_info[1].append(pi[0]) #this is for the protein normalized ID protein_info[2].append(pi[4]) return tuple(protein_info) def protein_glycosylated_in_sent(pmid,protein_list,tsv_file,sent_ind): protein_str_list=[p[0] for p in protein_list] protein_info=[False,[],[]] with open(tsv_file) as csvfile: spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') for row in spamreader: col_title=row break pmid_index=col_title.index('doc_id') Source_index=col_title.index('PSource') SiteSource_index=col_title.index('SiteSource') NProtein_index=col_title.index('NProtein') NID_index=col_title.index('NID') sent_index_index=col_title.index('sent_index') for row in spamreader: row=[ri.decode('utf-8') for ri in row ] protein_name_tsv=row[NProtein_index] pmid_tsv=row[pmid_index] sent_index_tsv=int(row[sent_index_index]) if str(pmid)==str(pmid_tsv) and protein_name_tsv in protein_str_list and sent_index_tsv==sent_ind: protein_info[0]=True protein_info[1].append(protein_list[protein_str_list.index(protein_name_tsv)][0]) #this is for the protein normalized ID protein_info[2].append(protein_list[protein_str_list.index(protein_name_tsv)][4]) return tuple(protein_info) def protein_is_head_word_of_np_in_sent(pmid,protein_list,sentence_entity,sent_index): #for np dic abstract_np_file='' if path.exists('glygen_set_abstract_np.json'): abstract_np_file='glygen_set_abstract_np.json' if abstract_np_file: with open(abstract_np_file) as jfile: abstract_np_dic=json.load(jfile) else: abstract_np_dic={} if abstract_np_dic and str(pmid) in abstract_np_dic: #print('Load the preprocessed NP!') nps=abstract_np_dic[str(pmid)] np_start=0 np_end=0 np_end_list=[] for npi in nps: if npi[1]==str(sent_index): np_offset=npi[3] np_offset=np_offset.split(':') np_start=int(np_offset[0]) np_end=int(np_offset[1]) np_end_list.append(np_end) protein_info=[False,[],[]] for senInfo in sentence_entity: #senText_original = title_abstract[senInfo["charStart"]:senInfo["charEnd"]] if senInfo["index"]==sent_index: for pi in protein_list: if pi[1]>=int(senInfo["charStart"]) and pi[2]<=int(senInfo["charEnd"]) and pi[3]!='Fterm': if pi[2] in np_end_list or pi[2]-1 in np_end_list or pi[2]+1 in np_end_list: protein_info[0]=True protein_info[1].append(pi[0]) #this is for the protein normalized ID protein_info[2].append(pi[4]) return tuple(protein_info) def protein_in_first_np_of_first_sent(pmid,protein_list): #for np dic abstract_np_file='' if path.exists('glygen_set_abstract_np.json'): abstract_np_file='glygen_set_abstract_np.json' if abstract_np_file: with open(abstract_np_file) as jfile: abstract_np_dic=json.load(jfile) else: abstract_np_dic={} if abstract_np_dic and str(pmid) in abstract_np_dic: #print('Load the preprocessed NP!') nps=abstract_np_dic[str(pmid)] np_start=0 np_end=0 for npi in nps: if npi[1]=='1': np_offset=npi[3] np_offset=np_offset.split(':') np_start=int(np_offset[0]) np_end=int(np_offset[1]) np_text=npi[2] break found_protein_list=[] for pi in protein_list: if (int(pi[1])>=np_start and int(pi[1])<=np_end) \ or (np_start>=int(pi[1]) and np_start<=int(pi[2])): found_protein_list.append(pi) return found_protein_list if __name__ == '__main__': #test pmidFile='' task_name = sys.argv[1] db_name = sys.argv[2] file_input1=task_name+'_cterm.tsv' file_input2=task_name+'_postprocessing.tsv' output_file=task_name+'_postprocessing_site_fusion.tsv' final_file =task_name+'_site_fusion_final.tsv' site_substrate_fusion(file_input1,file_input2,output_file,db_name,pmidFile) extrac_certain_columns_from_postprocessed_file(output_file,final_file)