from __future__ import unicode_literals, print_function import os,csv import sys reload(sys); #sys.setdefaultencoding("utf8") # nlputis codes. sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from protolib.python import document_pb2, rpc_pb2, edgRules_pb2 from utils.rpc.iterator import request_iter_docs, edg_request_iter_docs from utils.rpc import grpcapi from utils.helper import DocHelper from utils.param_helper import ParamHelper from utils.edg_relations import EdgArg, EdgRelation, EdgRelations import glob from collections import defaultdict import re import pandas as pd rels_to_print = set(["gly","attach","contain","located","theme","from","with","link","mod","ofin", "group_of", "quantifier_of", "is_a", "member_collection", "part_whole", "inat", "gly_prop_isa", "gly_prop_quant_of", "gly_prop_group_of", "gly_prop_mc", "gly_part_whole","dep","used","subunit","chain"]) #rels_to_print = set(["subunit","chain"]) #rels_to_print = set(["ofin"]) columns = ("doc_id", "sent_index", "relation_id", "relation", "trigger", "trigger_offset",\ "arg_num", "arg_head", "arg_protein","arg_domain","arg_site","arg_sugar","arg_head_offset", "arg_base_np", "arg_base_np_offset",\ "arg_np", "arg_np_offset", "sent_text") #print("\t".join(columns)) def split_edge_name(edge_name): if re.search(r'arg[0-9]+_',edge_name): tokens = edge_name.split("_") rel_name = tokens[1] arg_number = tokens[0] if rel_name in rels_to_print: return (rel_name, arg_number) else: return (None,None) else: return (None,None) def get_offset(proto_obj, doc): if type(proto_obj) == document_pb2.Sentence.Constituent: token_start = doc.token[proto_obj.token_start] token_end = doc.token[proto_obj.token_end] char_start = token_start.char_start char_end = token_end.char_end return str(char_start)+":"+str(char_end) else: return str(proto_obj.char_start)+":"+str(proto_obj.char_end) def print_edg_relations(edg_relations, doc): row_list=[] relation_id = 0 helper = DocHelper(doc) sentences = doc.sentence for trigger_rel, args in edg_relations.items(): sent_index, relation, trigger_index = trigger_rel trigger = doc.token[trigger_index] trigger_offset = get_offset(trigger, doc) sentence = sentences[sent_index] for arg in args: arg_num, arg_head_index = arg arg_head = doc.token[arg_head_index] np_cst_index = helper.getParentNPIndexFromLeafTokenIndex(sentence,arg_head_index) base_np_cst_index = helper.getParentNPIndexFromLeafTokenIndex1(sentence,arg_head_index) np_cst = sentence.constituent[np_cst_index] base_np_cst = sentence.constituent[base_np_cst_index] # to_print = (doc.doc_id, str(sent_index), str(relation_id), relation, trigger.word, trigger_offset,\ # arg_num, arg_head.word, get_offset(arg_head, doc), helper.text(base_np_cst),\ # get_offset(base_np_cst, doc), helper.text(np_cst), get_offset(np_cst, doc), helper.text(sentence)) base_noun_phrase = re.sub("\n"," ",helper.text(base_np_cst)) full_noun_phrase = re.sub("\n"," ",helper.text(np_cst)) sentence_text = re.sub("\n"," ",helper.text(sentence)) to_print = (doc.doc_id, str(sent_index), str(relation_id), relation, trigger.word, trigger_offset,\ arg_num, arg_head.word,'','','','', get_offset(arg_head, doc) , base_noun_phrase,\ get_offset(base_np_cst, doc), full_noun_phrase, get_offset(np_cst, doc), sentence_text) row_list.append(list(to_print)) relation_id+=1 return row_list def merge_same_trigger(doc): helper = DocHelper(doc) trigger_to_arg = defaultdict(set) sentences = doc.sentence for sent in sentences: sent_index = sent.index for dep in sent.dependency_extra: trigger = dep.gov_index arg = dep.dep_index edge_name = dep.relation relation, arg_num = split_edge_name(edge_name) if relation: trigger_to_arg[(sent_index, relation, trigger)].add((arg_num, arg)) return trigger_to_arg def run_process_edg_relations(input_file_pmids, db,rule_phase0_filename,server_ip,port_num,edgFile): edg_pmid_set=set() with open(edgFile) as csvfile1: spamreader = csv.reader(csvfile1, delimiter=str('\t'), quotechar=str('|'), quoting=csv.QUOTE_MINIMAL) for row in spamreader: col_title=row break pmid_index=col_title.index('doc_id') for row in spamreader: pmid_=row[pmid_index] edg_pmid_set.add(pmid_) #client = MongoClient('localhost') #db = client.medline_current.text #####Iterate through all files in Input directory and create doc_list #input_file_pmids = sys.argv[1] pmidFH = open(input_file_pmids, "r") pmids = pmidFH.readlines() pmidFH.close() document_list = list() pmids_count = 0 for pmid in pmids: pmid = pmid.strip() #print ("#"+pmid+"#") if pmid in edg_pmid_set: continue doc_id = pmid db_doc = db.find_one({'docId': pmid}) if db_doc: pmids_count += 1 doc_text = db_doc['text'] raw_doc = document_pb2.Document() raw_doc.text = doc_text raw_doc.doc_id = doc_id document_list.append(raw_doc) if len(document_list)==0: return None else: print("Update EDG file!") #rule_phase0_filename = sys.argv[2] fh0 = open(rule_phase0_filename, "r") rule0_lines = fh0.readlines() fh0.close() ####NEED TO UPDDATE PARAM_HELPER param_helper = ParamHelper("NA","NA",rule0_lines,[],[]) edg_rules = edgRules_pb2.EdgRules() param_helper.setRuleProtoAttributes(edg_rules) #param_helper.setDocProtoAttributes(raw_doc) # This is a simple function to make requests out of a list of documents. We # put 5 documents in each request. requests = edg_request_iter_docs(document_list, edg_rules, request_size=5, request_type=rpc_pb2.EdgRequest.PARSE_BLLIP) # Given a request iterator, send requests in parallel and get responses. responses_queue = grpcapi.get_queue(server=server_ip, port=port_num, request_thread_num=10, iterable_request=requests, edg_request_processor=True) count = 0 for response in responses_queue: for doc in response.document: #print(doc) helper = DocHelper(doc) sentences = doc.sentence doc_id = doc.doc_id #print(edg_rules) edg_relations = merge_same_trigger(doc) #print(edg_relations) row_list=print_edg_relations(edg_relations, doc) count += 1 if doc_id not in edg_pmid_set: with open(edgFile, 'a') as csvfile: spamwriter = csv.writer(csvfile, delimiter=str('\t'), quotechar=str('|'), quoting=csv.QUOTE_MINIMAL) for ri in row_list: ri=[rii.encode('utf-8') for rii in ri ] #print(ri) spamwriter.writerow(ri) def exstract_subset_edgFile(pmidFile,edgFile,pmidListInMongo,db_name_to): pmidList = pd.read_csv(pmidFile,header=None).iloc[:,0].tolist() pmidList=list(set(pmidList)) pmidList=[str(pi) for pi in pmidList] pmidListInMongo=[str(pi) for pi in pmidListInMongo] #filter out the processed pmids pmidList=[pi for pi in pmidList if pi not in pmidListInMongo] edgFile_subset=db_name_to+'.tsv' with open(edgFile_subset, 'w') as csvfile: spamwriter = csv.writer(csvfile, delimiter=str('\t'), quotechar=str('|'), quoting=csv.QUOTE_MINIMAL) with open(edgFile) as csvfile1: spamreader = csv.reader(csvfile1, delimiter=str('\t'), quotechar=str('|'), quoting=csv.QUOTE_MINIMAL) for row in spamreader: col_title=row break spamwriter.writerow(col_title) pmid_index=col_title.index('doc_id') for row in spamreader: pmid_=row[pmid_index] if str(pmid_) in pmidList: row=[rii.encode('utf-8') for rii in row ] spamwriter.writerow(row) return edgFile_subset if __name__ == '__main__': run_process_edg_relations()