#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import division,print_function
import time,re,sys,os,ast
import pandas as pd
import pymongo
from pymongo import MongoClient
import random,operator
from collections import OrderedDict
from bson.son import SON
from bson.codec_options import CodecOptions
import urllib,urllib2
import json
from xml.dom import minidom
from xml.etree import ElementTree as ET
from io import StringIO
from bs4 import BeautifulSoup
from itertools import ifilter
import glob
import fnmatch
from multiprocessing import Pool
from functools import partial
import itertools


from flask import Flask,jsonify,request, render_template, send_from_directory, url_for,Response
from flask import session
from flask import send_file

#--- create database instances---
# Environment variables
mongodb_host = os.environ.get("MONGODB_HOST","0.0.0.0") # change to biotm2.cis.udel.edu before dockerizing
mongodb_port = os.environ.get("MONGODB_PORT","27017")
db_name = os.environ.get("DBNAME","glygen") # change database name for your own dbName
textCollectionName = os.environ.get("COLLECTION_TEXT","glygenText")
entityCollectionName = os.environ.get("COLLECTION_ENTITY","glygenEntities")

# Database URI
MONGODB_URI = 'mongodb://'+mongodb_host+':'+mongodb_port+'/'

# Database object
client = MongoClient(MONGODB_URI)
opts = CodecOptions(document_class=SON)

# Database
dbName = client[db_name] # glygen

# Collection
textCollection = dbName[textCollectionName].with_options(codec_options=opts)
entityCollection = dbName[entityCollectionName].with_options(codec_options=opts)
pmidListMongo = textCollection.distinct('docId')

def queryNCBI(termString,minPmid,maxPmid):
    print(termString)
    # ncbi_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=%s&retstart=%s&retmax=%s" %(termString,minPmid,maxPmid)
    # https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=cancer

    '''
    termString is the query string
    '''

    values = { 'db' : "pubmed", 'term' : termString,'retstart' : minPmid, 'retmax' : maxPmid, 'mindate':"1990/01/01", 'maxdate': "2018/12/31"}
    data = urllib.urlencode(values)

    ncbi_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?"


    req = urllib2.Request(ncbi_url, data)
    my_response = urllib2.urlopen(req)

    # my_response = urllib2.urlopen(ncbi_url)
    htmlResp = my_response.read().decode('utf-8')
    t = ET.fromstring(htmlResp)

    pmidList = []
    for child in t.iter('Id'):
        pmidList.append(child.text)

    countList = []
    for child in t.iter('Count'):
        countList.append(child.text)

    return (pmidList,countList)

if __name__ == "__main__":


    # pmidListMongo is the list of docId from your database

    userquery = "GBM AND EGFR"
    fixedQuery = " AND (miRNA OR microRNA OR miR)"
    query = userquery + fixedQuery

    minPmid = 0
    maxPmid = 8000
    (pmidList,countList) = queryNCBI(query,str(minPmid),str(maxPmid))
    count = int(countList[0])
    # print(len(pmidList),count)
    pmidListFoundInMongo = list(set(pmidList).intersection(set(pmidListMongo)))
    # print(pmidListFoundInMongo)