# -*- coding: utf-8 -*-
"""
Created on Mon June  13 12:22:40 2017

@author: bartosz
"""

from pathlib import Path
import json
import re
import csv
import os

####################33
####### INPUT
###############################

outputFileName = "3_med"
disciplineNumber = "3"
queries = ["aetiological diagnosis",
            "atherosclerosis diagnosis",
            "atherosclerosis disease",
            "breast cancer mortality rate",
            "cardiac chamber size",
            "central nervous system signalling",
            "chamber quantification",
            "chronic kidney disease diagnosis",
            "chronic kidney disease guideline",
            "chronic kidney disease progression",
            "chronic kidney disease treatment",
            "complication classification",
            "complication management",
            "composite international diagnostic interview",
            "drug craving",
            "drug discovery setting",
            "drug permeability",
            "dsm-iv disorder",
            "echocardiography contrast",
            "fvc maneuver",
            "hcc treatment",
            "heart ischemia",
            "hepatic artery catheterization",
            "hepatobiliary surgeons",
            "high plasma concentrations",
            "hypertension prevention",
            "incentive-sensitization theory",
            "low-density lipoprotein effects",
            "lung cancer risk factor",
            "metabolic syndrome diagnosis",
            "neuritic plaque",
            "neuroadaptation",
            "neurofibrillary change",
            "neuropil thread",
            "obesity measure",
            "plasma cholesterol concentration",
            "pressor agent",
            "thiazide-type diuretic",
            "type 2 diabetes mellitus risk"]

#########################################
#######   functions
#########################################


def normaliseText(text):
    "function normalisng text to lowercase letters, removing all non-alphanumeric signs"
    text = text.lower()
    text = re.sub('[^a-zA-Z0-9\n\.]', ' ', text)
    text = text.replace(".", "")
    text = " ".join(text.split())
    return (text)

def FilterForQueryInTitle (stringSearchedFor, jsonList):
    query = normaliseText(stringSearchedFor)
    listToRemove = []
    for i in range(len(jsonList)):
        titleNormalised = normaliseText(jsonList[i]['Ti'])
        pluralQuery = str(query+"s") #as Scopus takes into account plural forms of queries (e.g. treatment -> treatments), this was introduced to check whether the exact form of the query is in title (MA, GS - do not take plural forms into account)
        if (query not in titleNormalised or pluralQuery in titleNormalised):
                listToRemove.append(i)
    invertedList = sorted(listToRemove, reverse=True)
    for x in invertedList:
        jsonList.pop(x)
    
    return jsonList

#google scholar function to unify schema!!!! omitting [CITATION] in Google Scholar -> 

def googleScholarSchemaUnifier (json_data):
    outputContent = []
    outputIndexedContent = []
    #shortening output - removing abstract, bag-of-words, list of sources
    for element in json_data:
        #print (element)
        #create new element which is to have unified schema
        
        shortenedEntity = {}
        if 'authors' in element: shortenedEntity['AA'] = element['authors']
        if 'cites' in element: shortenedEntity['CC'] = element['cites']
        if 'title' in element: shortenedEntity['Ti'] = normaliseText(element['title'])
        if 'source' in element: shortenedEntity['Venue'] = element['source']
        if 'year' in element: shortenedEntity['Y'] = element['year']
        if 'type' in element: shortenedEntity['gsType'] = element ['type']
        if 'article_url' in element: shortenedEntity['S'] = element ['article_url']
        
        #if already transformed to a new format, copy
        if 'AA' in element: shortenedEntity = element
        if 'SU' in element: 
            shortenedEntity['S'] = element ['SU']
            del shortenedEntity['SU']
        if 'Ti' in element: shortenedEntity['Ti'] = normaliseText(element['Ti'])
        
        outputIndexedContent.append(shortenedEntity) #if select only docs without [citation] marking in gs, change to outputContent
        
        #count of GS docs without '[CITATION]' type documents (without verified source / link)
        """if 'gsType' not in shortenedEntity: 
            outputIndexedContent.append(shortenedEntity)
        elif shortenedEntity['gsType'] != 'CITATION':
            outputIndexedContent.append(shortenedEntity)"""
        
    return(outputContent, outputIndexedContent)

def parsingJsonFile (fullFileName):
    with open(fullFileName, encoding='utf-8') as gs_data_file:    
        json_data = json.load(gs_data_file)
    
    #print(json_data)
    # filter for query in title of each document (iterate through list)
    (gsListUnifiedWithCITATION, gsListUnifiedWithoutCITATION)  = googleScholarSchemaUnifier (json_data)
    #print(gsListUnifiedWithoutCITATION[0])
    
    gsListFiltered = FilterForQueryInTitle (query, gsListUnifiedWithoutCITATION)
    
    # create stats: how many documents with phrase in title found for each query
    noDocs = len(gsListFiltered)
    noCitations = 0
    for doc in gsListFiltered:
        if 'CC' in doc: docCitations = int(doc['CC'])
        else: docCitations=0
        noCitations = noCitations + docCitations
    
    
    
    # save to a joint file (append to large array), print stats (?)
    for doc in gsListFiltered:
        gsListShortened.append(doc)
        
    with open(fullFileName, 'w+') as outfile:
        json.dump(gsListFiltered, outfile)
    queryStats = [query, noDocs]
    print ("query: %s, docs: %s, citations: %s" % (query, noDocs, noCitations))
    
    return queryStats

########################################
#code
##########################################


noDocsStats = []
noCitationStats = []
gsListShortened = []
noKeywordFilesFound = 0
stats = []            

for query in queries:
    # from list of queries, read query, find file with that query in title
    
    query2 = normaliseText(query)
    queryFileName = query2.replace(" ", "_")
    fullFileName = Path("data/individual_queries/gs_"+disciplineNumber+"_"+queryFileName+".json")
    
    if fullFileName.exists():
        #print ("<%s> data found" % query)
        noKeywordFilesFound = noKeywordFilesFound + 1
        fullFileName = str("data/individual_queries/gs_"+disciplineNumber+"_"+queryFileName+".json")
        queryStats = parsingJsonFile(fullFileName)
        stats.append(queryStats)
    else:
        print ("<%s> data NOT FOUND, filename: %s" % (query, fullFileName))
    
# save
#saving shortened json 

if noKeywordFilesFound == len(queries):
    print ("All queries data files found: OK!")


statsFileName = str('logs/stats/gs_%s.csv' % outputFileName)
dataFileName = str('data/disciplines/google_scholar_keyword_%s.json' % outputFileName)


#verify if folder structre exists, otherwise create
os.makedirs(os.path.dirname(statsFileName), exist_ok=True)
os.makedirs(os.path.dirname(dataFileName), exist_ok=True)

#print (stats)
#save stats and filtered data
with open(statsFileName,'w+', newline='') as f:
    w = csv.writer(f)
    w.writerows(stats)
    #for entry in stats:
    #    w.writerow(entry)

with open(dataFileName, 'w+') as outfile:
    json.dump(gsListShortened, outfile)
    
    

# -*- coding: utf-8 -*-
"""
Created on Mon June  13 12:22:40 2017

@author: bartosz
"""

from pathlib import Path
import json
import re
import csv
import os

####################33
####### INPUT
###############################

outputFileName = "3_med"
disciplineNumber = "3"
queries = ["aetiological diagnosis",
            "atherosclerosis diagnosis",
            "atherosclerosis disease",
            "breast cancer mortality rate",
            "cardiac chamber size",
            "central nervous system signalling",
            "chamber quantification",
            "chronic kidney disease diagnosis",
            "chronic kidney disease guideline",
            "chronic kidney disease progression",
            "chronic kidney disease treatment",
            "complication classification",
            "complication management",
            "composite international diagnostic interview",
            "drug craving",
            "drug discovery setting",
            "drug permeability",
            "dsm-iv disorder",
            "echocardiography contrast",
            "fvc maneuver",
            "hcc treatment",
            "heart ischemia",
            "hepatic artery catheterization",
            "hepatobiliary surgeons",
            "high plasma concentrations",
            "hypertension prevention",
            "incentive-sensitization theory",
            "low-density lipoprotein effects",
            "lung cancer risk factor",
            "metabolic syndrome diagnosis",
            "neuritic plaque",
            "neuroadaptation",
            "neurofibrillary change",
            "neuropil thread",
            "obesity measure",
            "plasma cholesterol concentration",
            "pressor agent",
            "thiazide-type diuretic",
            "type 2 diabetes mellitus risk"]

#########################################
#######   functions
#########################################


def normaliseText(text):
    "function normalisng text to lowercase letters, removing all non-alphanumeric signs"
    text = text.lower()
    text = re.sub('[^a-zA-Z0-9\n\.]', ' ', text)
    text = text.replace(".", "")
    text = " ".join(text.split())
    return (text)

def FilterForQueryInTitle (stringSearchedFor, jsonList):
    query = normaliseText(stringSearchedFor)
    listToRemove = []
    for i in range(len(jsonList)):
        titleNormalised = normaliseText(jsonList[i]['Ti'])
        pluralQuery = str(query+"s") #as Scopus takes into account plural forms of queries (e.g. treatment -> treatments), this was introduced to check whether the exact form of the query is in title (MA, GS - do not take plural forms into account)
        if (query not in titleNormalised or pluralQuery in titleNormalised):
                listToRemove.append(i)
    invertedList = sorted(listToRemove, reverse=True)
    for x in invertedList:
        jsonList.pop(x)
    
    return jsonList

#google scholar function to unify schema!!!! omitting [CITATION] in Google Scholar -> 

def googleScholarSchemaUnifier (json_data):
    outputContent = []
    outputIndexedContent = []
    #shortening output - removing abstract, bag-of-words, list of sources
    for element in json_data:
        #print (element)
        #create new element which is to have unified schema
        
        shortenedEntity = {}
        if 'authors' in element: shortenedEntity['AA'] = element['authors']
        if 'cites' in element: shortenedEntity['CC'] = element['cites']
        if 'title' in element: shortenedEntity['Ti'] = normaliseText(element['title'])
        if 'source' in element: shortenedEntity['Venue'] = element['source']
        if 'year' in element: shortenedEntity['Y'] = element['year']
        if 'type' in element: shortenedEntity['gsType'] = element ['type']
        if 'article_url' in element: shortenedEntity['S'] = element ['article_url']
        
        #if already transformed to a new format, copy
        if 'AA' in element: shortenedEntity = element
        if 'SU' in element: 
            shortenedEntity['S'] = element ['SU']
            del shortenedEntity['SU']
        if 'Ti' in element: shortenedEntity['Ti'] = normaliseText(element['Ti'])
        
        outputIndexedContent.append(shortenedEntity) #if select only docs without [citation] marking in gs, change to outputContent
        
        #count of GS docs without '[CITATION]' type documents (without verified source / link)
        """if 'gsType' not in shortenedEntity: 
            outputIndexedContent.append(shortenedEntity)
        elif shortenedEntity['gsType'] != 'CITATION':
            outputIndexedContent.append(shortenedEntity)"""
        
    return(outputContent, outputIndexedContent)

def parsingJsonFile (fullFileName):
    with open(fullFileName, encoding='utf-8') as gs_data_file:    
        json_data = json.load(gs_data_file)
    
    #print(json_data)
    # filter for query in title of each document (iterate through list)
    (gsListUnifiedWithCITATION, gsListUnifiedWithoutCITATION)  = googleScholarSchemaUnifier (json_data)
    #print(gsListUnifiedWithoutCITATION[0])
    
    gsListFiltered = FilterForQueryInTitle (query, gsListUnifiedWithoutCITATION)
    
    # create stats: how many documents with phrase in title found for each query
    noDocs = len(gsListFiltered)
    noCitations = 0
    for doc in gsListFiltered:
        if 'CC' in doc: docCitations = int(doc['CC'])
        else: docCitations=0
        noCitations = noCitations + docCitations
    
    
    
    # save to a joint file (append to large array), print stats (?)
    for doc in gsListFiltered:
        gsListShortened.append(doc)
        
    with open(fullFileName, 'w+') as outfile:
        json.dump(gsListFiltered, outfile)
    queryStats = [query, noDocs]
    print ("query: %s, docs: %s, citations: %s" % (query, noDocs, noCitations))
    
    return queryStats

########################################
#code
##########################################


noDocsStats = []
noCitationStats = []
gsListShortened = []
noKeywordFilesFound = 0
stats = []            

for query in queries:
    # from list of queries, read query, find file with that query in title
    
    query2 = normaliseText(query)
    queryFileName = query2.replace(" ", "_")
    fullFileName = Path("data/individual_queries/gs_"+disciplineNumber+"_"+queryFileName+".json")
    
    if fullFileName.exists():
        #print ("<%s> data found" % query)
        noKeywordFilesFound = noKeywordFilesFound + 1
        fullFileName = str("data/individual_queries/gs_"+disciplineNumber+"_"+queryFileName+".json")
        queryStats = parsingJsonFile(fullFileName)
        stats.append(queryStats)
    else:
        print ("<%s> data NOT FOUND, filename: %s" % (query, fullFileName))
    
# save
#saving shortened json 

if noKeywordFilesFound == len(queries):
    print ("All queries data files found: OK!")


statsFileName = str('logs/stats/gs_%s.csv' % outputFileName)
dataFileName = str('data/disciplines/google_scholar_keyword_%s.json' % outputFileName)


#verify if folder structre exists, otherwise create
os.makedirs(os.path.dirname(statsFileName), exist_ok=True)
os.makedirs(os.path.dirname(dataFileName), exist_ok=True)

#print (stats)
#save stats and filtered data
with open(statsFileName,'w+', newline='') as f:
    w = csv.writer(f)
    w.writerows(stats)
    #for entry in stats:
    #    w.writerow(entry)

with open(dataFileName, 'w+') as outfile:
    json.dump(gsListShortened, outfile)
    
    

# -*- coding: utf-8 -*- Created on Mon June  13 12:22:40 2017
 
 @author: bartosz ####################33 ####### INPUT ############################### ######################################### #######   functions ######################################### #as Scopus takes into account plural forms of queries (e.g. treatment -> treatments), this was introduced to check whether the exact form of the query is in title (MA, GS - do not take plural forms into account) #google scholar function to unify schema!!!! omitting [CITATION] in Google Scholar -> #shortening output - removing abstract, bag-of-words, list of sources #print (element) #create new element which is to have unified schema #if already transformed to a new format, copy #if select only docs without [citation] marking in gs, change to outputContent #count of GS docs without '[CITATION]' type documents (without verified source / link) if 'gsType' not in shortenedEntity: 
             outputIndexedContent.append(shortenedEntity)
         elif shortenedEntity['gsType'] != 'CITATION':
             outputIndexedContent.append(shortenedEntity) #print(json_data) # filter for query in title of each document (iterate through list) #print(gsListUnifiedWithoutCITATION[0]) # create stats: how many documents with phrase in title found for each query # save to a joint file (append to large array), print stats (?) ######################################## #code ########################################## # from list of queries, read query, find file with that query in title #print ("<%s> data found" % query) # save #saving shortened json #verify if folder structre exists, otherwise create #print (stats) #save stats and filtered data #for entry in stats: #    w.writerow(entry)