Study of the temporal evolution of a corpus through word embedding¶
Author(s):¶
- Ale Abdo abdo@member.fsf.org
License:¶
Project:¶
Reference repository for this file:¶
https://gitlab.com/solstag/oncologister-pvt
Contributions are welcome, get in touch with the author(s).
In [1]:
# This notebook requires Pandas, Gensim and Lxml
import re, multiprocessing, numpy, json, os, sys, math
import pandas, gensim
from collections import defaultdict, namedtuple, OrderedDict
from copy import deepcopy
import matplotlib.pyplot as plt
from lxml import html
%matplotlib inline
In [2]:
# Options used to run the notebook
opt_your_initials = 'aha' # SET THIS BEFORE RUNNING, i.e. 'aha', 'jpc', ...
opt_data_source = 'getpapersdir'
opt_training_mode = 'randomfill'
opt_model_iter = 5
opt_name_extra = ''
opt_model_name = '-'.join( [opt_your_initials, opt_data_source, opt_training_mode, str(opt_model_iter)] )
opt_model_name += '-' + opt_name_extra if opt_name_extra else ''
opt_load_models = True
In [3]:
# Source abstracts from JSON files generated by esmo.py
def get_esmo(ayears, esmo_path):
# get converted PDF files
for year in os.listdir(esmo_path):
year_path = os.path.join(esmo_path,year,'xml')
for sec in os.listdir(year_path):
if sec.endswith('.json'):
sec_path = os.path.join(year_path,sec)
with open(sec_path) as sec_file:
content = json.load(sec_file)
ayears[int(year)].abstracts.extend( [ l['text'] for l in item['abstract'] ]
for item in content if item['separator']!=None )
# get converted HTML files
for year in os.listdir(esmo_path):
year_path = os.path.join(esmo_path,year,'content')
for sec in os.listdir(year_path):
if sec.endswith('.json'):
sec_path = os.path.join(year_path,sec)
with open(sec_path) as sec_file:
content = json.load(sec_file)
for a in content['abstracts']:
htext = html.fromstring(a['text'])
htext.find('h2').getparent().remove(htext.find('h2'))
abstract = [ a['title'],
' '.join(a['authors'].keys()),
' '.join(sum(a['authors'].values(),[])),
htext.text_content() ]
ayears[int(year)].abstracts.append( abstract )
# Source abstracts from JSON files generated by ContentMine's getpapers.js
def get_getpapers(ayears, getpapers_path):
names = [n for n in os.listdir(getpapers_path) if os.path.isdir(os.path.join(getpapers_path, n))]
noabstract = []
for name in names:
with open(os.path.join(getpapers_path, name, 'eupmc_result.json')) as f:
content = json.load(f)
try:
year = int(content["journalInfo"][0]["yearOfPublication"][0])
abstract = content["abstractText"][0]
pmid = content["id"][0]
citation = int(content["citedByCount"][0])
title = content["title"][0]
if any( map( lambda x: len(x)!=1,
(content["abstractText"], content["id"], content["citedByCount"],
content["journalInfo"][0]["yearOfPublication"]) ) ):
raise Exception('Non-unique item!')
if not len(abstract):
raise Exception('Empty abstract!')
ayears[year].abstracts.append([abstract])
ayears[year].pmids.append(pmid)
ayears[year].citations.append(citation)
ayears[year].titles.append(title)
except KeyError:
noabstract.append(name)
if noabstract:
print('In <'+getpapers_path+'> missing "abstractText":', ', '.join(noabstract))
In [4]:
# Create the main data structure
ArticleSeries = namedtuple('Articles', ['abstracts', 'pmids', 'citations','titles'])
ayears = defaultdict(lambda: ArticleSeries([], [], [], []))
# ESMO abstracts from esmo.py
if opt_data_source=='esmo':
esmo_path='../data/esmo/auto_esmo'
get_esmo(ayears, esmo_path)
# Getpapers single dir
if opt_data_source=='getpaperssingle':
getpapers_path = '/home/aedesaegypti/contentmine/papers/diarrhea_angola_orig/'
get_getpapers(ayears, getpapers_path)
# Getpapers dir store
if opt_data_source=='getpapersdir':
getpapers_store = '../data/gotpapers/'
getpapers_re = re.compile('gp-mesh_Breast_Neoplasms-research_article-en-')
getpapers_paths = [os.path.join(getpapers_store, name)
for name in os.listdir(getpapers_store)
if os.path.isdir(os.path.join(getpapers_store, name)) \
and getpapers_re.match(name)]
# Tell us where we'll look for papers
print('## Abstract paths:')
for path in getpapers_paths:
print(path)
print()
print('## Defective abstracts:')
for getpapers_path in getpapers_paths:
get_getpapers(ayears, getpapers_path)
print()
# Transform into OrderedDict of DataFrames
ayears = OrderedDict( sorted( ayears.items() ) )
for year in ayears:
ayears[year] = pandas.DataFrame( ayears[year]._asdict() )
# Tell us what we've got
print('## Abstract count for {}:'.format(opt_data_source))
for year in ayears:
print(year, len(ayears[year]))
print('Total:', sum( map( len, ayears.values() ) ), '\n')
In [5]:
# Split words and do some cleaning
def linestowords(document):
splitter = re.compile('[%=:;,"\'\(\)\[\]\{\}\<\>\.]+')
words = [word
for line in document
for inter in line.casefold().split()
for dirty in splitter.split(inter)
for word in [dirty.strip('.-')]
if len(word)>1 and not word.isdigit()]
return words
for year in ayears:
ayears[year]['originals'] = ayears[year]['abstracts']
ayears[year]['abstracts'] = [ linestowords(abstract) for abstract in ayears[year]['abstracts'] ]
ayears[year] = ayears[year][ ayears[year]['abstracts'].apply(bool) ]
ayears[year].index = range( len( ayears[year] ) )
# Identify and join "bigrams" so ['são', 'paulo'] becomes ['são_paulo']
bigram = gensim.models.phrases.Phrases( [ abstract
for year in ayears
for abstract in ayears[year]['abstracts'] ] )
for year in ayears:
ayears[year]['abstracts'] = [ bigram[abstract] for abstract in ayears[year]['abstracts'] ]
In [6]:
# Modeling time!
yearmodels=dict()
if opt_load_models:
# Load existing models from saved files
for year in ayears:
model_path = '-'.join(['w2v', opt_model_name, str(year)])
if os.path.exists( model_path ):
yearmodels[year] = gensim.models.Word2Vec.load( model_path )
missing_years = set(ayears).difference( set(yearmodels) )
if missing_years:
basemodel_path = '-'.join(['w2v', opt_model_name, 'base'])
if os.path.exists( basemodel_path ):
# Load existing basemodel
basemodel = gensim.models.Word2Vec.load( basemodel_path )
else:
# Create the base model, hs=1 and negative=0 are required by .score()
basemodel = gensim.models.Word2Vec(workers=multiprocessing.cpu_count(),
iter = opt_model_iter,
hs=1, negative=0
)
# Create a copy of the abstracts as the original is needed for analysis
model_ayears = deepcopy(ayears)
# Shuffle abstracts, because order matters in training
for year in model_ayears:
index = list(model_ayears[year].index)
numpy.random.shuffle(index)
model_ayears[year].index = index
model_ayears[year].sort_index(inplace=True)
# Balance abstracts if we chose to
if opt_training_mode == 'nothing':
pass
elif opt_training_mode == 'randomfill':
# Fill them up to match the largest corpus to see if this fixes the size bias
maxsize = max( map( len, model_ayears.values() ) )
for year in model_ayears:
size = len(model_ayears[year])
factor = int(maxsize/size)
rest = maxsize - size*factor
model_ayears[year] = pandas.concat( [ model_ayears[year] for i in range(factor) ] \
+ [ model_ayears[year][:rest] ], ignore_index=True )
elif opt_training_mode == 'randomsample':
# Cut to same size to see if this fixes the size bias
minsize = 1000 # min( map(len, model_ayears.values() )) -> 96 and 98 were strong
for year in model_ayears:
model_ayears[year] = model_ayears[year][:minsize]
else:
raise Exception('opt_training_mode not found: ' + opt_training_mode)
# Build common vocabulary from all years
basemodel.build_vocab( pandas.concat( [model_ayears[year]['abstracts'] for year in model_ayears],
ignore_index=True ) )
# Save the basemodel
basemodel.save( basemodel_path )
print(basemodel)
# Create a model for each year and train it
for year in sorted(missing_years):
yearmodels[year] = deepcopy(basemodel)
yearmodels[year].train(model_ayears[year]['abstracts'], total_examples=len(model_ayears[year]))
yearmodels[year].save( '-'.join(['w2v', opt_model_name, str(year)]) )
In [7]:
def calc_probs(ayears, yearmodels):
for year_a in ayears:
# the log likelihood of each sentence in this review under each w2v representation
llhd = pandas.DataFrame( ( numpy.float128(yearmodels[year_m].score( ayears[year_a]['abstracts'],
len(ayears[year_a]) ) )
for year_m in ayears ),
index=list(ayears) )
# now exponentiate to get likelihoods,
lhd = numpy.exp(llhd - llhd[llhd.index!=year_a].max(axis=0)) # subtract row max to avoid numeric overload
# normalize across models (years) to get sentence-year probabilities
plhd = (lhd/lhd[lhd.index!=year_a].sum(axis=0))
# add probabilities to our yearly dataframe
llhd.index = ['l'+str(c) for c in llhd.index]
plhd.index = ['p'+str(c) for c in plhd.index]
ayears[year_a] = pandas.concat( [ayears[year_a], llhd.T, plhd.T], axis=1)
In [8]:
pi = lambda x: ['p'+str(y) for y in x] if isinstance(x, list) else 'p'+str(x)
li = lambda x: ['l'+str(y) for y in x] if isinstance(x, list) else 'l'+str(x)
In [9]:
calc_probs(ayears, yearmodels)
In [10]:
def gen_plots(ayears):
probs = [pi(year) for year in ayears]
for year in ayears:
ayears[year]['year']=year
allyears = pandas.concat( [ayears[year] for year in ayears] )
for year in ayears:
allyears.loc[allyears.year==year, pi(year)] = numpy.NaN
# Each subplot is the model trained from that year crossed with the corpus for each year
axs = allyears.boxplot( probs, by='year', whis=[5,95], showmeans=1, layout=(9,3),
figsize=(17,17), return_type='axes')
for ax in axs.values():
ax.set_xticklabels([x.get_text()[-2:] for x in ax.get_xticklabels()])
plt.title('Models by data')
# Each subplot is the corpus for that year crossed with each of the models
plt.figure(figsize=(17,17))
for i, target in enumerate( ayears ):
plt.subplot(9, 3, i+1)
ax = allyears.loc[allyears.year==target].boxplot(probs, whis=[5,95], showmeans=1, return_type='axes')
ax.set_title(str(target))
ax.set_xticklabels([x.get_text()[-2:] for x in ax.get_xticklabels()])
plt.title('Data by models')
plt.tight_layout()
# Histogram of abstracts per year
plt.figure(figsize=(17,10))
plt.bar(ayears, [len(ayears[years]) for years in ayears])
plt.title('Number of abstracts per year')
In [11]:
#gen_plots(ayears)
In [12]:
def calc_lratios(ayears):
# Calculate the ratio between an abstracts likelihood in its own year and on the next
nextyear = dict( (y, list(ayears)[i+1]) for i, y in enumerate(list(ayears)[:-1]) )
prevyear = dict( (y, list(ayears)[i-1]) for i, y in enumerate(list(ayears)[1:]) )
nextyears = dict( (y, list(ayears)[i+1:]) for i, y in enumerate(list(ayears)[:-1]) )
prevyears = dict( (y, list(ayears)[i-1:]) for i, y in enumerate(list(ayears)[1:]) )
xi=li
def lrsavg(ayears, year, target):
return ayears[year].loc[ :, xi(target) ].sub(
ayears[year].loc[ :, xi(target) ].mean(axis=0), axis=1 if isinstance(target, list) else 0 )
for year in list(ayears):
ayears[year]['lrsavg'] = lrsavg(ayears, year, year)
for year in list(ayears)[:-1]:
ayears[year]['lrnext'] = ayears[year].loc[ :, xi(nextyear[year]) ] - ayears[year].loc[ :, xi(year) ]
ayears[year]['lrsavgnext'] = lrsavg(ayears, year, nextyear[year]).sub(lrsavg(ayears, year, year),axis=0)
ayears[year]['lrnexts'] = ayears[year].loc[ :, xi(nextyears[year]) ].sub(
ayears[year].loc[ :, xi(year) ], axis=0 ).sum(axis=1)
ayears[year]['lrsavgnexts'] = lrsavg(ayears, year, nextyears[year]).sub(
lrsavg(ayears, year, year), axis=0 ).sum(axis=1)
ayears[year]['lrsavgnextmax'] = lrsavg(ayears, year, nextyears[year]).sub(
lrsavg(ayears, year, year), axis=0 ).max(axis=1)
for year in list(ayears)[1:]:
ayears[year]['lrprev'] = ayears[year].loc[ :, xi(year) ] - ayears[year].loc[ :, xi(prevyear[year]) ]
for year in list(ayears)[1:-1]:
ayears[year]['lrjump'] = ayears[year].loc[ :, xi(nextyear[year]) ] - \
ayears[year].loc[ :, xi(prevyear[year]) ]
def find_lratios(ayear, year, ratio='lrjump'):
# Find the 10 largest ratios for a given year
return ayears[year][ratio].sort_values(ascending=False)[:10]
def print_lratios(ayears):
for year in list(ayears)[1:-1]:
print(year, '\n', find_lratios(ayears, year), '\n')
In [13]:
calc_lratios(ayears)
In [78]:
def print_lcorr(ayears, ratio='lrjump'):
for year in list(ayears)[1:-1]:
print('\n\n', year, '\n')
cyears = ayears[year][[ratio,'citations']]
print('\n\npearson\n', cyears.corr())
print('\n\kendall\n', cyears.corr('kendall'))
print('\n\spearman\n', cyears.corr('spearman'))
def plot_lcorr(ayears, ratio='lrjump'):
years = list(ayears)[-11:-1]
plt.figure(figsize=(17, len(years)*3))
for i, target in enumerate( years ):
ayear = ayears[target]
ax = plt.subplot(int(len(years)/2)+1, 2, i+1)
ax.set_title(ratio+' '+str(target))
ax.scatter(ayear['citations'], ayear[ratio])
bins = numpy.logspace( numpy.log10(ayear.citations.min()+1),
numpy.log10(ayear.citations.max()+1.2) ) - 1.1
ayear['citations-bin']=pandas.cut(ayear.citations, bins, labels=bins[1:])
r=ayear[['citations-bin', ratio]].groupby('citations-bin').mean()[ratio]
ax.plot(numpy.array(r.index), r, color='red', linewidth=2)
ax.axhline()
ax.set_xscale('log')
plt.xlim(xmin=0.8)
plt.tight_layout()
In [79]:
plot_lcorr(ayears, 'lrsavgnext')
plot_lcorr(ayears, 'lrsavgnexts')
plot_lcorr(ayears, 'lrsavgnextmax')
In [82]:
def pick_lcorr(ayears, year, ratio='lrsavgnexts'):
ayear = ayears[year]
low=ayear.sort_values(ratio)[:10]
high=ayear.sort_values(ratio)[-10:]
# print('Low {}:\n'.format(ratio), low.pmids)
# print('High {}:\n'.format(ratio), high.pmids)
mixed = numpy.array(pandas.concat([high.pmids, low.pmids]))
numpy.random.shuffle(mixed)
print('Mixed {}:\n'.format(ratio), mixed)
items = ayear.loc[ [x in mixed for x in ayear.pmids] , ['pmids','titles','originals'] ]
for i,p,t,o in items.itertuples():
print(p,'\n')
print(t,'\n')
print(o[0],'\n\n')
pick_lcorr(ayears, 2012, 'lrsavgnexts')
In [17]:
# Some inspection functions
def readindex(year, index):
# Reads the abstract with given index for the year given above
print( ' '.join(ayears[year]['abstracts'][index]) )
def wordinfo(word):
# Print count and similar words for a given word
for year in ayears:
m=yearmodels[year]
print(year)
print('Count: ', m.vocab[word].count)
print(m.most_similar(word, topn=3))
# Check if some abstracts are too small or too big
def list_abstracts_with_length(ayears, low=100, high=sys.maxsize):
for year in ayears:
selected=[]
for i, item in enumerate(ayears[year]['abstracts']):
if len(item) < low or len(item)>high:
selected.append((i,item))
print('len', year, [ len(x[1]) for x in selected])
print('idx', year, [ x[0] for x in selected])
print()
def wordstuff():
# Creates a list of words sorted by their total count
words = [ w for w, c in sorted(basemodel.vocab.items(), key=lambda x: x[1], reverse=True) ]
# Log plot of the word count in the order set above
plt.bar( [i for i in range(len(words))], numpy.log([basemodel.vocab[w].count for w in words]) )
# Prints some words with extreme counts:
for w in words[:10]+words[-10:]:
print(w, basemodel.vocab[w].count)
In [ ]: