glueberry

You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

7.9 KiB

Raw Blame History

In [1]:

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO )

In [2]:

from collections import defaultdict
from gensim import corpora

documents = [
    "Human machine interface for lab abc computer applications",
    "A survey of user opinion of computer system response time",
    "The EPS user interface management system",
    "System and human system engineering testing of EPS",
    "Relation of user perceived response time to error measurement",
    "The generation of random binary unordered trees",
    "The intersection graph of paths in trees",
    "Graph minors IV Widths of trees and well quasi ordering",
    "Graph minors A survey",
]

# remove common words and tokenize
stoplist = set('for a of the and to in'.split())
texts = [
    [word for word in document.lower().split() if word not in stoplist]
    for document in documents
]

# remove words that appear only once
frequency = defaultdict(int)
for text in texts:
    for token in text:
        frequency[token] += 1

texts = [
    [token for token in text if frequency[token] > 1]
    for text in texts
]

dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]

2022-05-23 18:15:06,938 : INFO : adding document #0 to Dictionary<0 unique tokens: []>
2022-05-23 18:15:06,939 : INFO : built Dictionary<12 unique tokens: ['computer', 'human', 'interface', 'response', 'survey']...> from 9 documents (total 29 corpus positions)
2022-05-23 18:15:06,939 : INFO : Dictionary lifecycle event {'msg': "built Dictionary<12 unique tokens: ['computer', 'human', 'interface', 'response', 'survey']...> from 9 documents (total 29 corpus positions)", 'datetime': '2022-05-23T18:15:06.939467', 'gensim': '4.2.0', 'python': '3.10.2 (tags/v3.10.2:a58ebcc, Jan 17 2022, 14:12:15) [MSC v.1929 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22000-SP0', 'event': 'created'}

In [3]:

from gensim import models

tfidf = models.TfidfModel(corpus) # 1. initialize the model

2022-05-23 18:16:10,608 : INFO : collecting document frequencies
2022-05-23 18:16:10,609 : INFO : PROGRESS: processing document #0
2022-05-23 18:16:10,609 : INFO : TfidfModel lifecycle event {'msg': 'calculated IDF weights for 9 documents and 12 features (28 matrix non-zeros)', 'datetime': '2022-05-23T18:16:10.609938', 'gensim': '4.2.0', 'python': '3.10.2 (tags/v3.10.2:a58ebcc, Jan 17 2022, 14:12:15) [MSC v.1929 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22000-SP0', 'event': 'initialize'}

In [5]:

doc_bow = [(0,1), (1,1)]
print(tfidf[doc_bow]) # 2. use the model to transform vectors

[(0, 0.7071067811865476), (1, 0.7071067811865476)]

In [6]:

corpus_tfidf = tfidf[corpus]
for doc in corpus_tfidf:
    print(doc)

[(0, 0.5773502691896257), (1, 0.5773502691896257), (2, 0.5773502691896257)]
[(0, 0.44424552527467476), (3, 0.44424552527467476), (4, 0.44424552527467476), (5, 0.3244870206138555), (6, 0.44424552527467476), (7, 0.3244870206138555)]
[(2, 0.5710059809418182), (5, 0.4170757362022777), (7, 0.4170757362022777), (8, 0.5710059809418182)]
[(1, 0.49182558987264147), (5, 0.7184811607083769), (8, 0.49182558987264147)]
[(3, 0.6282580468670046), (6, 0.6282580468670046), (7, 0.45889394536615247)]
[(9, 1.0)]
[(9, 0.7071067811865475), (10, 0.7071067811865475)]
[(9, 0.5080429008916749), (10, 0.5080429008916749), (11, 0.695546419520037)]
[(4, 0.6282580468670046), (10, 0.45889394536615247), (11, 0.6282580468670046)]

In [9]:

lsi_model = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2) # initialize an LSI transf
corpus_lsi = lsi_model[corpus_tfidf] # create a double wrapper over the original

2022-05-23 18:40:30,354 : INFO : using serial LSI version on this node
2022-05-23 18:40:30,354 : INFO : updating model with new documents
2022-05-23 18:40:30,355 : INFO : preparing a new chunk of documents
2022-05-23 18:40:30,358 : INFO : using 100 extra samples and 2 power iterations
2022-05-23 18:40:30,359 : INFO : 1st phase: constructing (12, 102) action matrix
2022-05-23 18:40:30,360 : INFO : orthonormalizing (12, 102) action matrix
2022-05-23 18:40:30,362 : INFO : 2nd phase: running dense svd on (12, 9) matrix
2022-05-23 18:40:30,363 : INFO : computing the final decomposition
2022-05-23 18:40:30,364 : INFO : keeping 2 factors (discarding 47.565% of energy spectrum)
2022-05-23 18:40:30,365 : INFO : processed documents up to #9
2022-05-23 18:40:30,365 : INFO : topic #0(1.594): 0.703*"trees" + 0.538*"graph" + 0.402*"minors" + 0.187*"survey" + 0.061*"system" + 0.060*"time" + 0.060*"response" + 0.058*"user" + 0.049*"computer" + 0.035*"interface"
2022-05-23 18:40:30,365 : INFO : topic #1(1.476): 0.460*"system" + 0.373*"user" + 0.332*"eps" + 0.328*"interface" + 0.320*"response" + 0.320*"time" + 0.293*"computer" + 0.280*"human" + 0.171*"survey" + -0.161*"trees"
2022-05-23 18:40:30,365 : INFO : LsiModel lifecycle event {'msg': 'trained LsiModel<num_terms=12, num_topics=2, decay=1.0, chunksize=20000> in 0.01s', 'datetime': '2022-05-23T18:40:30.365269', 'gensim': '4.2.0', 'python': '3.10.2 (tags/v3.10.2:a58ebcc, Jan 17 2022, 14:12:15) [MSC v.1929 64 bit (AMD64)]', 'platform': 'Windows-10-10.0.22000-SP0', 'event': 'created'}

In [12]:

lsi_model.print_topic(2)

Out[12]:

''

In [ ]:

7.9 KiB Raw Blame History

7.9 KiB

Raw Blame History