worked on NLTK tokenize + POS tagger
parent
45ec6e0060
commit
cba64a7b99
Binary file not shown.
@ -0,0 +1,30 @@
|
|||||||
|
import nltk
|
||||||
|
|
||||||
|
file=open('faceapp.txt','r')
|
||||||
|
raw=file.read()
|
||||||
|
tokens = nltk.word_tokenize(raw)
|
||||||
|
faceapp = nltk.Text(tokens)
|
||||||
|
|
||||||
|
|
||||||
|
# my stopwords are common words I don't want to count, like "a", "an", "the".
|
||||||
|
stopwords = set(line.strip() for line in open('stopwords.txt'))
|
||||||
|
|
||||||
|
# dictionary
|
||||||
|
wordcount = {}
|
||||||
|
|
||||||
|
# spliting words from punctuation so "book" and "book!" counts as the same word
|
||||||
|
for word in raw.lower().split():
|
||||||
|
word = word.replace(".","")
|
||||||
|
word = word.replace(",","")
|
||||||
|
word = word.replace(":","")
|
||||||
|
word = word.replace("\"","")
|
||||||
|
word = word.replace("!","")
|
||||||
|
word = word.replace("“","")
|
||||||
|
word = word.replace("‘","")
|
||||||
|
word = word.replace("*","")
|
||||||
|
word = word.replace("(","")
|
||||||
|
word = word.replace(")","")
|
||||||
|
|
||||||
|
|
||||||
|
faceapp.concordance('a')
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1 @@
|
|||||||
|
If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account . If you permit others to use your account credentials , you are responsible for the activities of such users that occur in connection with your account .
|
@ -0,0 +1,65 @@
|
|||||||
|
* {
|
||||||
|
margin:0;
|
||||||
|
padding:0;
|
||||||
|
border:0;
|
||||||
|
outline:0;
|
||||||
|
|
||||||
|
}
|
||||||
|
body {
|
||||||
|
font-family: Helvetica,sans-serif;
|
||||||
|
font-weight: normal;
|
||||||
|
letter-spacing: 1px;
|
||||||
|
word-spacing: 2px;
|
||||||
|
font-size: 16px;
|
||||||
|
line-height: 1.3;
|
||||||
|
|
||||||
|
}
|
||||||
|
.menu {
|
||||||
|
background: #c0c0c0;
|
||||||
|
width: 100%;
|
||||||
|
padding: 10px;
|
||||||
|
font-size: 14px;
|
||||||
|
position:fixed;
|
||||||
|
}
|
||||||
|
.faceapp {
|
||||||
|
padding-top: 50px;
|
||||||
|
padding-left: 10px;
|
||||||
|
width: 40%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.coloniality-100 {
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 5px solid red;
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
span:hover {
|
||||||
|
color: grey;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
.coloniality-90 {
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 4px solid blue;
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
.coloniality-50 {
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 3px solid #b0ff00;
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
.coloniality-40 {
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 2px solid pink;
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
.sentence:hover {
|
||||||
|
background-color: #FFFF00
|
||||||
|
}
|
@ -0,0 +1,198 @@
|
|||||||
|
from __future__ import division
|
||||||
|
import glob
|
||||||
|
from nltk import *
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
import nltk
|
||||||
|
import codecs
|
||||||
|
from nltk import sent_tokenize, word_tokenize, pos_tag
|
||||||
|
from nltk.probability import FreqDist
|
||||||
|
from nltk.corpus import stopwords
|
||||||
|
nltk.download('stopwords')
|
||||||
|
|
||||||
|
|
||||||
|
#open the txt file, read, and tokenize
|
||||||
|
file = open('faceapp.txt','r')
|
||||||
|
text = file.read()
|
||||||
|
x = 1
|
||||||
|
|
||||||
|
#stopwords
|
||||||
|
default_stopwords = set(stopwords.words('english'))
|
||||||
|
custom_stopwords = set(codecs.open('stopwords.txt', 'r').read().splitlines())
|
||||||
|
all_stopwords = default_stopwords | custom_stopwords
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
print('''<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title></title>
|
||||||
|
<style>
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
# font-family: Belgika;
|
||||||
|
# font-weight: 8th;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
font-size: 20px;
|
||||||
|
line-height: 1.2;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
.NNP {
|
||||||
|
background-color: pink;
|
||||||
|
}
|
||||||
|
|
||||||
|
.VBP {
|
||||||
|
}
|
||||||
|
|
||||||
|
.VBP:hover {
|
||||||
|
background-color: gold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.NN {
|
||||||
|
background-color: LightSkyBlue;
|
||||||
|
}
|
||||||
|
|
||||||
|
.NNS {
|
||||||
|
background-color: Aquamarine;
|
||||||
|
}
|
||||||
|
|
||||||
|
.paragraph {
|
||||||
|
font-family: helvetica;
|
||||||
|
font-weight: regular;
|
||||||
|
width: 70%;
|
||||||
|
float: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
.top_words {
|
||||||
|
font-family: Belgika;
|
||||||
|
font-weight: 8th;
|
||||||
|
font-size: 9pt;
|
||||||
|
width: 25%;
|
||||||
|
float: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>''')
|
||||||
|
|
||||||
|
|
||||||
|
# my stopwords are common words I don't want to count, like "a", "an", "the".
|
||||||
|
|
||||||
|
print('<div class ="paragraph">')
|
||||||
|
# for sentence in sent_tokenize(text):
|
||||||
|
print('<span>')
|
||||||
|
|
||||||
|
tokenized = word_tokenize(text)
|
||||||
|
tagged = pos_tag(tokenized)
|
||||||
|
|
||||||
|
# for HTML
|
||||||
|
for word, pos in tagged:
|
||||||
|
print('<span class="{}">{}</span>'.format(pos, word))
|
||||||
|
|
||||||
|
print('</span>')
|
||||||
|
|
||||||
|
print('</div>')
|
||||||
|
|
||||||
|
# filtering stopwords
|
||||||
|
tokens_without_stopwords = nltk.FreqDist(words.lower() for words in tokenized if words.lower() not in all_stopwords)
|
||||||
|
print(tokens_without_stopwords)
|
||||||
|
|
||||||
|
# for read_whole_text in tokens_without_stopwords:
|
||||||
|
# whole_text_tokenized =
|
||||||
|
# print(whole_text_tokenized)
|
||||||
|
|
||||||
|
# #filtered words in sentence
|
||||||
|
# filtered_sentence = (" ").join(tokens_without_stopwords)
|
||||||
|
# print(filtered_sentence)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
print('<div class="top_words"> colonial words:')
|
||||||
|
|
||||||
|
frequency_word = FreqDist(tokens_without_stopwords)
|
||||||
|
top_words = tokens_without_stopwords.most_common(100)
|
||||||
|
|
||||||
|
for chosen_words, frequency in top_words:
|
||||||
|
print('<br><span class="chosen_words">{}({}) </span>'.format(chosen_words, frequency))
|
||||||
|
|
||||||
|
# new_html = open('output.html', 'wb') # open the output file
|
||||||
|
# new_html.write('''</div></body></html>''')
|
||||||
|
# new_html.close() # close the output file
|
||||||
|
|
||||||
|
|
||||||
|
print('''</div></body></html>''')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# # for new_file in tokens_without_stopwords:
|
||||||
|
# appendFile = open('tokenized_words.txt', 'a')
|
||||||
|
# appendFile.write(" " + new_file)
|
||||||
|
# appendFile.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# #shows only stopwords
|
||||||
|
# processed_word_list = []
|
||||||
|
|
||||||
|
# for word in tokenized:
|
||||||
|
# # print(word)
|
||||||
|
# if word not in all_stopwords:
|
||||||
|
# processed_word_list.append('*')
|
||||||
|
# else:
|
||||||
|
# processed_word_list.append(word)
|
||||||
|
# print(processed_word_list)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# # # result putting in a graph
|
||||||
|
# top_words_plot = frequency_word.plot(10)
|
||||||
|
# print(top_words_plot)
|
@ -1,65 +0,0 @@
|
|||||||
* {
|
|
||||||
margin:0;
|
|
||||||
padding:0;
|
|
||||||
border:0;
|
|
||||||
outline:0;
|
|
||||||
|
|
||||||
}
|
|
||||||
body {
|
|
||||||
font-family: Helvetica,sans-serif;
|
|
||||||
font-weight: normal;
|
|
||||||
letter-spacing: 1px;
|
|
||||||
word-spacing: 2px;
|
|
||||||
font-size: 16px;
|
|
||||||
line-height: 1.3;
|
|
||||||
|
|
||||||
}
|
|
||||||
.menu {
|
|
||||||
background: #c0c0c0;
|
|
||||||
width: 100%;
|
|
||||||
padding: 10px;
|
|
||||||
font-size: 14px;
|
|
||||||
position:fixed;
|
|
||||||
}
|
|
||||||
.faceapp {
|
|
||||||
padding-top: 50px;
|
|
||||||
padding-left: 10px;
|
|
||||||
width: 40%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.coloniality-100 {
|
|
||||||
text-decoration: none;
|
|
||||||
border-bottom: 5px solid red;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
span:hover {
|
|
||||||
color: grey;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
.coloniality-90 {
|
|
||||||
text-decoration: none;
|
|
||||||
border-bottom: 4px solid blue;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
.coloniality-50 {
|
|
||||||
text-decoration: none;
|
|
||||||
border-bottom: 3px solid #b0ff00;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
.coloniality-40 {
|
|
||||||
text-decoration: none;
|
|
||||||
border-bottom: 2px solid pink;
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
.sentence:hover {
|
|
||||||
background-color: #FFFF00
|
|
||||||
}
|
|
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,198 @@
|
|||||||
|
from __future__ import division
|
||||||
|
import glob
|
||||||
|
from nltk import *
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
import nltk
|
||||||
|
import codecs
|
||||||
|
from nltk import sent_tokenize, word_tokenize, pos_tag
|
||||||
|
from nltk.probability import FreqDist
|
||||||
|
from nltk.corpus import stopwords
|
||||||
|
nltk.download('stopwords')
|
||||||
|
|
||||||
|
|
||||||
|
#open the txt file, read, and tokenize
|
||||||
|
file = open('faceapp.txt','r')
|
||||||
|
text = file.read()
|
||||||
|
x = 1
|
||||||
|
|
||||||
|
#stopwords
|
||||||
|
default_stopwords = set(stopwords.words('english'))
|
||||||
|
custom_stopwords = set(codecs.open('stopwords.txt', 'r').read().splitlines())
|
||||||
|
all_stopwords = default_stopwords | custom_stopwords
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
print('''<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title></title>
|
||||||
|
<style>
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
# font-family: Belgika;
|
||||||
|
# font-weight: 8th;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
font-size: 20px;
|
||||||
|
line-height: 1.2;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
.NNP {
|
||||||
|
background-color: pink;
|
||||||
|
}
|
||||||
|
|
||||||
|
.VBP {
|
||||||
|
}
|
||||||
|
|
||||||
|
.VBP:hover {
|
||||||
|
background-color: gold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.NN {
|
||||||
|
background-color: LightSkyBlue;
|
||||||
|
}
|
||||||
|
|
||||||
|
.NNS {
|
||||||
|
background-color: Aquamarine;
|
||||||
|
}
|
||||||
|
|
||||||
|
.paragraph {
|
||||||
|
font-family: helvetica;
|
||||||
|
font-weight: regular;
|
||||||
|
width: 70%;
|
||||||
|
float: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
.top_words {
|
||||||
|
font-family: Belgika;
|
||||||
|
font-weight: 8th;
|
||||||
|
font-size: 9pt;
|
||||||
|
width: 25%;
|
||||||
|
float: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>''')
|
||||||
|
|
||||||
|
|
||||||
|
# my stopwords are common words I don't want to count, like "a", "an", "the".
|
||||||
|
|
||||||
|
print('<div class ="paragraph">')
|
||||||
|
# for sentence in sent_tokenize(text):
|
||||||
|
print('<span>')
|
||||||
|
|
||||||
|
tokenized = word_tokenize(text)
|
||||||
|
tagged = pos_tag(tokenized)
|
||||||
|
|
||||||
|
# for HTML
|
||||||
|
for word, pos in tagged:
|
||||||
|
print('<span class="{}">{}</span>'.format(pos, word))
|
||||||
|
|
||||||
|
print('</span>')
|
||||||
|
|
||||||
|
print('</div>')
|
||||||
|
|
||||||
|
# filtering stopwords
|
||||||
|
tokens_without_stopwords = nltk.FreqDist(words.lower() for words in tokenized if words.lower() not in all_stopwords)
|
||||||
|
print(tokens_without_stopwords)
|
||||||
|
|
||||||
|
# for read_whole_text in tokens_without_stopwords:
|
||||||
|
# whole_text_tokenized =
|
||||||
|
# print(whole_text_tokenized)
|
||||||
|
|
||||||
|
# #filtered words in sentence
|
||||||
|
# filtered_sentence = (" ").join(tokens_without_stopwords)
|
||||||
|
# print(filtered_sentence)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
print('<div class="top_words"> colonial words:')
|
||||||
|
|
||||||
|
frequency_word = FreqDist(tokens_without_stopwords)
|
||||||
|
top_words = tokens_without_stopwords.most_common(100)
|
||||||
|
|
||||||
|
for chosen_words, frequency in top_words:
|
||||||
|
print('<br><span class="chosen_words">{}({}) </span>'.format(chosen_words, frequency))
|
||||||
|
|
||||||
|
# new_html = open('output.html', 'wb') # open the output file
|
||||||
|
# new_html.write('''</div></body></html>''')
|
||||||
|
# new_html.close() # close the output file
|
||||||
|
|
||||||
|
|
||||||
|
print('''</div></body></html>''')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# # for new_file in tokens_without_stopwords:
|
||||||
|
# appendFile = open('tokenized_words.txt', 'a')
|
||||||
|
# appendFile.write(" " + new_file)
|
||||||
|
# appendFile.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# #shows only stopwords
|
||||||
|
# processed_word_list = []
|
||||||
|
|
||||||
|
# for word in tokenized:
|
||||||
|
# # print(word)
|
||||||
|
# if word not in all_stopwords:
|
||||||
|
# processed_word_list.append('*')
|
||||||
|
# else:
|
||||||
|
# processed_word_list.append(word)
|
||||||
|
# print(processed_word_list)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# # # result putting in a graph
|
||||||
|
# top_words_plot = frequency_word.plot(10)
|
||||||
|
# print(top_words_plot)
|
@ -0,0 +1,178 @@
|
|||||||
|
from __future__ import division
|
||||||
|
import glob
|
||||||
|
from nltk import *
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
import nltk
|
||||||
|
import codecs
|
||||||
|
from nltk import sent_tokenize, word_tokenize, pos_tag
|
||||||
|
from nltk.probability import FreqDist
|
||||||
|
from nltk.corpus import stopwords
|
||||||
|
nltk.download('stopwords')
|
||||||
|
|
||||||
|
|
||||||
|
#open the txt file, read, and tokenize
|
||||||
|
file = open('faceapp.txt','r')
|
||||||
|
text = file.read()
|
||||||
|
|
||||||
|
#stopwords
|
||||||
|
default_stopwords = set(stopwords.words('english'))
|
||||||
|
custom_stopwords = set(codecs.open('stopwords.txt', 'r').read().splitlines())
|
||||||
|
all_stopwords = default_stopwords | custom_stopwords
|
||||||
|
|
||||||
|
print('''<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title></title>
|
||||||
|
<style>
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-40th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-16th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
@font-face {
|
||||||
|
font-family: "Belgika";
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.eot");
|
||||||
|
src: url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.woff") format("woff"),
|
||||||
|
url("http://bohyewoo.com/webfonts/belgika/belgika-8th-webfont.svg#filename") format("svg");
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
font-family: helvetica;
|
||||||
|
font-weight: regular;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
font-size: 20px;
|
||||||
|
line-height: 1.2;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
.NNP {
|
||||||
|
background-color: pink;
|
||||||
|
}
|
||||||
|
|
||||||
|
.VBP {
|
||||||
|
background-color: gold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.NN {
|
||||||
|
background-color: LightSkyBlue;
|
||||||
|
}
|
||||||
|
|
||||||
|
.NNS {
|
||||||
|
background-color: Aquamarine;
|
||||||
|
}
|
||||||
|
|
||||||
|
.paragraph {
|
||||||
|
width: 70%;
|
||||||
|
float: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
.top_words {
|
||||||
|
font-size: 9pt;
|
||||||
|
width: 25%;
|
||||||
|
float: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>''')
|
||||||
|
|
||||||
|
|
||||||
|
# my stopwords are common words I don't want to count, like "a", "an", "the".
|
||||||
|
|
||||||
|
print('<div class ="paragraph">')
|
||||||
|
for sentence in sent_tokenize(text):
|
||||||
|
print('<span>')
|
||||||
|
|
||||||
|
tokenized = word_tokenize(sentence)
|
||||||
|
tagged = pos_tag(tokenized)
|
||||||
|
|
||||||
|
# for HTML
|
||||||
|
for word, pos in tagged:
|
||||||
|
print('<span class="{}">{}</span>'.format(pos, word))
|
||||||
|
|
||||||
|
print('</span>')
|
||||||
|
print('</div>')
|
||||||
|
|
||||||
|
# filtering stopwords
|
||||||
|
tokens_without_stopwords = nltk.FreqDist(words.lower() for words in tokenized if words.lower() not in all_stopwords)
|
||||||
|
print(tokens_without_stopwords)
|
||||||
|
|
||||||
|
# for read_whole_text in tokens_without_stopwords:
|
||||||
|
# whole_text_tokenized =
|
||||||
|
# print(whole_text_tokenized)
|
||||||
|
|
||||||
|
# #filtered words in sentence
|
||||||
|
# filtered_sentence = (" ").join(tokens_without_stopwords)
|
||||||
|
# print(filtered_sentence)
|
||||||
|
|
||||||
|
print('<div class="top_words"> colonial words:')
|
||||||
|
|
||||||
|
frequency_word = FreqDist(tokens_without_stopwords)
|
||||||
|
top_words = tokens_without_stopwords.most_common(10)
|
||||||
|
|
||||||
|
for chosen_words, frequency in top_words:
|
||||||
|
print('<br><span class="chosen_words">{}({}) </span>'.format(chosen_words, frequency))
|
||||||
|
|
||||||
|
|
||||||
|
print('''</div></body></html>''')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# for new_file in tokens_without_stopwords:
|
||||||
|
# appendFile = open('tokenized_words.txt', 'a')
|
||||||
|
# appendFile.write(" " + new_file)
|
||||||
|
# appendFile.close()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# #shows only stopwords
|
||||||
|
# processed_word_list = []
|
||||||
|
|
||||||
|
# for word in tokenized:
|
||||||
|
# # print(word)
|
||||||
|
# if word not in all_stopwords:
|
||||||
|
# processed_word_list.append('*')
|
||||||
|
# else:
|
||||||
|
# processed_word_list.append(word)
|
||||||
|
# print(processed_word_list)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# # # result putting in a graph
|
||||||
|
# top_words_plot = frequency_word.plot(10)
|
||||||
|
# print(top_words_plot)
|
Loading…
Reference in New Issue