You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
196 lines
6.2 KiB
Python
196 lines
6.2 KiB
Python
import html5lib
|
|
from xml.etree import ElementTree as ET
|
|
from PIL import Image
|
|
from nltk import FreqDist
|
|
from nltk.corpus import stopwords
|
|
import random
|
|
import glob
|
|
import time
|
|
from fpdf import FPDF
|
|
import os
|
|
import shutil
|
|
|
|
path1 = './temp'
|
|
if not os.path.isdir(path1):
|
|
os.makedirs(path1)
|
|
os.makedirs('./temp/crops4')
|
|
os.makedirs('./temp/crops7')
|
|
os.makedirs('./temp/crops_more')
|
|
|
|
stopwords.words('english')
|
|
sr = set(stopwords.words('english'))
|
|
|
|
def cleanstopwords(list):
|
|
"This cleans stopwords from a list of words"
|
|
clean_words = list[:]
|
|
for word in list:
|
|
if word.lower() in sr:
|
|
clean_words.remove(word)
|
|
return clean_words
|
|
|
|
def findmostcommon(list, int):
|
|
"This finds the most common words and returns a list"
|
|
fdist = FreqDist(word.lower() for word in list)
|
|
mostcommon = fdist.most_common(int)
|
|
mostcommon_list = [i[0] for i in mostcommon]
|
|
return mostcommon_list
|
|
|
|
def findleastcommon(list):
|
|
"This finds the least common words and returns a list"
|
|
fdist = FreqDist(word.lower() for word in list)
|
|
leastcommon = fdist.most_common()
|
|
for i in leastcommon:
|
|
if (i[1] <= 1):
|
|
leastcommon_list.append(i[0])
|
|
return leastcommon_list
|
|
|
|
def coordinates(attribute):
|
|
"This extracts the box coordinates of words from an hocr / html element tree"
|
|
c = attribute # 'title' is the word in the html tag
|
|
c, r = c.split(";") # split the attribute into two sections
|
|
c = c.split(" ")[1:] # split again and discard the elements which aren't useful
|
|
c = [int(x) for x in c] # put coordinates into list as integers
|
|
return c
|
|
|
|
def filternone(word_raw):
|
|
if word_raw is None:
|
|
remove = None
|
|
word = 'y'
|
|
else:
|
|
word = element.text.strip(',".!:;()')
|
|
return word
|
|
|
|
x = -1
|
|
leastcommon_list = []
|
|
allwords = []
|
|
scanimg = sorted(glob.glob('images-tiff/*.tiff'))
|
|
hocr = sorted(glob.glob('hocr/*.html'))
|
|
num = 0
|
|
maximum = 20 / len(scanimg) # this helps the script remove words in a way that is proportional to number of pages scanned
|
|
|
|
# loop through every image in scanimg folder
|
|
for i in scanimg:
|
|
x = x + 1
|
|
limit = 15 - (x * maximum) # this helps the script remove words in a way that is proportional to number of pages scanned
|
|
iim = Image.open(i) # iim is initial image
|
|
oim = Image.new("RGB", iim.size, (255, 255, 255)) #oim is output image
|
|
|
|
# open corresponding hocr file
|
|
f = open(hocr[x])
|
|
print ('Reading scanned image and hocr file, filtering least common words.')
|
|
print ('')
|
|
|
|
t = html5lib.parse(f, namespaceHTMLElements=False)
|
|
|
|
# loop through every word in hocr file to analyse words and find least common
|
|
for element in t.findall(".//span[@class='ocrx_word']"):
|
|
word = filternone(element.text)
|
|
allwords.append(word)
|
|
|
|
clean_words = cleanstopwords(allwords) #clean stopwords
|
|
findleastcommon(clean_words) #find least common words and add them to list
|
|
mostcommon_list = findmostcommon(clean_words, 30) #find most common words and add them to list
|
|
|
|
print ('The most common words until text', x+1, 'are:', mostcommon_list)
|
|
print ('The least common words until text', x+1, 'are:', leastcommon_list)
|
|
|
|
print ('')
|
|
|
|
# loop through every word in hocr file to extract coordinates, then remove or paste into output image
|
|
|
|
print ('Processing word coordinates and replacing least common words with most common words.')
|
|
print ('')
|
|
|
|
for element in t.findall(".//span[@class='ocrx_word']"):
|
|
word = filternone(element.text)
|
|
c = coordinates(element.attrib['title'])
|
|
num = num + 1
|
|
|
|
wim = iim.crop(c) # wim is word image
|
|
|
|
#extract coordinates
|
|
if word.lower() in mostcommon_list and len(word) > 1 and len(word) <= 5:
|
|
wim.save ("temp/crops4/wimreplace{}.png".format(num))
|
|
elif word in mostcommon_list and len(word) <= 7 :
|
|
wim.save ("temp/crops7/wimreplace{}.png".format(num))
|
|
elif word in mostcommon_list and len(word) > 7 :
|
|
wim.save ("temp/crops_more/wimreplace{}.png".format(num))
|
|
|
|
if x > 0:
|
|
# use PIL to crop out every box, then paste it according to if rule
|
|
randomimg4 = random.choice(glob.glob('temp/crops4/*.png'))
|
|
randomimg7 = random.choice(glob.glob('temp/crops7/*.png'))
|
|
randomimg_more = random.choice(glob.glob('temp/crops_more/*.png'))
|
|
|
|
wimreplace4 = Image.open(randomimg4)
|
|
wimreplace7 = Image.open(randomimg7)
|
|
wimreplace_more = Image.open(randomimg_more)
|
|
|
|
wimcolor4 = Image.new('RGBA', wimreplace4.size, (250, 230, 0, 90))
|
|
wimcolor7 = Image.new('RGBA', wimreplace7.size, (250, 230, 0, 90))
|
|
wimcolor_more = Image.new('RGBA', wimreplace_more.size, (250, 230, 0, 90))
|
|
|
|
out4 = Image.alpha_composite(wimreplace4.convert('RGBA'), wimcolor4)
|
|
out7 = Image.alpha_composite(wimreplace7.convert('RGBA'), wimcolor7)
|
|
out_more = Image.alpha_composite(wimreplace_more.convert('RGBA'), wimcolor_more)
|
|
|
|
if word.lower() in leastcommon_list and len(word) <= 3:
|
|
oim.paste(wim, (c[0], c[1], c[2], c[3]))
|
|
|
|
elif word.lower() in leastcommon_list and len(word) < 8:
|
|
oim.paste(out4, (c[0], c[1]))
|
|
|
|
elif word.lower() in leastcommon_list and len(word) < 11:
|
|
oim.paste(out7, (c[0], c[1]))
|
|
|
|
elif word.lower() in leastcommon_list and len(word) > 8:
|
|
oim.paste(out_more, (c[0], c[1]))
|
|
|
|
else:
|
|
oim.paste(wim, (c[0], c[1], c[2], c[3]))
|
|
|
|
else:
|
|
oim.paste(wim, (c[0], c[1], c[2], c[3]))
|
|
|
|
|
|
#-------------------------------------------------------------------------------#
|
|
# save images
|
|
n = i.replace("images-tiff/","output/erase-replace/").replace(".tiff", "")
|
|
oim.save("{}-{}replace.jpg".format(n, x))
|
|
|
|
#-------------------------------------------------------------------------------#
|
|
# save images into PDF
|
|
outputs = glob.glob('output/erase-replace/*replace.jpg')
|
|
print ('')
|
|
print ("Saving to PDF: output/erase-replace/Replace.pdf")
|
|
|
|
def makePdf(pdfFileName, listPages, dir = ''):
|
|
if (dir):
|
|
dir += "/"
|
|
cover = Image.open(dir + str(listPages[0]))
|
|
width, height = cover.size
|
|
pdf = FPDF(unit = "pt", format = [width, height])
|
|
for page in listPages:
|
|
pdf.add_page()
|
|
pdf.image(dir + str(page), 0, 0)
|
|
pdf.output(dir + pdfFileName + ".pdf", "F")
|
|
|
|
makePdf('output/erase-replace/Replace', outputs, dir = '')
|
|
|
|
#clean up previous jpg and png files
|
|
files = glob.glob('./output/erase-replace/*replace.jpg')
|
|
for f in files:
|
|
os.remove(f)
|
|
|
|
shutil.rmtree('./temp/')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
|
|
# TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
|
|
|
# 0. You just DO WHAT THE FUCK YOU WANT TO.
|