You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
55 lines
1.3 KiB
Python
55 lines
1.3 KiB
Python
from urllib.request import urlopen
|
|
import markdown, html5lib
|
|
|
|
|
|
sample_text = """
|
|
# https://hub.xpub.nl/bootleglibrary/book/374
|
|
This is an annotation of Mrs. Gersande's Binding Index
|
|
|
|
|
|
# https://hub.xpub.nl/bootleglibrary/book/348
|
|
Telegraph Telephone Teletype
|
|
|
|
More TEXT HERERERE!!!!!!!!!!!!!!
|
|
|
|
# unrelated
|
|
Stuff now
|
|
"""
|
|
|
|
pad_url = "https://pad.xpub.nl/p/boring_old_tomato_sandwiches"
|
|
pad_text_url = pad_url + "/export/txt"
|
|
f = urlopen(pad_text_url)
|
|
|
|
pad_text = f.read().decode('utf-8')
|
|
pad_text = sample_text
|
|
print (pad_text)
|
|
print ()
|
|
|
|
# print (pad_text)
|
|
# Turn pad text into html text
|
|
html = markdown.markdown(pad_text)
|
|
print (html)
|
|
print ()
|
|
|
|
# Turn html text in an elementtree
|
|
t = html5lib.parseFragment(html, namespaceHTMLElements=False)
|
|
print (t)
|
|
|
|
# create a "database" of paragraphs associated with each URL given in an H1
|
|
paragraphs_by_header = {}
|
|
current_header = None
|
|
|
|
for elt in t:
|
|
if elt.tag == "h1" and elt.text is not None:
|
|
print (("HEADER"), elt.text)
|
|
current_header = elt.text.strip()
|
|
elif elt.tag == "p":
|
|
if current_header:
|
|
if current_header not in paragraphs_by_header:
|
|
paragraphs_by_header[current_header] = []
|
|
paragraphs_by_header[current_header].append(elt)
|
|
|
|
from pprint import pprint
|
|
pprint(paragraphs_by_header)
|
|
|