You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
50 lines
1.4 KiB
Python
50 lines
1.4 KiB
Python
from urllib.request import urlopen
|
|
import markdown, html5lib
|
|
|
|
|
|
def parse_pad (pad_text):
|
|
print ("parse", pad_text)
|
|
html = markdown.markdown(pad_text)
|
|
t = html5lib.parseFragment(html, namespaceHTMLElements=False)
|
|
# create a "database" of paragraphs associated with each URL given in an H1
|
|
paragraphs_by_header = {}
|
|
current_header = None
|
|
for elt in t:
|
|
if elt.tag == "h1" and elt.text is not None:
|
|
# print (("HEADER"), elt.text)
|
|
current_header = elt.text.strip()
|
|
elif elt.tag == "p":
|
|
if current_header:
|
|
if current_header not in paragraphs_by_header:
|
|
paragraphs_by_header[current_header] = []
|
|
paragraphs_by_header[current_header].append(elt)
|
|
return paragraphs_by_header
|
|
|
|
def curl (url):
|
|
return urlopen(url).read().decode('utf-8')
|
|
|
|
|
|
if __name__== "__main__":
|
|
# THIS CODE ONLY HAPPENS WHEN YOU RUN THE SCRIPT DIRECTLY
|
|
pad_url = "https://pad.xpub.nl/p/boring_old_tomato_sandwiches"
|
|
pad_text_url = pad_url + "/export/txt"
|
|
pad_text = curl(pad_text_url)
|
|
|
|
sample_text = """
|
|
# https://hub.xpub.nl/bootleglibrary/book/374
|
|
This is an annotation of Mrs. Gersande's Binding Index
|
|
|
|
# https://hub.xpub.nl/bootleglibrary/book/348
|
|
Telegraph Telephone Teletype
|
|
|
|
More TEXT HERERERE!!!!!!!!!!!!!!
|
|
|
|
# unrelated
|
|
Stuff now
|
|
"""
|
|
|
|
paragraphs_by_header = parse_pad(pad_text)
|
|
from pprint import pprint
|
|
pprint(paragraphs_by_header)
|
|
|