wehub-resource-sync
53 lines
1.9 KiB
Python
53 lines
1.9 KiB
Python
|
|
""" This example demonstrates how to parse various web sources into a Library through HTML scraping.
|
|
|
|
When parsing websites, please follow best practices, ethical guidelines and common sense - as a good example,
|
|
see https://monashdatafluency.github.io/python-web-scraping/section-5-legal-and-ethical-considerations/
|
|
|
|
To use the WebSite Parser requires several additional python libraries to be installed:
|
|
|
|
pip3 install beautifulsoup4
|
|
pip3 install lxml
|
|
pip3 install requests
|
|
pip3 install urllib3
|
|
|
|
"""
|
|
|
|
from llmware.parsers import Parser, WebSiteParser
|
|
|
|
|
|
def parsing_web_sources_in_memory():
|
|
|
|
""" In this example. we will access the WebSiteParser through the general Parser class, with the main use
|
|
case of integrating a small HTML site into a library with inclusion of other file types.
|
|
|
|
We recommend checking the website output first in memory, before automatically adding to a DB - as
|
|
usually the extracted text will require some post-processing to remove redundancies, potential formatting or JS -
|
|
and as a general safety check on the content. """
|
|
|
|
print(f"\nExample - Parsing Web Sources")
|
|
|
|
# parse website directly - here are a few ideas for rapid testing
|
|
# please be respectful in keeping requests at low volume
|
|
|
|
# high volume global website
|
|
website = "https://www.cnbc.com"
|
|
|
|
# come visit NYC
|
|
website = "https://www.ny.com/general/centers.html"
|
|
# website = "https://bronxzoo.com"
|
|
# website = "https://en.wikipedia.org/wiki/Jalen_Brunson"
|
|
|
|
website_parsed_output = Parser().parse_website(website, write_to_db=False, save_history=False, get_links=False)
|
|
|
|
# look at the first 10 text blocks extracted
|
|
for x in range(0,min(10, len(website_parsed_output))):
|
|
print("text blocks extracted: ", website_parsed_output[x])
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
parsing_web_sources_in_memory()
|