2022-04-09 23:06:53 +01:00
|
|
|
"""Script to scrape article text from http://sxwjw.shaanxi.gov.cn.
|
|
|
|
|
|
|
|
|
|
Article contents are in a div with the class `message-box`.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
from utils.linkutils import read_links
|
|
|
|
|
from utils.scrapeutils import download_link_texts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
"""Collect and output article text."""
|
|
|
|
|
with open("articles-shanxi/links.csv", "r") as f:
|
|
|
|
|
links = read_links(f)
|
|
|
|
|
|
2022-04-09 23:48:00 +01:00
|
|
|
# Broken links
|
|
|
|
|
links = links[:254]
|
|
|
|
|
|
2022-04-09 23:32:08 +01:00
|
|
|
download_link_texts(links, "message-box", "articles-shanxi", encoding="UTF-8")
|
2022-04-09 23:06:53 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|
