"""Script to scrape article text from http://sxwjw.shaanxi.gov.cn.

Article contents are in a div with the class `message-box`.
"""

from utils.linkutils import read_links
from utils.scrapeutils import download_link_texts


def main():
    """Collect and output article text."""
    with open("articles-shanxi/links.csv", "r") as f:
        links = read_links(f)

    # Broken links
    links = links[:254]

    download_link_texts(links, "message-box", "articles-shanxi", encoding="UTF-8")


if __name__ == "__main__":
    main()