weibo-crawler
weibo-crawler copied to clipboard
用了user_id_list.txt下载三个博主,不知道为啥,第一个博主只下载了200个博文,然后下载照片时候经常出现debug
{ "user_id_list": "user_id_list.txt", "only_crawl_original": 0, "remove_html_tag": 1, "since_date": "2018-01-01", "start_page": 1, "page_weibo_count": 10, "write_mode": ["csv"], "original_pic_download": 1, "retweet_pic_download": 1, "original_video_download": 1, "retweet_video_download": 1, "original_live_photo_download": 1, "retweet_live_photo_download": 1, "download_comment":0, "comment_max_download_count":1000, "download_repost": 0, "repost_max_download_count": 1000, "user_id_as_folder_name": 0, "cookie": "your cookie", "mysql_config": { "host": "localhost", "port": 3306, "user": "root", "password": "123456", "charset": "utf8mb4" }, "store_binary_in_sqlite": 0, "mongodb_URI": "mongodb://[username:password@]host[:port][/[defaultauthdb][?options]]", "post_config": { "api_url": "https://api.example.com", "api_token": "" } }