该代码基本上可下载绝大部分小说网站的小说，有问题也只需要调整小部分的代码。
1、修改一下url变量，改为小说目录列表页面。
2、修改第34行“for i in range(20, len(tag_h3)):”20这个值，跟踪进去可调整从第几章开始下载。
3、第36行也得改成网站首地址。
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
# -*- coding: utf-8 -*- """ Spyder Editor This is a temporary script file. """ from bs4 import BeautifulSoup import requests import time import os import threading headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'} saveDir = 'd:/鸿天神尊.txt' url = 'https://www.tycqzw.com/62_62436/' # 获得所有章节的URL def get_topic(url): headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'} response = requests.get(url, headers=headers) response.encoding = response.apparent_encoding soup = BeautifulSoup(response.content, 'html.parser') tag_h3 = soup.find_all('dd') f = open(saveDir, mode='a', encoding='utf8') try: for i in range(520, len(tag_h3)): topic_url = tag_h3[i].find('a').get('href') # 获得主题链接 topic_url = 'https://www.tycqzw.com/'+topic_url # 主题页面 url txt = tag_h3[i].find('a').get_text() # 主题名称 print(topic_url +' ' + txt) f.write(get_txt(topic_url, txt)) finally: f.close() # 下载链接页面中的文本内容 def get_txt(url, txt): response = requests.get(url, headers=headers) response.encoding = response.apparent_encoding soup = BeautifulSoup(response.content, 'html.parser') tag_txt = soup.find('div', id='content').get_text() #(separator='\n') i = tag_txt.find('(本章完)') return txt+'\n'+tag_txt[:i] if __name__ == '__main__': print('===== 主程序开始 =====') start = time.perf_counter() get_topic(url) tt = time.perf_counter() - start print('===== 主程序结束，用时 {:.2f}s ====='.format(tt)) |
