Python爬虫实战：手把手教你抓取笔趣阁小说
0
0
0
还在为小说资源找不到、无法阅读而烦恼吗？本文教你使用 Python 动手抓取笔趣阁小说，提供完整源码，代码简单清晰，无需复杂配置，复制即可运行。
1
爬取小说标题
将 BOOK_URL 替换为目标小说页面的 URL 地址，用于爬取小说标题。
以 "斗破苍穹 " 为例，打开该小说页面后：
- 按 F12 打开浏览器开发者工具；
- 在页面中定位到标题元素（如 <h1> 标签）；
- 编写爬虫代码，提取标题内容并在控制台输出结果。
import requests
from bs4 import BeautifulSoup
BOOK_URL = "https://www.xinbiqug.com/book/481604/"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"
}
# 1. 请求页面
r = requests.get(BOOK_URL, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
# 2. 解析标题
soup = BeautifulSoup(r.text, "html.parser")
title = soup.select_one("h1.title") or soup.select_one("h1")
print(f"小说标题: {title.get_text(strip=True)}")
2
爬取章节目录列表
12
继续使用开发者工具在页面找我们需要的元素，在页面找到a标签的内容，我使用代码直接爬取chapterList下的所有a标签，并把a标签代表的链接一起获取。
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup
BOOK_URL = "https://www.xinbiqug.com/book/481604/"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"
}
# 1. 请求页面
r = requests.get(BOOK_URL, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
# 2. 解析标题
soup = BeautifulSoup(r.text, "html.parser")
title = soup.select_one("h1.title") or soup.select_one("h1")
print(f"小说标题: {title.get_text(strip=True)}")
# 获取章节目录列表
chapters = []
for a in soup.select("dl.chapterlist a"):
href = a.get("href")
if href:
chapters.append({
"title": a.get_text(strip=True),
"url": urljoin(BOOK_URL, href) # 自动拼接相对路径
})
print(f"共找到 {len(chapters)} 个章节")
for i, ch in enumerate(chapters[:3], 1):
print(f" {i}. {ch['title']} -> {ch['url']}")
3
爬取正文内容
根据获取到链接列表，访问对应章节页面，提取正文内容。可以在控制台看到文章内容。
import re
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup
BOOK_URL = "https://www.xinbiqug.com/book/481604/"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"
}
# 1. 请求页面
r = requests.get(BOOK_URL, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
# 2. 解析标题
soup = BeautifulSoup(r.text, "html.parser")
title = soup.select_one("h1.title") or soup.select_one("h1")
print(f"小说标题: {title.get_text(strip=True)}")
# 获取章节目录列表
chapters = []
for a in soup.select("dl.chapterlist a"):
href = a.get("href")
if href:
chapters.append({
"title": a.get_text(strip=True),
"url": urljoin(BOOK_URL, href) # 自动拼接相对路径
})
print(f"共找到 {len(chapters)} 个章节")
for i, ch in enumerate(chapters[:3], 1):
print(f" {i}. {ch['title']} -> {ch['url']}")
def clean(text):
"""清洗常见笔趣阁广告水印"""
text = re.sub(r'本站.*?阅读|最新网址.*?com|笔趣阁.*?更新', '', text, flags=re.I)
return re.sub(r'\n{3,}', '\n\n', text).strip()
def download_chapter(url):
r = requests.get(url, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
soup = BeautifulSoup(r.text, "html.parser")
content_div = soup.select_one("#content")
if not content_div:
return None
# 移除script/div等干扰标签
for tag in content_div.find_all(["script", "div"]):
tag.decompose()
raw_text = content_div.get_text(separator="\n", strip=True)
return clean(raw_text)
# 测试：下载第一个章节
if chapters:
text = download_chapter(chapters[0]["url"])
print(f"\n [{chapters[0]['title']}]")
print(text if text else "获取失败")
4
创建输出目录
以书名为文件夹名，在本地创建保存目录。
import os
import re
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup
BOOK_URL = "https://www.xinbiqug.com/book/481604/"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"
}
OUTPUT_DIR = "novel_output"
# 1. 请求页面
r = requests.get(BOOK_URL, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
# 2. 解析标题
soup = BeautifulSoup(r.text, "html.parser")
title = soup.select_one("h1.title") or soup.select_one("h1")
print(f"小说标题: {title.get_text(strip=True)}")
# 创建目录
def safe_name(name):
"""替换文件名非法字符"""
return re.sub(r'[<>:"/\\|?*]', '_', name)
book_dir = os.path.join(OUTPUT_DIR, safe_name(title.get_text(strip=True)))
os.makedirs(book_dir, exist_ok=True)
print(f"输出目录已就绪: {book_dir}")
# 获取章节目录列表
chapters = []
for a in soup.select("dl.chapterlist a"):
href = a.get("href")
if href:
chapters.append({
"title": a.get_text(strip=True),
"url": urljoin(BOOK_URL, href) # 自动拼接相对路径
})
print(f"共找到 {len(chapters)} 个章节")
for i, ch in enumerate(chapters[:3], 1):
print(f" {i}. {ch['title']} -> {ch['url']}")
def clean(text):
"""清洗常见笔趣阁广告水印"""
text = re.sub(r'本站.*?阅读|最新网址.*?com|笔趣阁.*?更新', '', text, flags=re.I)
return re.sub(r'\n{3,}', '\n\n', text).strip()
def download_chapter(url):
r = requests.get(url, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
soup = BeautifulSoup(r.text, "html.parser")
content_div = soup.select_one("#content")
if not content_div:
return None
# 移除script/div等干扰标签
for tag in content_div.find_all(["script", "div"]):
tag.decompose()
raw_text = content_div.get_text(separator="\n", strip=True)
return clean(raw_text)
# 测试：下载第一个章节
if chapters:
text = download_chapter(chapters[0]["url"])
print(f"\n [{chapters[0]['title']}]")
5
写入txt文件
import os
import re
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup
from tqdm import tqdm
BOOK_URL = "https://www.xinbiqug.com/book/481604/"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"
}
OUTPUT_DIR = "novel_output"
# 1. 请求页面
r = requests.get(BOOK_URL, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
# 2. 解析标题
soup = BeautifulSoup(r.text, "html.parser")
title = soup.select_one("h1.title") or soup.select_one("h1")
print(f"小说标题: {title.get_text(strip=True)}")
# 创建目录
def safe_name(name):
"""替换文件名非法字符"""
return re.sub(r'[<>:"/\\|?*]', '_', name)
book_dir = os.path.join(OUTPUT_DIR, safe_name(title.get_text(strip=True)))
os.makedirs(book_dir, exist_ok=True)
print(f"输出目录已就绪: {book_dir}")
# 获取章节目录列表
chapters = []
for a in soup.select("dl.chapterlist a"):
href = a.get("href")
if href:
chapters.append({
"title": a.get_text(strip=True),
"url": urljoin(BOOK_URL, href) # 自动拼接相对路径
})
print(f"共找到 {len(chapters)} 个章节")
for i, ch in enumerate(chapters[:3], 1):
print(f" {i}. {ch['title']} -> {ch['url']}")
def clean(text):
"""清洗常见笔趣阁广告水印"""
text = re.sub(r'本站.*?阅读|最新网址.*?com|笔趣阁.*?更新', '', text, flags=re.I)
return re.sub(r'\n{3,}', '\n\n', text).strip()
def download_chapter(url):
r = requests.get(url, headers=HEADERS, timeout=15)
r.encoding = r.apparent_encoding or "utf-8"
soup = BeautifulSoup(r.text, "html.parser")
content_div = soup.select_one("#content")
if not content_div:
return None
# 移除script/div等干扰标签
for tag in content_div.find_all(["script", "div"]):
tag.decompose()
raw_text = content_div.get_text(separator="\n", strip=True)
return clean(raw_text)
existing_files = set(os.listdir(book_dir))
total = len(chapters)
downloaded = 0
skipped = 0
for i, ch in enumerate(tqdm(chapters, desc="下载进度"), 1):
filename = f"{i:04d}_{safe_name(ch['title'])}.txt"
filepath = os.path.join(book_dir, filename)
# 断点续传：已存在则跳过
if filename in existing_files:
skipped += 1
continue
content = download_chapter(ch["url"])
if content:
with open(filepath, "w", encoding="utf-8") as f:
f.write(f"《{ch['title']}》\n\n{content}")
downloaded += 1
else:
print(f"\n第{i}章下载失败: {ch['title']}")
print(f"\n 完成! 新下载:{downloaded} | 跳过:{skipped} | 总计:{total}")
0
0
0
qq空间 
微博 
复制链接 
分享 
本文未经作者允许授权，禁止转载 更多相关项目
猜你喜欢
评论/提问(已发布 0 条)
评论 
收藏 
分享 
下载 
举报 
回顶部