Spider Report Examples¶

from datetime import datetime, timedelta
import random
from email_widget import Email, ProgressWidget, TableWidget, StatusWidget
from email_widget.core.enums import TextType, ProgressTheme, StatusType

# 模拟爬虫任务数据
spider_tasks = [
    {
        'name': '电商产品信息采集',
        'target_count': 10000,
        'completed_count': 8500,
        'success_rate': 95.2,
        'avg_speed': 120,  # 条/分钟
        'status': '运行中',
        'start_time': datetime.now() - timedelta(hours=2)
    },
    {
        'name': '新闻资讯爬取',
        'target_count': 5000,
        'completed_count': 5000,
        'success_rate': 98.8,
        'avg_speed': 200,
        'status': '已完成',
        'start_time': datetime.now() - timedelta(hours=1, minutes=30)
    },
    {
        'name': '用户评论数据',
        'target_count': 20000,
        'completed_count': 12000,
        'success_rate': 92.1,
        'avg_speed': 80,
        'status': '运行中',
        'start_time': datetime.now() - timedelta(hours=3)
    }
]

# 创建爬虫监控报告
email = Email("爬虫任务监控报告")

email.add_title("🕷️ 爬虫任务监控报告", TextType.TITLE_LARGE)
email.add_text(f"报告时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")

# 任务概览统计
email.add_title("📊 任务概览", TextType.SECTION_H2)

total_tasks = len(spider_tasks)
running_tasks = sum(1 for task in spider_tasks if task['status'] == '运行中')
completed_tasks = sum(1 for task in spider_tasks if task['status'] == '已完成')
total_collected = sum(task['completed_count'] for task in spider_tasks)

overview_stats = [
    ("任务总数", f"{total_tasks}", "🎯"),
    ("运行中", f"{running_tasks}", "🔄"),
    ("已完成", f"{completed_tasks}", "✅"),
    ("总采集量", f"{total_collected:,}", "📦")
]

for title, value, icon in overview_stats:
    email.add_card(title=title, content=value, icon=icon)

# 各任务详细进度
email.add_title("📈 任务进度详情", TextType.SECTION_H2)

for task in spider_tasks:
    # 计算进度百分比
    progress_percent = (task['completed_count'] / task['target_count']) * 100

    # 根据状态设置主题色
    if task['status'] == '已完成':
        theme = ProgressTheme.SUCCESS
        status_type = StatusType.SUCCESS
    elif task['success_rate'] > 95:
        theme = ProgressTheme.INFO
        status_type = StatusType.SUCCESS
    elif task['success_rate'] > 90:
        theme = ProgressTheme.WARNING
        status_type = StatusType.WARNING
    else:
        theme = ProgressTheme.ERROR
        status_type = StatusType.ERROR

    # 任务状态卡片
    status_widget = StatusWidget()
    status_widget.set_title(task['name']) \
                 .set_status(task['status']) \
                 .set_status_type(status_type) \
                 .set_description(f"成功率: {task['success_rate']:.1f}% | 速度: {task['avg_speed']}条/分钟")
    email.add_widget(status_widget)

    # 进度条
    email.add_progress(
        value=progress_percent,
        label=f"{task['completed_count']:,}/{task['target_count']:,} ({progress_percent:.1f}%)",
        theme=theme
    )

# 详细数据表格
email.add_title("📋 任务详细数据", TextType.SECTION_H2)

table = TableWidget()
table.set_headers(["任务名称", "目标数量", "已完成", "完成率", "成功率", "平均速度", "运行时长"])

for task in spider_tasks:
    runtime = datetime.now() - task['start_time']
    runtime_str = f"{runtime.seconds // 3600}h {(runtime.seconds % 3600) // 60}m"

    progress_percent = (task['completed_count'] / task['target_count']) * 100

    table.add_row([
        task['name'],
        f"{task['target_count']:,}",
        f"{task['completed_count']:,}",
        f"{progress_percent:.1f}%",
        f"{task['success_rate']:.1f}%",
        f"{task['avg_speed']}条/分钟",
        runtime_str
    ])

table.set_striped(True)
email.add_widget(table)

# 性能分析
email.add_title("⚡ 性能分析", TextType.SECTION_H2)

avg_success_rate = sum(task['success_rate'] for task in spider_tasks) / len(spider_tasks)
fastest_task = max(spider_tasks, key=lambda x: x['avg_speed'])
slowest_task = min(spider_tasks, key=lambda x: x['avg_speed'])

performance_text = f"""
**爬虫性能分析：**

📊 **整体表现**
• 平均成功率: {avg_success_rate:.1f}%
• 最快任务: {fastest_task['name']} ({fastest_task['avg_speed']}条/分钟)
• 最慢任务: {slowest_task['name']} ({slowest_task['avg_speed']}条/分钟)

💡 **优化建议**
• 成功率低于90%的任务需要检查反爬策略
• 考虑调整并发数以提高采集速度
• 监控目标网站的响应时间变化
"""

email.add_text(performance_text.strip())

email.export_html("spider_monitor.html")
print("✅ 爬虫监控报告已生成：spider_monitor.html")

Spider Report Examples¶

Spider Task Monitoring¶

Data Collection Progress Tracking¶

爬虫任务监控报告

1. 📊 任务概览

🎯 任务总数

🔄 运行中

✅ 已完成

📦 总采集量

2. 📈 任务进度详情

3. 📋 任务详细数据

4. ⚡ 性能分析

Data Quality Report¶

Collected Data Quality Check¶

数据质量检查报告

1. 📊 数据质量概览

📦 总记录数

✅ 有效记录

🎯 数据质量率

🔄 重复率

2. 📈 质量指标分析

3. 🔍 字段质量详情

4. ⚠️ 数据问题统计

5. 💡 质量改进建议

Exception Monitoring Report¶

Spider Exception and Error Analysis¶

爬虫异常监控报告

1. 📊 异常统计概览

🚨 异常总数

🔍 异常类型

⚠️ 最多异常

⏰ 时间范围

2. 📈 异常类型分布

异常类型分布图

3. 📋 异常详情列表

4. 💡 异常分析与建议

Collection Efficiency Optimization¶

Spider Performance Analysis Report¶

爬虫性能优化分析

1. 📊 性能配置对比

2. 🔍 配置详细分析

3. 💡 性能优化建议

Data Collection Summary¶

Complete Spider Project Report¶

爬虫项目综合报告

1. 📊 项目整体概况

🌐 目标网站

📦 总采集量

✅ 平均成功率

🎯 数据质量率

2. 🏆 关键成果

3. 💡 经验总结

Learning Summary¶

🎯 Professional Skills¶

📊 Report Types¶

💡 Best Practices¶

🚀 Application Value¶