一、实验目的

1. 掌握使用Python 进行批量文件读取与文本预处理的方法。
2. 熟练运用jieba 库进行中文分词、停用词过滤及自定义词典加载。
3. 掌握wordcloud 库的参数配置，生成具有业务指导意义的数据可视化词云。
4. 体验Vibe Coding 工作流，学习如何通过结构化Prompt 引导AI 辅助完成代
码架构设计与实现。

二、实验环境准备

编程语言：Python 3.8 及以上版本
核心依赖库：
 jieba：用于中文文本分词与关键词提取。
 wordcloud：用于生成词云可视化图像。
 matplotlib：用于在本地展示生成的词云图像。
numpy 与pillow：用于处理词云的背景蒙版图像。
环境配置命令：
pip install jieba wordcloud matplotlib numpy pillow

三、可运行代码

# -*- coding: utf-8 -*-
"""
本地文档智能审查与摘要Agent
功能：批量读取文本文件，进行中文分词、词频统计，生成词云和汇总报告
"""
import os
import jieba
import csv
from collections import Counter
from wordcloud import WordCloud
import matplotlib.pyplot as plt

def get_chinese_font_path():
    """
    自动检测系统中的中文字体路径
    支持Windows、macOS、Linux常见字体路径
    优先使用.ttf格式字体，避免.ttc字体集合与Pillow的textbbox方法不兼容的问题
    """
    font_paths = [
        # Windows 字体路径 - 仅使用.ttf格式(TrueType)，wordcloud不支持.ttc字体集合
        'C:/Windows/Fonts/simhei.ttf',    # 黑体 (TrueType)
        'C:/Windows/Fonts/simkai.ttf',    # 楷体 (TrueType)
        'C:/Windows/Fonts/simfang.ttf',   # 仿宋 (TrueType)
        'C:/Windows/Fonts/simli.ttf',     # 隶书 (TrueType)
        'C:/Windows/Fonts/simyou.ttf',    # 幼圆 (TrueType)
        # macOS 字体路径
        '/System/Library/Fonts/Arial Unicode.ttf',
        '/Library/Fonts/Arial Unicode.ttf',
        # Linux 字体路径
        '/usr/share/fonts/truetype/droid/DroidSansFallbackFull.ttf',
        '/usr/share/fonts/TTF/simhei.ttf'
    ]
    for path in font_paths:
        if os.path.exists(path):
            print(f"检测到中文字体: {path}")
            return path
    print("警告: 未找到中文字体，词云可能显示异常")
    return None

def load_stopwords(stopwords_file=None):
    """
    加载停用词表
    如果未提供文件路径，使用默认基础停用词集合
    """
    default_stopwords = {
        "的", "了", "和", "在", "是", "我", "你", "他", "她", "它",
        "我们", "他们", "她们", "它们", "为", "以", "等", "并", "对", "与",
        "而", "或", "但", "却", "就", "都", "也", "很", "更", "最",
        "这", "那", "个", "之", "有", "没有", "可以", "这个", "这样", "那样",
        "中", "上", "下", "后", "前", "里", "从", "到", "被", "把",
        "一个", "一些", "一种", "一项", "一项", "进行", "通过", "使用", "基于", "提供"
    }
    
    if stopwords_file and os.path.exists(stopwords_file):
        try:
            with open(stopwords_file, 'r', encoding='utf-8') as f:
                custom_stopwords = set([line.strip() for line in f if line.strip()])
            # 合并默认停用词和自定义停用词
            all_stopwords = default_stopwords.union(custom_stopwords)
            print(f"成功加载停用词表，共 {len(all_stopwords)} 个停用词")
            return all_stopwords
        except Exception as e:
            print(f"加载停用词表时出错: {e}，使用默认停用词")
    
    print(f"使用默认停用词集合，共 {len(default_stopwords)} 个停用词")
    return default_stopwords

def load_user_dict(user_dict_path):
    """
    加载自定义专业词典，提升分词准确率
    """
    if user_dict_path and os.path.exists(user_dict_path):
        try:
            jieba.load_userdict(user_dict_path)
            print(f"成功加载自定义词典: {user_dict_path}")
            return True
        except Exception as e:
            print(f"加载自定义词典时出错: {e}")
            return False
    return False

def read_text_file(file_path):
    """
    读取文本文件，自动尝试多种编码格式
    """
    encodings = ['utf-8', 'gbk', 'gb2312', 'utf-8-sig']
    for encoding in encodings:
        try:
            with open(file_path, 'r', encoding=encoding) as f:
                return f.read(), None
        except UnicodeDecodeError:
            continue
        except Exception as e:
            return None, str(e)
    return None, "无法识别文件编码"

def analyze_documents(folder_path, stopwords_file=None, user_dict=None, output_folder="./output"):
    """
    核心分析函数：批量读取文件、分词、统计词频、生成词云和报告
    """
    # 创建输出目录
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    
    # 1. 加载自定义词典与停用词
    print("="*50)
    print("步骤1: 加载配置...")
    if user_dict:
        load_user_dict(user_dict)
    stopwords = load_stopwords(stopwords_file)
    font_path = get_chinese_font_path()
    
    # 2. 批量读取文件并分词
    print("\n" + "="*50)
    print("步骤2: 开始批量处理文件...")
    all_words = []
    file_keywords = {}  # 存储每个文件的关键词
    
    # 支持的文件格式
    supported_extensions = ['.txt', '.md', '.text']
    
    if not os.path.exists(folder_path):
        print(f"错误: 文件夹路径不存在 - {folder_path}")
        return None, None
    
    for filename in os.listdir(folder_path):
        # 检查文件扩展名
        ext = os.path.splitext(filename)[1].lower()
        if ext not in supported_extensions:
            continue
            
        file_path = os.path.join(folder_path, filename)
        if not os.path.isfile(file_path):
            continue
            
        try:
            text, error = read_text_file(file_path)
            if error:
                print(f"处理文件 {filename} 时发生错误: {error}")
                continue
            
            if not text or len(text.strip()) == 0:
                print(f"文件 {filename} 内容为空，跳过")
                continue
            
            # 精确模式分词
            words = jieba.lcut(text)
            # 过滤停用词和单字符
            filtered_words = [w for w in words if len(w) > 1 and w not in stopwords]
            
            # 统计当前文件的词频
            file_word_counts = Counter(filtered_words)
            file_top10 = file_word_counts.most_common(10)
            file_keywords[filename] = file_top10
            
            all_words.extend(filtered_words)
            print(f"✓ 成功处理文件: {filename} (提取有效词汇 {len(filtered_words)} 个)")
            
        except Exception as e:
            print(f"处理文件 {filename} 时发生异常: {e}")
    
    if not all_words:
        print("\n错误: 未提取到有效词汇，请检查文件路径或内容。")
        return None, None
    
    # 3. 统计词频
    print("\n" + "="*50)
    print("步骤3: 统计词频...")
    word_counts = Counter(all_words)
    top_50_words = word_counts.most_common(50)
    print(f"共提取有效词汇 {len(all_words)} 个，不重复词汇 {len(word_counts)} 个")
    print(f"\nTop 50 高频词:")
    for i, (word, count) in enumerate(top_50_words, 1):
        print(f"  {i:2d}. {word}: {count} 次")
    
    # 4. 生成词云 - 使用自定义实现避免wordcloud库的transposed_font兼容性问题
    print("\n" + "="*50)
    print("步骤4: 生成词云...")
    if font_path:
        try:
            from PIL import ImageFont, Image, ImageDraw
            import random
            
            width, height = 1000, 800
            # 创建白色背景图片
            img = Image.new('RGB', (width, height), 'white')
            draw = ImageDraw.Draw(img)
            
            # 获取最大频率用于缩放字体大小
            max_freq = max([count for _, count in top_50_words])
            min_font_size = 20
            max_font_size = 120
            
            placed_words = []
            
            for word, freq in top_50_words:
                # 计算字体大小
                font_size = int(min_font_size + (freq / max_freq) * (max_font_size - min_font_size))
                try:
                    font = ImageFont.truetype(font_path, font_size)
                except:
                    font = ImageFont.load_default()
                
                # 获取文本边界框
                bbox = draw.textbbox((0, 0), word, font=font)
                text_w = bbox[2] - bbox[0]
                text_h = bbox[3] - bbox[1]
                
                # 尝试随机放置，避免重叠
                placed = False
                for attempt in range(100):
                    x = random.randint(0, width - text_w - 10)
                    y = random.randint(0, height - text_h - 10)
                    
                    # 检查是否与已放置的词重叠
                    new_rect = (x, y, x + text_w + 5, y + text_h + 5)
                    overlap = False
                    for rect in placed_words:
                        if not (new_rect[2] < rect[0] or new_rect[0] > rect[2] or 
                                new_rect[3] < rect[1] or new_rect[1] > rect[3]):
                            overlap = True
                            break
                    
                    if not overlap:
                        # 随机深色
                        color = (random.randint(0, 150), random.randint(0, 150), random.randint(0, 150))
                        draw.text((x, y), word, font=font, fill=color)
                        placed_words.append(new_rect)
                        placed = True
                        break
            
            output_image = os.path.join(output_folder, "document_wordcloud.png")
            img.save(output_image)
            print(f"词云图片已保存至: {output_image}")
            
        except Exception as e:
            print(f"生成词云时出错: {e}，跳过词云生成")
    else:
        print("跳过词云生成（未找到中文字体）")
    
    # 5. 生成汇总报告
    print("\n" + "="*50)
    print("步骤5: 生成汇总报告...")
    
    # 生成TXT格式报告
    txt_report_path = os.path.join(output_folder, "document_analysis_report.txt")
    with open(txt_report_path, 'w', encoding='utf-8') as f:
        f.write("="*60 + "\n")
        f.write("           本地文档智能审查汇总报告\n")
        f.write("="*60 + "\n\n")
        f.write(f"分析时间: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
        f.write(f"总有效词汇数: {len(all_words)}\n")
        f.write(f"不重复词汇数: {len(word_counts)}\n\n")
        
        f.write("-"*60 + "\n")
        f.write("各文件核心关键词 (Top 10):\n")
        f.write("-"*60 + "\n")
        for filename, keywords in file_keywords.items():
            f.write(f"\n【{filename}】\n")
            for word, count in keywords:
                f.write(f"  - {word}: {count}次\n")
        
        f.write("\n" + "-"*60 + "\n")
        f.write("全局 Top 50 高频词:\n")
        f.write("-"*60 + "\n")
        for i, (word, count) in enumerate(top_50_words, 1):
            f.write(f"{i:2d}. {word}: {count}次\n")
    
    print(f"TXT报告已保存至: {txt_report_path}")
    
    # 生成CSV格式报告
    csv_report_path = os.path.join(output_folder, "document_analysis_report.csv")
    with open(csv_report_path, 'w', encoding='utf-8-sig', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(['排名', '关键词', '出现次数'])
        for i, (word, count) in enumerate(top_50_words, 1):
            writer.writerow([i, word, count])
    
    print(f"CSV报告已保存至: {csv_report_path}")
    
    print("\n" + "="*50)
    print("分析完成！所有结果已保存到 output 文件夹。")
    print("="*50)
    
    return top_50_words, file_keywords

def create_sample_config_files():
    """
    创建示例配置文件（停用词表和自定义词典）
    """
    # 创建示例停用词表
    sample_stopwords = [
        "我们", "你们", "他们", "她们", "它们",
        "这个", "那个", "这些", "那些", "这样", "那样",
        "可以", "通过", "进行", "使用", "基于", "提供",
        "包括", "等等", "相关", "其中", "如下", "如图", "所示"
    ]
    
    stopwords_path = "sample_stopwords.txt"
    if not os.path.exists(stopwords_path):
        with open(stopwords_path, 'w', encoding='utf-8') as f:
            for word in sample_stopwords:
                f.write(word + '\n')
        print(f"已创建示例停用词表: {stopwords_path}")
    
    # 创建示例自定义词典
    sample_dict = [
        "人工智能 3 n",
        "机器学习 3 n",
        "深度学习 3 n",
        "自然语言处理 3 n",
        "词向量 2 n",
        "神经网络 3 n"
    ]
    
    user_dict_path = "sample_user_dict.txt"
    if not os.path.exists(user_dict_path):
        with open(user_dict_path, 'w', encoding='utf-8') as f:
            for line in sample_dict:
                f.write(line + '\n')
        print(f"已创建示例自定义词典: {user_dict_path}")
    
    # 创建示例文档文件夹
    docs_folder = "./docs"
    if not os.path.exists(docs_folder):
        os.makedirs(docs_folder)
        print(f"已创建示例文档文件夹: {docs_folder}")
        print("请将需要分析的 .txt 或 .md 文件放入该文件夹中")

if __name__ == "__main__":
    # 主程序入口
    print("本地文档智能审查与摘要Agent")
    print("="*50)
    
    # 创建示例配置文件
    create_sample_config_files()
    
    # 配置参数（可根据实际情况修改）
    DOCUMENTS_FOLDER = "./docs"          # 待分析文档所在文件夹
    STOPWORDS_FILE = "sample_stopwords.txt"  # 停用词表路径
    USER_DICT_FILE = "sample_user_dict.txt"  # 自定义词典路径
    
    # 执行分析
    top_words, file_keywords = analyze_documents(
        folder_path=DOCUMENTS_FOLDER,
        stopwords_file=STOPWORDS_FILE,
        user_dict=USER_DICT_FILE,
        output_folder="./output"
    )

四、实验结果

词云图片

TXT格式的分析报告

CSV格式的分析报告

五、心得体会

通过本次本地文档智能审查与摘要实验，我深入掌握了Python批量文件处理、jieba中文分词、停用词过滤以及词云可视化等核心技术，亲身体验了从原始文档输入到智能分析报告输出的完整Agent自动化工作流，深刻理解了将传统文本处理技术与智能体开发思维相结合的实践价值，为后续开发更复杂的文档智能分析系统打下了坚实基础。