一、个人简介

💖💖作者：计算机编程果茶熊
💙💙个人简介：曾长期从事计算机专业培训教学，担任过编程老师，同时本人也热爱上课教学，擅长Java、微信小程序、Python、Golang、安卓Android等多个IT方向。会做一些项目定制化开发、代码讲解、答辩教学、文档编写、也懂一些降重方面的技巧。平常喜欢分享一些自己开发中遇到的问题的解决办法，也喜欢交流技术，大家有技术代码这一块的问题可以问我！
💛💛想说的话：感谢大家的关注与支持！
💜💜
网站实战项目
安卓/小程序实战项目
大数据实战项目
计算机毕业设计选题
💕💕文末获取源码联系计算机编程果茶熊

二、系统介绍

大数据框架：Hadoop+Spark（Hive需要定制修改）
开发语言：Java+Python（两个版本都支持）
数据库：MySQL
后端框架：SpringBoot(Spring+SpringMVC+Mybatis)+Django（两个版本都支持）
前端：Vue+Echarts+HTML+CSS+JavaScript+jQuery

《大模型岗位数据分析与可视化系统》是一个基于大数据技术构建的综合性数据分析平台，专门针对人工智能和大模型相关岗位市场进行深度数据挖掘与可视化展示。系统采用Hadoop+Spark大数据框架作为核心计算引擎，通过HDFS实现海量岗位数据的分布式存储，利用Spark SQL进行高效的数据处理与分析。后端基于Django框架构建RESTful API服务，前端使用Vue+ElementUI+Echarts技术栈打造现代化的用户交互界面。系统核心功能涵盖用户权限管理、模型岗位数据的全生命周期管理、企业招聘偏好智能分析、市场行情趋势预测、岗位薪酬结构分析、热门技能需求统计以及实时数据可视化大屏展示。通过Pandas和NumPy进行数据预处理与特征工程，结合MySQL数据库实现结构化数据的持久化存储，为求职者、企业HR以及行业研究人员提供全方位的大模型岗位市场洞察服务。

三、视频解说

四、部分功能展示

五、部分代码展示

在这里插入代码片
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, count, avg, max, min, desc, when, regexp_extract
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
import pandas as pd
import numpy as np
import json
from datetime import datetime, timedelta

spark = SparkSession.builder.appName("AIJobAnalysisSystem").config("spark.sql.adaptive.enabled", "true").config("spark.sql.adaptive.coalescePartitions.enabled", "true").getOrCreate()

@csrf_exempt
def enterprise_preference_analysis(request):
    job_df = spark.read.format("jdbc").option("url", "jdbc:mysql://localhost:3306/ai_job_db").option("dbtable", "job_positions").option("user", "root").option("password", "password").load()
    company_df = spark.read.format("jdbc").option("url", "jdbc:mysql://localhost:3306/ai_job_db").option("dbtable", "companies").option("user", "root").option("password", "password").load()
    joined_df = job_df.join(company_df, job_df.company_id == company_df.id, "inner")
    skill_preference = joined_df.groupBy("company_name", "company_type").agg(count("*").alias("job_count"), avg("salary_min").alias("avg_salary_min"), avg("salary_max").alias("avg_salary_max"))
    education_preference = joined_df.groupBy("company_name", "education_requirement").agg(count("*").alias("count")).orderBy(desc("count"))
    experience_preference = joined_df.groupBy("company_name", "experience_requirement").agg(count("*").alias("count")).orderBy(desc("count"))
    location_preference = joined_df.groupBy("company_name", "job_location").agg(count("*").alias("count")).orderBy(desc("count"))
    skill_keywords = joined_df.select("company_name", regexp_extract("job_description", r"(Python|机器学习|深度学习|NLP|CV|大模型|Transformer|PyTorch|TensorFlow)", 1).alias("skill")).filter(col("skill") != "")
    skill_stats = skill_keywords.groupBy("company_name", "skill").agg(count("*").alias("mention_count")).orderBy(desc("mention_count"))
    company_scale_analysis = joined_df.groupBy("company_scale").agg(count("*").alias("position_count"), avg("salary_min").alias("avg_min_salary"), avg("salary_max").alias("avg_max_salary"))
    industry_trend = joined_df.groupBy("industry_type").agg(count("*").alias("job_count"), avg(col("salary_min") + col("salary_max")).alias("avg_salary")).orderBy(desc("job_count"))
    preference_data = {"skill_preference": skill_preference.toPandas().to_dict('records'), "education_preference": education_preference.toPandas().to_dict('records'), "experience_preference": experience_preference.toPandas().to_dict('records'), "location_preference": location_preference.toPandas().to_dict('records'), "skill_stats": skill_stats.toPandas().to_dict('records'), "company_scale_analysis": company_scale_analysis.toPandas().to_dict('records'), "industry_trend": industry_trend.toPandas().to_dict('records')}
    return JsonResponse(preference_data, safe=False)

@csrf_exempt
def market_trend_analysis(request):
    job_df = spark.read.format("jdbc").option("url", "jdbc:mysql://localhost:3306/ai_job_db").option("dbtable", "job_positions").option("user", "root").option("password", "password").load()
    monthly_trend = job_df.groupBy("publish_month").agg(count("*").alias("job_count"), avg("salary_min").alias("avg_min_salary"), avg("salary_max").alias("avg_max_salary")).orderBy("publish_month")
    city_distribution = job_df.groupBy("job_location").agg(count("*").alias("job_count"), avg(col("salary_min") + col("salary_max")).alias("avg_salary")).orderBy(desc("job_count"))
    position_level_trend = job_df.groupBy("position_level", "publish_month").agg(count("*").alias("count")).orderBy("publish_month", "position_level")
    growth_rate_df = monthly_trend.toPandas()
    growth_rate_df['growth_rate'] = growth_rate_df['job_count'].pct_change() * 100
    growth_rate_df['salary_growth_rate'] = growth_rate_df['avg_max_salary'].pct_change() * 100
    hot_companies = job_df.groupBy("company_name").agg(count("*").alias("job_count")).orderBy(desc("job_count")).limit(20)
    urgent_positions = job_df.filter(col("urgency_level") == "紧急").groupBy("position_name").agg(count("*").alias("urgent_count")).orderBy(desc("urgent_count"))
    weekend_trend = job_df.groupBy("work_type").agg(count("*").alias("count"), avg("salary_max").alias("avg_salary")).orderBy(desc("count"))
    competition_index = job_df.groupBy("position_name").agg(count("*").alias("supply"), avg("applicant_count").alias("demand")).withColumn("competition_ratio", col("demand") / col("supply"))
    market_activity = job_df.groupBy("publish_date").agg(count("*").alias("daily_posts")).orderBy("publish_date")
    trend_data = {"monthly_trend": monthly_trend.toPandas().to_dict('records'), "city_distribution": city_distribution.toPandas().to_dict('records'), "position_level_trend": position_level_trend.toPandas().to_dict('records'), "growth_rate": growth_rate_df.to_dict('records'), "hot_companies": hot_companies.toPandas().to_dict('records'), "urgent_positions": urgent_positions.toPandas().to_dict('records'), "weekend_trend": weekend_trend.toPandas().to_dict('records'), "competition_index": competition_index.toPandas().to_dict('records'), "market_activity": market_activity.toPandas().to_dict('records')}
    return JsonResponse(trend_data, safe=False)

@csrf_exempt
def salary_analysis(request):
    job_df = spark.read.format("jdbc").option("url", "jdbc:mysql://localhost:3306/ai_job_db").option("dbtable", "job_positions").option("user", "root").option("password", "password").load()
    salary_by_experience = job_df.groupBy("experience_requirement").agg(avg("salary_min").alias("avg_min"), avg("salary_max").alias("avg_max"), count("*").alias("job_count")).orderBy("experience_requirement")
    salary_by_education = job_df.groupBy("education_requirement").agg(avg("salary_min").alias("avg_min"), avg("salary_max").alias("avg_max"), count("*").alias("job_count")).orderBy("education_requirement")
    salary_by_position = job_df.groupBy("position_name").agg(avg("salary_min").alias("avg_min"), avg("salary_max").alias("avg_max"), count("*").alias("job_count")).orderBy(desc("avg_max"))
    salary_by_city = job_df.groupBy("job_location").agg(avg("salary_min").alias("avg_min"), avg("salary_max").alias("avg_max"), count("*").alias("job_count")).orderBy(desc("avg_max"))
    salary_distribution = job_df.select("salary_min", "salary_max", ((col("salary_min") + col("salary_max")) / 2).alias("avg_salary"))
    salary_ranges = salary_distribution.withColumn("salary_range", when(col("avg_salary") < 10000, "10k以下").when(col("avg_salary") < 20000, "10k-20k").when(col("avg_salary") < 30000, "20k-30k").when(col("avg_salary") < 50000, "30k-50k").otherwise("50k以上"))
    range_stats = salary_ranges.groupBy("salary_range").agg(count("*").alias("count")).orderBy("salary_range")
    company_salary_rank = job_df.groupBy("company_name").agg(avg("salary_max").alias("avg_max_salary"), count("*").alias("position_count")).filter(col("position_count") >= 3).orderBy(desc("avg_max_salary"))
    skill_salary_impact = job_df.select("salary_max", regexp_extract("job_description", r"(Python|机器学习|深度学习|NLP|CV|大模型|Transformer|PyTorch|TensorFlow)", 1).alias("skill")).filter(col("skill") != "").groupBy("skill").agg(avg("salary_max").alias("avg_salary"), count("*").alias("count")).orderBy(desc("avg_salary"))
    bonus_analysis = job_df.select("company_name", "salary_max", when(col("job_description").contains("股票期权"), 1).otherwise(0).alias("has_stock"), when(col("job_description").contains("年终奖"), 1).otherwise(0).alias("has_bonus")).groupBy("company_name").agg(avg("salary_max").alias("base_salary"), avg("has_stock").alias("stock_ratio"), avg("has_bonus").alias("bonus_ratio"))
    salary_trend_monthly = job_df.groupBy("publish_month").agg(avg("salary_max").alias("monthly_avg_salary")).orderBy("publish_month")
    percentile_analysis = salary_distribution.selectExpr("percentile_approx(avg_salary, 0.25) as p25", "percentile_approx(avg_salary, 0.5) as p50", "percentile_approx(avg_salary, 0.75) as p75", "percentile_approx(avg_salary, 0.9) as p90")
    salary_data = {"salary_by_experience": salary_by_experience.toPandas().to_dict('records'), "salary_by_education": salary_by_education.toPandas().to_dict('records'), "salary_by_position": salary_by_position.toPandas().to_dict('records'), "salary_by_city": salary_by_city.toPandas().to_dict('records'), "range_stats": range_stats.toPandas().to_dict('records'), "company_salary_rank": company_salary_rank.toPandas().to_dict('records'), "skill_salary_impact": skill_salary_impact.toPandas().to_dict('records'), "bonus_analysis": bonus_analysis.toPandas().to_dict('records'), "salary_trend_monthly": salary_trend_monthly.toPandas().to_dict('records'), "percentile_analysis": percentile_analysis.toPandas().to_dict('records')}
    return JsonResponse(salary_data, safe=False)

六、部分文档展示

七、END

💕💕文末获取源码联系计算机编程果茶熊