---

前引

---

最近需求也变多了一些 真的得还是抓紧时间挤时间出来学啊
然后kaggle又去入门写了一个 其实之前也是有数据去跑了一些小项目代码

---

从零开始的推荐系统学习之路（十二）---- 动手学深度学习系列 Kaggle预测房价 & Kaggle加州预测房价

---

1、Kaggle实战：预测房价

---

1、预测房价简单讲讲

---

里面除了一些我懒得再搓的代码 基本上都是手搓的 也是感觉很有收获啊 尤其是中间一直跑出来数据是nan 后面发现是一些奇奇怪怪的原因 是因为前面初始化net和后面代码不在一个块里面

然后又出现了一些奇奇怪怪的原因 就是数据一直不下降 后面发现是因为调节学习率的原因 特别有意思 还是颇有收获 时间很紧张 任务很多！

然后我尝试了一下 mlp 过拟合了。。 呵呵

House Prices - Advanced Regression Techniques

---

2、代码实现

---

%matplotlib inline
import numpy as np
import pandas as pd
import torch
from torch import nn
from d2l import torch as d2l

from sklearn.model_selection import KFold

import hashlib
import os
import tarfile
import zipfile
import requests

#@save
DATA_HUB = dict()
DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/'


def download(name, cache_dir=os.path.join('..', 'data')):  #@save
    """下载一个DATA_HUB中的文件，返回本地文件名"""
    assert name in DATA_HUB, f"{name} 不存在于 {DATA_HUB}"
    url, sha1_hash = DATA_HUB[name]
    os.makedirs(cache_dir, exist_ok=True)
    fname = os.path.join(cache_dir, url.split('/')[-1])
    if os.path.exists(fname):
        sha1 = hashlib.sha1()
        with open(fname, 'rb') as f:
            while True:
                data = f.read(1048576)
                if not data:
                    break
                sha1.update(data)
        if sha1.hexdigest() == sha1_hash:
            return fname  # 命中缓存
    print(f'正在从{url}下载{fname}...')
    r = requests.get(url, stream=True, verify=True)
    with open(fname, 'wb') as f:
        f.write(r.content)
    return fname


def download_extract(name, folder=None):  #@save
    """下载并解压zip/tar文件"""
    fname = download(name)
    base_dir = os.path.dirname(fname)
    data_dir, ext = os.path.splitext(fname)
    if ext == '.zip':
        fp = zipfile.ZipFile(fname, 'r')
    elif ext in ('.tar', '.gz'):
        fp = tarfile.open(fname, 'r')
    else:
        assert False, '只有zip/tar文件可以被解压缩'
    fp.extractall(base_dir)
    return os.path.join(base_dir, folder) if folder else data_dir

def download_all():  #@save
    """下载DATA_HUB中的所有文件"""
    for name in DATA_HUB:
        download(name)



DATA_HUB['kaggle_house_train'] = (  #@save
    DATA_URL + 'kaggle_house_pred_train.csv',
    '585e9cc93e70b39160e7921475f9bcd7d31219ce')

DATA_HUB['kaggle_house_test'] = (  #@save
    DATA_URL + 'kaggle_house_pred_test.csv',
    'fa19780a7b011d9b009e8bff8e99922a8ee2eb90')



kaggle_house_train_data = pd.read_csv(download('kaggle_house_train'))
kaggle_house_test_data = pd.read_csv(download('kaggle_house_test'))

kaggle_house_train_data.shape, kaggle_house_test_data.shape




kaggle_house_train_data.describe(), kaggle_house_test_data.describe()




kaggle_house_train_data.info()





all_features = pd.concat((kaggle_house_train_data.iloc[:, 1:-1], kaggle_house_test_data.iloc[:, 1:]), axis=0)
print(all_features.shape)
all_features.head()





numberic_cols = all_features.select_dtypes(exclude=['object']).columns
all_features[numberic_cols] = all_features[numberic_cols].apply(lambda x : (x - x.mean()) / x.std())
all_features[numberic_cols] = all_features[numberic_cols].fillna(0)
print(all_features.shape)
all_features.head()

# all_features.apply(lambda x : (x - x.mean()) / x.std())




for col in all_features.columns:
    print(all_features[col].describe())




train_data_df = all_features.iloc[:len(kaggle_house_train_data)]
label_df = kaggle_house_train_data.SalePrice


test_data_df = all_features.iloc[len(kaggle_house_train_data):]
train_data_df.shape, test_data_df.shape




num_layer1 = train_data_df.shape[1]
num_layer2 = 128
output = 1

# def get_net():
#     net = nn.Sequential(nn.Linear(num_layer1, num_layer2), nn.Dropout(0.1), nn.Linear(num_layer2, output))
    
#     net[0].weight.data.normal_(0, 0.01), net[0].bias.data.zero_()
#     net[2].weight.data.normal_(0, 0.01), net[2].bias.data.zero_()

#     return net

def get_net():
    net = nn.Sequential(nn.Linear(num_layer1, output))
    
    net[0].weight.data.normal_(0, 0.01), net[0].bias.data.zero_()
    # net[2].weight.data.normal_(0, 0.01), net[2].bias.data.zero_()

    return net

# net[1].weight.data.normal_(0, 0.01), net[1].bias.data.zero_()
# net[2].weight.data.normal_(0, 0.01), net[2].bias.data.zero_()
# net[0].weight.data, net[0].bias.data

# def rmse(y_hat, y, eps=1e-8):
#     y_hat = torch.nan_to_num(y_hat, nan=1.0)  # 将NaN替换为1
#     # print(y_hat, torch.log(torch.abs(y_hat)), ' ', torch.log(y))
#     return torch.sqrt((torch.max(torch.log(y_hat)) - torch.log(y)) ** 2)

# loss = rmse
mse_loss_fn = nn.MSELoss()

def log_rmse(net, features, labels):
    # 为了在取对数时进一步稳定该值，将小于1的值设置为1
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
    rmse = torch.sqrt(mse_loss_fn(torch.log(clipped_preds), torch.log(labels)))
    return rmse.item()

batch_size = 50
num_epochs = 100
weight_decay = 0.001
learning_rate = 1

train_data = torch.tensor(train_data_df.values, dtype=torch.float32)
labels = torch.tensor(label_df.values, dtype=torch.float32).reshape(-1, 1)
test_data = torch.tensor(test_data_df.values, dtype=torch.float32)
loss = nn.MSELoss()
# train_data_df.iloc[0], labels[0]
# train_data_df.iloc[1], labels[1]

# net(train_data[0]), torch.log(labels[0])

def train(net, train_features, train_labels, test_features, test_labels,
          num_epochs, learning_rate, weight_decay, batch_size):
    train_ls, test_ls = [], []
    trainer = torch.optim.Adam(params=net.parameters(), weight_decay=weight_decay, lr=learning_rate)
    
    for epoch in range(num_epoch):
        for X, y in d2l.load_array((train_data, labels), batch_size, is_train=True):
            l = loss(net(X), y)
            # l = loss(net(train_features), train_labels)
            trainer.zero_grad()
            l.backward()
            trainer.step()

        train_ls.append(log_rmse(net, train_features, train_labels))
        if test_labels is not None:
            test_ls.append(log_rmse(net, test_features, test_labels))
            
        # print(epoch + 1, ' ', torch.sqrt(loss(net(train_data), labels)).item())
    
    return train_ls, test_ls



kf = KFold(n_splits=5, shuffle=True, random_state=42)

for i, (train_idx, val_idx) in enumerate(kf.split(train_data)):
    train_l_sum, valid_l_sum = 0, 0
    train_X, train_y = train_data[train_idx], labels[train_idx]
    val_X, val_y = train_data[val_idx], labels[val_idx]
    net = get_net()
    train_ls, valid_ls = train(net, train_X, train_y, val_X, val_y, num_epochs, learning_rate,
                               weight_decay, batch_size)
    train_l_sum += train_ls[-1]
    valid_l_sum += valid_ls[-1]
    d2l.plot(list(range(1, num_epochs + 1)), [train_ls, valid_ls],
             xlabel='epoch', ylabel='rmse', xlim=[1, num_epochs],
             legend=['train', 'valid'], yscale='log')
    print(f'训练log rmse{float(train_ls[-1]):f}', f'验证log rmse{float(valid_ls[-1]):f}')




def train_and_pred(train_features, test_features, train_labels, test_data,
                   num_epochs, lr, weight_decay, batch_size):
    net = get_net()
    train_ls, _ = train(net, train_features, train_labels, None, None,
                        num_epochs, lr, weight_decay, batch_size)
    d2l.plot(np.arange(1, num_epochs + 1), [train_ls], xlabel='epoch',
             ylabel='log rmse', xlim=[1, num_epochs], yscale='log')
    print(f'训练log rmse：{float(train_ls[-1]):f}')
    # 将网络应用于测试集。
    preds = net(test_features).detach().numpy()
    # 将其重新格式化以导出到Kaggle
    test_data['SalePrice'] = pd.Series(preds.reshape(1, -1)[0])
    submission = pd.concat([test_data['Id'], test_data['SalePrice']], axis=1)
    submission.to_csv('submission.csv', index=False)

train_and_pred(train_data, test_data, labels, kaggle_house_test_data,
               num_epochs, learning_rate, weight_decay, batch_size)

---

3、kaggle提交

---

这里很简单 就是生成csv文件 然后拖动到kaggle submit即可
最上面的是用了一个两层神经网络 过拟合了。。。。

---

2、Kaggle加州预测房价

---

这个数据集就明显大了很多 然后需要处理一下了
一共80多M 可以当作一个比较正式的数据去处理了

Kaggle竞赛入门讲解-加州房价预测

---

这东西搞了两天 学习了很多很多 下面也可以详细讲一下 我的一个大概的思路
一天晚上 + 大半天的时间

---

1、熟悉数据 & 大概思路讲解

---

我先说一下 完全自己没思路的时候大概怎么处理的

首先还是先把没用的数据给尝试去剔除了
首先是用pd.describe() 大概看了一下数据分布 把数据分成了两部分 numeric 和 object

然后 循环去打印数据 describe 发现有部分数据是几乎没有聚合 是无用的 这些部分数据就先直接丢弃

然后我刚开始的时候 对于数值型数值 那个时候 确实也没有分析 相关系数 然后都把这些变量作为参数传入了 后面第二天的时候 去分析了 pearson 和 spearman 把一些关键的变量也才提取出来

刚开始处理数据的时候 主要是根据名字 去自己认为是否是相关的 我还专门去处理了 例如 BedRooms的数据 去计数里面含Bedroom的个数 约等于认为是其卧室的数量

然后尝试去把所有的字符串变量都放进纬度向量了 其中大部分都是根据value_count正则化 取的top20 看看包含了多少的数据 如果包含了90多的 且unique数据很大 则直接取了

后面发现第一层input纬度大概缩减后 第一次是2w多纬度 第二次是1w多纬度 都发现了一个问题 过拟合 4层的神经网络 因为训练样本只有3w多个 而且数据确实也都没有处理 全丢进去去处理了 后面也尝试了PCA降维 效果也不好 而且训练也慢

中间因为没有处理异常值 例如去标准化的时候 没有处理inf 没有处理na 对one-hot变量也去标准化 导致了很多问题吧 损失loss == nan

总之倒腾到凌晨5点 最后可以训练出来了 但是测试集rmse0.2出头 训练集rmse0.42 我用dropout 和 l2正则化 都没有很有效的解决这些问题。。。

---

到了今天 然后去稍微借鉴了一下思路 我觉得提升最多的就是对于数值型 确实应该算一下相关系数 然后 对于object类型的 确实也应该先处理 最有可能 聚合后纬度最小的数据

当然后面训练的时候 我也尝试去把label给log化了 因为我发现不对数化 学习率不太好调

后面总之 不停的尝试 总算有一些还算可以的结果吧

下面直接大概给一下代码

---

2、4层神经网络（dropout 0.5 丢弃太多 层数也很多）

---

num_input = all_features.shape[1]
# num_layer1 = 512
# num_layer2 = 256
# num_layer3 = 128
# num_layer4 = 64
# num_output = 1

num_input = all_features.shape[1]
num_layer1 = 128
num_layer2 = 64
num_layer3 = 32
# num_layer3 = 128
# num_layer4 = 64
num_output = 1

def get_net():
    net = nn.Sequential(nn.Linear(num_input, num_layer1), nn.ReLU(), nn.Dropout(0.5), 
                        nn.Linear(num_layer1, num_layer2), nn.ReLU(), nn.Dropout(0.2), 
                        nn.Linear(num_layer2, num_layer3), nn.ReLU(), nn.Dropout(0.1),
                        nn.Linear(num_layer3, num_output))

    for layer in net:
        if isinstance(layer, nn.Linear):
            nn.init.kaiming_normal_(layer.weight, nonlinearity='relu')
            nn.init.zeros_(layer.bias)

    return net

# net = get_net()
# trainer = torch.optim.Adam(params=net.parameters(), weight_decay=0.001)
    

---

3、3层MLP（dropout 0.2 0.1 batch_size 256） 0.22 score

---

---

4、3层MLP（dropout 0.0 0.0 batch_size 512） 0.19 score

---

---

5、含思考草稿部分（模型过拟合 & 欠拟合思路全代码）

---

%matplotlib inline
import numpy as np
import pandas as pd
import torch
from torch import nn
from d2l import torch as d2l
import sns

from sklearn.model_selection import KFold
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler



import pandas as pd
import numpy as np
import psutil
import os

def reduce_mem_usage(df):
    """ iterate through all the columns of a dataframe and modify the data type
        to reduce memory usage.        
    """
    start_mem = df.memory_usage().sum() / 1024**2
    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            elif str(col_type)[:4] == 'uint':
                if c_min > np.iinfo(np.uint8).min and c_max < np.iinfo(np.uint8).max:
                    df[col] = df[col].astype(np.uint8)
                elif c_min > np.iinfo(np.uint16).min and c_max < np.iinfo(np.uint16).max:
                    df[col] = df[col].astype(np.uint16)
                elif c_min > np.iinfo(np.uint32).min and c_max < np.iinfo(np.uint32).max:
                    df[col] = df[col].astype(np.uint32)
                elif c_min > np.iinfo(np.uint64).min and c_max < np.iinfo(np.uint64).max:
                    df[col] = df[col].astype(np.uint64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype('category')

    end_mem = df.memory_usage().sum() / 1024**2
    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
    
    return df

def show_memories():
    
    total_df_memory = 0
    for var_name, obj in list(globals().items()):
        if not var_name.startswith('_') and isinstance(obj, pd.DataFrame):
            obj_memory = obj.memory_usage(deep=True).sum() / (1024 ** 2)
            total_df_memory += obj_memory
            print(f"{var_name} memory usage: {obj_memory:.2f} MB")

    print(f"Total Memory Usage for DFs: {total_df_memory/1024:.2f} GB")
    print(f"Total Memory Usage: {psutil.Process(os.getpid()).memory_info().rss/1024**3:.2f} GB")



kaggle_house_train_data = pd.read_csv("california-house-prices/train.csv")
kaggle_house_test_data = pd.read_csv("california-house-prices/test.csv")
kaggle_house_train_data.shape, kaggle_house_test_data.shape
kaggle_house_train_data, kaggle_house_test_data = reduce_mem_usage(kaggle_house_train_data), reduce_mem_usage(kaggle_house_test_data)


kaggle_house_train_data.describe()


kaggle_house_test_data.describe()


all_features = pd.concat((kaggle_house_train_data.iloc[:, [1, *range(3,len(kaggle_house_train_data.columns))]], kaggle_house_test_data.iloc[:, 1:]))
all_features.shape


all_features.describe()



all_features.info()


# numeric_columns = all_features.select_dtypes(exclude="object").columns
# all_features[numeric_columns] = all_features[numeric_columns].apply(lambda x: (x - x.mean()) / x.std())
# all_features[numeric_columns] = all_features[numeric_columns].fillna(0)
# all_features[numeric_columns]

object_columns = all_features.select_dtypes(include="object").columns
print(object_columns)
for column in object_columns:
    print(all_features[column].describe())




# Address & Summary & State Drop

"""
应该删除的特征：
Address​ (地址) - 唯一性太高（78875/79065） ✅
Summary​ (描述) - 文本唯一性太高，除非做NLP处理 ✅
State​ (州) - 几乎全是CA（99.4%），方差太低 ✅

应该保留并进行特征工程的特征：
Type​ (房屋类型) - 保留，进行one-hot编码 ✅
Bedrooms​ (卧室数) - 保留，作为数值特征 ✅
Listed On​ (上市日期) - 保留，提取年、月、季度等特征 ✅ --------->>  SoldDiffTime（数字）
Last Sold On​ (上次售出日期) - 保留，计算时间间隔 ✅ --------->>  SoldDiffTime（数字）

需要特殊处理的特征：
Appliances included​ (包含设备) - 拆分为多个二元特征
学校特征​ (Elementary/Middle/High School) - 高基数，使用频率编码或分组

可能有信息但需处理的特征：
Parking​ 和 Parking features​ - 可能重复，提取关键词 ✅ --------->>  Parking
Heating​ 和 Heating features​ - 可能重复，需要检查相关性 ✅ --------->>  Heating
Cooling​ 和 Cooling features​ - 同上 ✅ --------->>  CoolingAggs
Flooring​ (地板) - 保留并编码

"""

drop_columns = ["Address", "State", "Summary", "Last Sold On", "Listed On", "Cooling", "Cooling features", "Heating", "Heating features",
                "Parking", "Parking features", "Flooring", "Appliances included"]
                    # 20    20                       20             20 100 
one_hot_columns = ["Type", "CoolingAndHeatingAggs", "ParkingAggs", "HeatingAggs", "Summary",]

numeric_columns = ["SoldDiffTime", "Bedrooms"]

# 计算间隔 
# df["sold diff time"] = (df['Listed On'] - df['Last Sold On']).dt.days.fillna(0)




data = {
    'Listed On': ['2019-10-24', '2019-10-16', '2019-08-25', '2019-10-24'],
    'Last Sold On': [None, '2019-08-30', None, '2016-08-30']
}
df = pd.DataFrame(data)

print("原始数据:")
print(df)
print(f"数据类型: {df['Listed On'].dtype}, {df['Last Sold On'].dtype}")

# 转换为 datetime
df['Listed On'] = pd.to_datetime(df['Listed On'])
df['Last Sold On'] = pd.to_datetime(df['Last Sold On'])
df["DiffTime"] = (df['Listed On'] - df['Last Sold On']).dt.days.fillna(0)
df["DiffTime"]

# all_features[["Listed On", "Last Sold On"]]



all_features.Summary.unique()



type_top_n_categories = all_features.Type.value_counts(normalize=True).head(20).index.to_list()
type_top_n_categories

test_df = all_features.Type.apply(lambda x: x if x in type_top_n_categories else "others")



all_features.Region.value_counts(normalize=True).head(500).sum()*100



all_features[["Heating", "Cooling"]].value_counts(normalize=True).head(400).sum()
# all_features.Cooling.value_counts(normalize=True).head(20).sum()
# all_features.Heating.value_counts(normalize=True).head(20).sum()



all_features[["Heating features", "Heating"]]
all_features.Cooling.describe()

all_features.Cooling.value_counts(normalize=True).head(100)*100

# cooling_freq = all_features.Cooling.value_counts(normalize=True).head(100) * 100
# print("前100个频率分布:")
# print(cooling_freq)
# print(f"\n总类别数: {all_features.Cooling.nunique()}")
# print(f"前10个类别覆盖: {cooling_freq.head(10).sum():.2f}%")
# print(f"前20个类别覆盖: {cooling_freq.head(50).sum():.2f}%")

# value_counts = all_features.Cooling.value_counts()

n = 20

# 2. 获取前N个类别
top_n_categories = all_features.Cooling.value_counts().head(n).index.tolist()
# print(top_n_categories)

def find_category(text):
    if isinstance(text, str) == False:
        return "others"
        
    if text in top_n_categories:
        return text
    for s1 in top_n_categories:
        if text.find(s1) != -1:
            return s1

    return "others"

# print(is_in("Multi-Zone, Central AC, Whole House"))


df = all_features.Cooling.apply(lambda x: find_category(x))
# all_features.Cooling.value_counts(normalize=True, dropna=False).head(20).sum()*100
# all_features.Type.value_counts(normalize=True).head(20).sum()*100
# all_features.Heating.value_counts(normalize=True, dropna=False).head(20).sum()*100
all_features.Parking.value_counts(normalize=True, dropna=False).head(100).sum()*100
# all_features.Flooring.value_counts(normalize=True, dropna=False).head(20).sum()



show_memories()


def count_bedrooms_simple(text):
    """
    最简单的卧室数量提取：
    1. 如果是纯数字，直接转为数字
    2. 否则，统计"Bedroom"出现次数
    3. 处理空值
    """
    if pd.isna(text) or text is None:
        return 0
    
    text_str = str(text).strip()
    
    # 1. 检查是否是纯数字
    if text_str.isdigit():
        return int(text_str)
    
    # 2. 统计"Bedroom"出现次数（不区分大小写）
    text_lower = text_str.lower()
    bedroom_count = text_lower.count('bedroom')
    
    return bedroom_count

count_bedrooms_simple("More than One Bedroom on Ground Floor, Master Suite / Retreate - 2+"), count_bedrooms_simple("3")
count_bedrooms_simple("Ground Floor Bedroom, Walk-in Closet, More than One Master Bedroom, Reverse Floor Plan"), count_bedrooms_simple("Master Bedroom on Ground Floor, Master Suite / Retreat, More than One Master Bedroom, Master Suite / Retreate - 2+")


# print(all_features.Bedrooms.unique())

test = all_features['Bedrooms'].apply(lambda x: 0 if pd.isna(x) else count_bedrooms_simple(x))
test
# test.Bedrooms = all_features.Bedrooms.fillna(0)



# Cooling取 20
top_n_categories = all_features.Cooling.value_counts().head(20).index.tolist()
print(top_n_categories)

# type 取20
type_top_n_categories = all_features.Type.value_counts(normalize=True).head(20).index.to_list()
# print(type_top_n_categories)

heating_n_categories = all_features.Heating.value_counts().head(20).index.tolist()
# print(heating_n_categories)

parking_n_categories = all_features.Parking.value_counts().head(100).index.tolist()
# print(parking_n_categories)

flooring_n_categories = all_features.Flooring.value_counts().head(20).index.tolist()
# print(flooring_n_categories)


def find_category(text, top_n):
    if isinstance(text, str) == False:
        return "others"
        
    if text in top_n:
        return text
    for s1 in top_n:
        if text.find(s1) != -1:
            return s1

    return "others"


# 先处理这些类 

all_features['Bedrooms'] = all_features['Bedrooms'].apply(lambda x: 0 if pd.isna(x) else count_bedrooms_simple(x))

all_features["SoldDiffTime"] = (pd.to_datetime(all_features['Listed On']) - pd.to_datetime(all_features['Last Sold On'])).dt.days.fillna(0)

all_features["CoolingAggs"] = all_features.Cooling.apply(lambda x: find_category(x, top_n_categories))
all_features["HeatingAggs"] = all_features.Heating.apply(lambda x: find_category(x, heating_n_categories))
all_features["ParkingAggs"] = all_features.Parking.apply(lambda x: find_category(x, parking_n_categories))
all_features["FlooringAggs"] = all_features.Flooring.apply(lambda x: find_category(x, flooring_n_categories))

all_features["Type"] = all_features["Type"].apply(lambda x: x if x in type_top_n_categories else "others")
# all_features = pd.get_dummies(all_features, dummy_na=True)
# all_features.shape



# all_features = all_features.drop(columns=drop_columns)
all_features[["CoolingAggs", "HeatingAggs", "ParkingAggs", "FlooringAggs"]]
# all_features.FlooringAggs.value_counts(normalize=).head(200).



# all_features = reduce_mem_usage(all_features)
all_features = all_features.drop(columns=drop_columns)
all_features = pd.get_dummies(all_features, dummy_na=True, dtype=float)
all_features.columns, all_features.dtypes, show_memories()



all_features = reduce_mem_usage(all_features)


numeric_columns = all_features.select_dtypes(exclude="object").columns
all_features[numeric_columns] = all_features[numeric_columns].fillna(0)


# all_features["Year built"].describe(), all_features["Year built"].info()
problem_cols = all_features.columns[all_features.mean().isna()].tolist()
if problem_cols:
    all_features[problem_cols] = all_features[problem_cols].astype(np.float32)
    print(f"已将 {len(problem_cols)} 个列转换为float32")
    
    # 检查转换后是否还有NaN
    still_problem = all_features[problem_cols].mean().isna().sum()
    print(f"转换后仍有 {still_problem} 个列的均值是NaN")


# scaler = StandardScaler()
# all_features_scale = scaler.fit_transform(all_features)
# pca = PCA(n_components=1024)
# all_feature_pca = pca.fit_transform(all_features_scale)


# print(f"\nPCA特征形状: {all_feature_pca.shape}")
# print(f"解释方差比例: {pca.explained_variance_ratio_.sum():.2%}")

# # 3. 验证PCA特征的性质
# print("\n验证PCA特征性质:")
# # 检查主成分之间的相关性（应该接近0）
# corr_matrix = np.corrcoef(all_feature_pca.T)
# np.fill_diagonal(corr_matrix, 0)  # 忽略对角线
# max_corr = np.abs(corr_matrix).max()
# print(f"主成分间最大相关系数: {max_corr:.6f} (应该接近0)")



def safe_std(x):
    """安全标准差计算"""
    # 确保是float32
    x_float32 = x.astype(np.float32) if x.dtype != np.float32 else x
    
    # 计算标准差
    std_val = x_float32.std()
    
    # 处理NaN和0标准差
    if pd.isna(std_val) or std_val < 1e-8:
        return 1.0  # 返回1避免除以0
    else:
        return std_val

all_features[numeric_columns] = all_features[numeric_columns].apply(lambda x: (x - x.mean()) / safe_std(x))
all_features.isna().any()


all_features.describe()



# print(all_features.dtypes)
# 获取原始列（不包含下划线）
original_cols = [col for col in all_features.columns if '_' not in col]

# 获取get_dummies生成的列的基础名称（第一个下划线前的部分）
dummy_base_cols = list(set([col.split('_')[0] for col in all_features.columns if '_' in col]))

print(f"原始列（无下划线）: {original_cols} 个")
print(f"独热编码基础列: {dummy_base_cols} 个")

# 合并去重
all_base_cols = sorted(set(original_cols + dummy_base_cols))


num_input = all_features.shape[1]
# num_layer1 = 512
# num_layer2 = 256
# num_layer3 = 128
# num_layer4 = 64
# num_output = 1

num_input = all_features.shape[1]
num_layer1 = 128
num_layer2 = 64
# num_layer3 = 128
# num_layer4 = 64
num_output = 1

def get_net():
    net = nn.Sequential(nn.Linear(num_input, num_layer1), nn.ReLU(), nn.Dropout(0.5), 
                        nn.Linear(num_layer1, num_layer2), nn.ReLU(), nn.Dropout(0.2), 
                        nn.Linear(num_layer2, num_output))

    for layer in net:
        if isinstance(layer, nn.Linear):
            nn.init.kaiming_normal_(layer.weight, nonlinearity='relu')
            nn.init.zeros_(layer.bias)

    return net

# net = get_net()
# trainer = torch.optim.Adam(params=net.parameters(), weight_decay=0.001)


   mse_loss_fn = nn.MSELoss()
loss = nn.MSELoss()

train_data = torch.tensor(all_features.iloc[:len(kaggle_house_train_data)].values, dtype=torch.float32)
train_labels = torch.tensor(kaggle_house_train_data["Sold Price"], dtype=torch.float32).reshape(-1, 1)

def log_rmse(net, features, labels):
    # 为了在取对数时进一步稳定该值，将小于1的值设置为1
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
    rmse = torch.sqrt(mse_loss_fn(torch.log(clipped_preds), torch.log(labels)))
    return rmse.item()



def train(net, train_features, train_labels, test_features, test_labels,
          num_epochs, learning_rate, weight_decay, batch_size):
    train_ls, test_ls = [], []
    trainer = torch.optim.Adam(params=net.parameters(), weight_decay=weight_decay, lr=learning_rate)
    print(train_features.shape, train_labels.shape)
    
    for epoch in range(num_epochs):
        for X, y in d2l.load_array((train_features, train_labels), batch_size, is_train=True):
            l = loss(net(X), y)
            # l = loss(net(train_features), train_labels)
            # print(X,  net(X))
            trainer.zero_grad()
            l.backward()
            trainer.step()
            # print("epoch:{}, l:{:.2f}".format(epoch + 1, torch.sqrt(l)))

        with torch.no_grad():
            net.eval()
            train_rmse_loss = log_rmse(net, train_features, train_labels)
            train_ls.append(train_rmse_loss)
            if test_labels is not None:
                test_ls.append(log_rmse(net, test_features, test_labels))
                
            print(epoch + 1, ' ', train_rmse_loss)
            net.train()  # 恢复训练模式
    
    return train_ls, test_ls


weight_decay = 0.01
learning_rate = 0.01
batch_size = 256
num_epochs = 100

train_data.shape, train_labels.shape, 3825000 ** 2

kf = KFold(n_splits=5, shuffle=True, random_state=42)
for i, (train_idx, val_idx) in enumerate(kf.split(train_data)):
    train_l_sum, valid_l_sum = 0, 0
    train_X, train_y = train_data[train_idx], train_labels[train_idx]
    print(train_X.shape, train_y.shape)
    val_X, val_y = train_data[val_idx], train_labels[val_idx]
    net = get_net()
    train_ls, valid_ls = train(net, train_X, train_y, val_X, val_y, num_epochs, learning_rate,
                               weight_decay, batch_size)
    train_l_sum += train_ls[-1]
    valid_l_sum += valid_ls[-1]
    d2l.plot(list(range(1, num_epochs + 1)), [train_ls, valid_ls],
             xlabel='epoch', ylabel='rmse', xlim=[1, num_epochs],
             legend=['train', 'valid'], yscale='log')
    print(f'训练log rmse{float(train_ls[-1]):f}', f'验证log rmse{float(valid_ls[-1]):f}')

---

---

6、final 代码部分

---

结果不是最优的 但确实这两天一直在手搓 思路上 和 熟练度上 确实提升了很多很多

%matplotlib inline
import numpy as np
import pandas as pd
import torch
from torch import nn
from d2l import torch as d2l
import seaborn as sns
from scipy import stats
import matplotlib.pyplot as plt

from sklearn.model_selection import KFold




import pandas as pd
import numpy as np
import psutil
import os

def reduce_mem_usage(df):
    """ iterate through all the columns of a dataframe and modify the data type
        to reduce memory usage.        
    """
    start_mem = df.memory_usage().sum() / 1024**2
    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            elif str(col_type)[:4] == 'uint':
                if c_min > np.iinfo(np.uint8).min and c_max < np.iinfo(np.uint8).max:
                    df[col] = df[col].astype(np.uint8)
                elif c_min > np.iinfo(np.uint16).min and c_max < np.iinfo(np.uint16).max:
                    df[col] = df[col].astype(np.uint16)
                elif c_min > np.iinfo(np.uint32).min and c_max < np.iinfo(np.uint32).max:
                    df[col] = df[col].astype(np.uint32)
                elif c_min > np.iinfo(np.uint64).min and c_max < np.iinfo(np.uint64).max:
                    df[col] = df[col].astype(np.uint64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype('category')

    end_mem = df.memory_usage().sum() / 1024**2
    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
    
    return df

def show_memories():
    total_df_memory = 0
    for var_name, obj in list(globals().items()):
        if not var_name.startswith('_') and isinstance(obj, pd.DataFrame):
            obj_memory = obj.memory_usage(deep=True).sum() / (1024 ** 2)
            total_df_memory += obj_memory
            print(f"{var_name} memory usage: {obj_memory:.2f} MB")

    print(f"Total Memory Usage for DFs: {total_df_memory/1024:.2f} GB")
    print(f"Total Memory Usage: {psutil.Process(os.getpid()).memory_info().rss/1024**3:.2f} GB")





kaggle_house_train_data = pd.read_csv("california-house-prices/train.csv")
kaggle_house_test_data = pd.read_csv("california-house-prices/test.csv")
kaggle_house_train_data.shape, kaggle_house_test_data.shape
# kaggle_house_train_data, kaggle_house_test_data = reduce_mem_usage(kaggle_house_train_data), reduce_mem_usage(kaggle_house_test_data)



kaggle_house_train_data.describe()



kaggle_house_train_data.describe(), kaggle_house_train_data.columns



all_features = pd.concat((kaggle_house_train_data.iloc[:, [1, *range(3,len(kaggle_house_train_data.columns))]], kaggle_house_test_data.iloc[:, 1:]))
all_features.shape, all_features.columns.to_list()



all_na_data = all_features.isna().sum()
all_data = all_features.shape[0] * all_features.shape[1]

print(all_na_data)
print(f'\nna data {all_na_data.sum() / all_data:.2f}')





def count_bedrooms_simple(text):
    """
    最简单的卧室数量提取：
    1. 如果是纯数字，直接转为数字
    2. 否则，统计"Bedroom"出现次数
    3. 处理空值
    """
    if pd.isna(text) or text is None:
        return 0
    
    text_str = str(text).strip()
    
    # 1. 检查是否是纯数字
    if text_str.isdigit():
        return int(text_str)
    
    # 2. 统计"Bedroom"出现次数（不区分大小写）
    text_lower = text_str.lower()
    bedroom_count = text_lower.count('bedroom')
    
    return bedroom_count

count_bedrooms_simple("More than One Bedroom on Ground Floor, Master Suite / Retreate - 2+"), count_bedrooms_simple("3")
count_bedrooms_simple("Ground Floor Bedroom, Walk-in Closet, More than One Master Bedroom, Reverse Floor Plan"), count_bedrooms_simple("Master Bedroom on Ground Floor, Master Suite / Retreat, More than One Master Bedroom, Master Suite / Retreate - 2+")


# print(all_features.Bedrooms.unique())

all_features['Bedrooms'] = all_features['Bedrooms'].apply(lambda x: 0 if pd.isna(x) else count_bedrooms_simple(x))
# test.Bedrooms = all_features.Bedrooms.fillna(0)




all_features['Last Sold On'] = (pd.to_datetime(all_features['Last Sold On'])).dt.year.fillna(0)
all_features['Listed On'] = (pd.to_datetime(all_features['Listed On'])).dt.year.fillna(0)



numeric_columns = all_features.select_dtypes(exclude="object").columns
all_features[numeric_columns] = all_features[numeric_columns].fillna(method="ffill").fillna(0)
all_features[numeric_columns].isna().mean()
# all_features[numeric_columns]




numeric_features = pd.concat((all_features[numeric_columns].iloc[:len(kaggle_house_train_data)], kaggle_house_train_data["Sold Price"]), axis=1)
# numeric_features = numeric_features.apply(lambda x: (x - x.mean()) / x.std())
numeric_features

# 绘制热力图
# sns.heatmap(corr_matrix,cmap=corr_cmap, center=0, vmin=-1, vmax=1, square=True, cbar_kws={"shrink": 0.8, "label": "相关系数"}, ax=ax)



pearson_corr = numeric_features.corr(method="pearson")
spearman_corr = numeric_features.corr(method="spearman")
numeric_features.columns
# spearman_corr, spearman_p = all_features.spearman(all_features[numeric_columns], df[target_col])



sns.set_style("darkgrid")
plt.figure(figsize=(12, 12))
fig, axes = plt.subplots(1, 2, figsize=(24, 24))

sns.heatmap(pearson_corr, annot=True, fmt='.2f', ax=axes[0])
sns.heatmap(spearman_corr, annot=True, fmt='.2f', ax=axes[1])

plt.show()



# Cooling取 20
top_n_categories = all_features.Cooling.value_counts().head(20).index.tolist()
print(top_n_categories)

# type 取20
type_top_n_categories = all_features.Type.value_counts(normalize=True).head(20).index.to_list()
# print(type_top_n_categories)

heating_n_categories = all_features.Heating.value_counts().head(20).index.tolist()
# print(heating_n_categories)

parking_n_categories = all_features.Parking.value_counts().head(100).index.tolist()
# print(parking_n_categories)

flooring_n_categories = all_features.Flooring.value_counts().head(20).index.tolist()
# print(flooring_n_categories)


def find_category(text, top_n):
    if isinstance(text, str) == False:
        return "others"
        
    if text in top_n:
        return text
    for s1 in top_n:
        if text.find(s1) != -1:
            return s1

    return "others"



all_features["CoolingAggs"] = all_features.Cooling.apply(lambda x: find_category(x, top_n_categories))
all_features["HeatingAggs"] = all_features.Heating.apply(lambda x: find_category(x, heating_n_categories))
all_features["ParkingAggs"] = all_features.Parking.apply(lambda x: find_category(x, parking_n_categories))
all_features["FlooringAggs"] = all_features.Flooring.apply(lambda x: find_category(x, flooring_n_categories))




# all_features.Type = all_features.Type.apply(lambda x: find_category(x, type_top_n_categories))


# show_memories()


keep_numeric_columns = ["Total interior livable area", "Bathrooms", "Full bathrooms", "Tax assessed value", "Annual tax amount", "Listed Price", "Last Sold Price"]

keep_object_columns = ["Type"]

# keep_columns


all_features[keep_numeric_columns] = all_features[keep_numeric_columns].apply(lambda x: (x - x.mean()) / x.std())
all_features = all_features[keep_numeric_columns + keep_object_columns]




all_features = pd.get_dummies(all_features, dummy_na=True, dtype=float)
all_features



num_input = all_features.shape[1]
# num_layer1 = 512
# num_layer2 = 256
# num_layer3 = 128
# num_layer4 = 64
# num_output = 1

num_input = all_features.shape[1]
num_layer1 = 128
num_layer2 = 32
# num_layer3 = 8
# num_layer3 = 128
# num_layer4 = 64
num_output = 1

weight_decay = 0.001
learning_rate = 0.001
batch_size = 256
num_epochs = 100

def get_net():
    net = nn.Sequential(nn.Linear(num_input, num_layer1), nn.ReLU(), nn.Dropout(0.0), 
                        nn.Linear(num_layer1, num_layer2), nn.ReLU(), nn.Dropout(0.0), 
                        # nn.Linear(num_layer2, num_layer3), nn.ReLU(), nn.Dropout(0.1), 
                        nn.Linear(num_layer2, num_output))


    for layer in net:
        if isinstance(layer, nn.Linear):
            nn.init.kaiming_normal_(layer.weight, nonlinearity='relu')
            nn.init.zeros_(layer.bias)

    return net

# net = get_net()
# trainer = torch.optim.Adam(params=net.parameters(), weight_decay=0.001)


    


mse_loss_fn = nn.MSELoss()
loss = nn.MSELoss()

train_data = torch.tensor(all_features.iloc[:len(kaggle_house_train_data)].values, dtype=torch.float32)
train_labels = torch.tensor(kaggle_house_train_data["Sold Price"], dtype=torch.float32).reshape(-1, 1)

train_labels = torch.log(train_labels + 1)  # 避免log(0)
test_data = torch.tensor(all_features.iloc[len(kaggle_house_train_data):].values, dtype=torch.float32)


def log_rmse(net, features, labels):
    # 为了在取对数时进一步稳定该值，将小于1的值设置为1
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
    rmse = torch.sqrt(mse_loss_fn(clipped_preds, labels))
    return rmse.item()




def train(net, train_features, train_labels, test_features, test_labels,
          num_epochs, learning_rate, weight_decay, batch_size):
    train_ls, test_ls = [], []
    trainer = torch.optim.Adam(params=net.parameters(), weight_decay=weight_decay, lr=learning_rate)
    print(train_features.shape, train_labels.shape)
    
    for epoch in range(num_epochs):
        for X, y in d2l.load_array((train_features, train_labels), batch_size, is_train=True):
            y_pred = net(X)
            l = loss(y_pred, y)
            # l = loss(net(train_features), train_labels)
            # print(net(X)[0], y[0])
            trainer.zero_grad()
            l.backward()
            trainer.step()
            if np.random.rand() >= 0.95:
                print("epoch:{}, l:{:.2f}, net[0]:{}, y[0]:{}".format(epoch + 1, torch.sqrt(l), net(X)[0], y[0]))

        with torch.no_grad():
            net.eval()
            train_rmse_loss = log_rmse(net, train_features, train_labels)
            train_ls.append(train_rmse_loss)
            if test_labels is not None:
                test_ls.append(log_rmse(net, test_features, test_labels))
                
            print(epoch + 1, ' ', train_rmse_loss, trainer.param_groups[0]['lr'])
            net.train()  # 恢复训练模式
    
    return train_ls, test_ls




train_data.shape, train_labels.shape, 3825000 ** 2

kf = KFold(n_splits=5, shuffle=True, random_state=42)
for i, (train_idx, val_idx) in enumerate(kf.split(train_data)):
    train_l_sum, valid_l_sum = 0, 0
    train_X, train_y = train_data[train_idx], train_labels[train_idx]
    print(train_X.shape, train_y.shape)
    val_X, val_y = train_data[val_idx], train_labels[val_idx]
    net = get_net()
    train_ls, valid_ls = train(net, train_X, train_y, val_X, val_y, num_epochs, learning_rate,
                               weight_decay, batch_size)
    train_l_sum += train_ls[-1]
    valid_l_sum += valid_ls[-1]
    d2l.plot(list(range(1, num_epochs + 1)), [train_ls, valid_ls],
             xlabel='epoch', ylabel='rmse', xlim=[1, num_epochs],
             legend=['train', 'valid'], yscale='log')
    print(f'训练log rmse{float(train_ls[-1]):f}', f'验证log rmse{float(valid_ls[-1]):f}')




def train_and_pred(train_features, test_features, train_labels, test_data,
                   num_epochs, lr, weight_decay, batch_size):
    net = get_net()
    train_ls, _ = train(net, train_features, train_labels, None, None,
                        num_epochs, lr, weight_decay, batch_size)
    d2l.plot(np.arange(1, num_epochs + 1), [train_ls], xlabel='epoch',
             ylabel='log rmse', xlim=[1, num_epochs], yscale='log')
    print(f'训练log rmse：{float(train_ls[-1]):f}')
    # 将网络应用于测试集。
    preds = torch.exp(net(test_features).detach()).numpy()
    # 将其重新格式化以导出到Kaggle
    test_data['Sold Price'] = pd.Series(preds.reshape(1, -1)[0])
    submission = pd.concat([test_data['Id'], test_data['Sold Price']], axis=1)
    submission.to_csv('submission.csv', index=False)

train_and_pred(train_data, test_data, train_labels, kaggle_house_test_data,
               num_epochs, learning_rate, weight_decay, batch_size)

---

7、kaggle 提交分数部分

---