干货 | 携程基于LSTM的广告库存预估算法

# 计算不同聚类个数下的聚类误差    n = 2    sse = []    for k in range(1, 10):        kmeans = KMeans(n_clusters=k, random_state=).fit(data)        sse.append(kmeans.inertia_)
    # 自动寻找肘部点    diff = np.diff(sse)    diff_r = diff[1:] / diff[:-1]    nice_k = np.argmin(diff_r) + n

#绘制SSE随簇数变化的曲线（如图4-5），观察曲线的形状。    plt.plot(range(1, 10), sse, 'bx-')    plt.xlabel('Number of Clusters')    plt.ylabel('SSE')    plt.title('Elbow Method for Optimal k')    plt.show()

import torchfrom torch import nn

class LSTM(nn.Module):    def __init__(self, emb_dims, out_dim, hidden_dim, mid_layers):        super(LSTM, self).__init__()        self.emb_layers = nn.ModuleList([nn.Embedding(x, y) for x, y in emb_dims])        self.rnn = nn.LSTM([sum(x) for x in zip(*emb_dims)][1] + 1, hidden_dim, mid_layers, batch_first=False)        self.reg = nn.Sequential(            nn.Linear(hidden_dim, hidden_dim),            nn.Tanh(),            nn.Linear(hidden_dim, out_dim),        )
    def forward(self, cat_data):        var_x = []        for cat in cat_data:            x = self.emb_layer(cat)            var_x.append(x)        stack = torch.stack(var_x)        y = self.rnn(stack)        return y

loader = Data.DataLoader(dataset=set, batch_size=size, shuffle=True, num_workers=numWorkers)

batch_x = batch_x.permute((1, , 2))

#保存模型:#net:模型#optimizer:优化器 state = {'net': net.state_dict(), 'optimizer': optimizer.state_dict()}torch.save(state, netPath)         #存储至Redis:def saveModel(modelName, netPath):    with open(modelPath, "rb") as f:        pth_model = f.read()    result = redis.set(modelName, pth_model)

# 在Redis中加载模型def readModel(modelName, path):    pthByte = redis.get(modelName)    with open(path, 'wb') as f:        f.write(pthByte)    return torch.load(path, map_location=lambda storage, loc: storage)  # 加载模型和优化器状态,用于训练 # 读取字典model = readModel(modelName, path)
# 加载模型net = LSTM(emb_dims, out_dim, mid_dim, mid_layers).to(device)net.load_state_dict(model['net'])
# 加载优化器optimizer = torch.optim.Adam(net.parameters(), lr=1e-2)optimizer.load_state_dict(model['optimizer'])

class EarlyStopping:             def __call__(self, val_loss):        score = val_loss        if self.best_score is None:            self.best_score = score        elif score > self.best_score:            if score <= x:                self.counter += 1            if self.counter >= self.patience:                self.early_stop = True        else:            self.best_score = score            self.counter = 

import torchmetrics
error = torchmetrics.WeightedMeanAbsolutePercentageError()# 调用模型得到预估值predict = net(valid_x)# 计算偏差error = error(predict, valid_y)error_list.append(error.item())

# 1、Listdel X[:]# 2、数组X = []# 3、字典X.clear()# 4、Tensor(此方式需要调用gc.collect()触发GC才可以真正清除内存)del X# 5、清理CUDA显存import torchtorch.cuda.empty_cache()

from flask import Flaskfrom flask import request, jsonify
@app.route('/model/ad/forecast', methods=['post'])def forecast():    # 在clickhouse中实时获取前七天的标签值，传入接口    prepareData = request.json.get('prepareData')    return forecast(prepareData)

# 在Redis中加载模型
def readModel(modelName, path):    pthByte = redis.get(modelName)    with open(path, 'wb') as f:        f.write(pthByte)    return torch.load(path, map_location=lambda storage, loc: storage)