add lab

Author: xinetzone

lab/__Init__.py

@@ -0,0 +1,88 @@
+import numpy as np
+
+from mxnet import nd
+from mxnet.gluon import data as gdata
+from mxnet.gluon.data.vision import transforms as gtf
+
+from datatools import Loader
+
+# 数据增强
+transform_train = gtf.Compose([
+    # 随机对图像裁剪出面积为原图像面积0.08~1倍、且高和宽之比在3/4~4/3的图像，再放缩为高和
+    # 宽都是为 224 的新图
+    gtf.RandomResizedCrop(
+        224, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)),
+    gtf.RandomFlipLeftRight(),
+    # 随机变化亮度、对比度和饱和度
+    gtf.RandomColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),
+    # 随机加噪声
+    gtf.RandomLighting(0.1),
+    gtf.ToTensor(),
+    # 对图像的每个通道做标准化
+    gtf.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+
+transform_test = gtf.Compose([
+    gtf.Resize(256),
+    # 将图像中央的高和宽均为 224 的正方形区域裁剪出来
+    gtf.CenterCrop(224),
+    gtf.ToTensor(),
+    gtf.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+
+
+class SimpleDataset:
+    def __init__(self, name, numLabels, root='E:/xdata/X.h5'):
+        import tables as tb
+        h5 = tb.open_file(root)
+        self.name = name
+        self._dataset = h5.root[name]
+        self.label_names = self._get_label_names(is_fine_labels=False)
+        self._split(numLabels)
+        self.testX, self.testY = self._dataset.testX[:], self._dataset.testY[:]
+        h5.close()
+
+    def _get_label_names(self, is_fine_labels=False):
+        if self.name != 'cifar100':
+            return np.asanyarray(self._dataset.label_names, dtype='U')
+        elif is_fine_labels:
+            return np.asanyarray(self._dataset.fine_label_names, dtype='U')
+        else:
+            return np.asanyarray(self._dataset.coarse_label_names, dtype='U')
+
+    def _split(self, numLabels):
+        from sklearn.model_selection import train_test_split
+        xTr = self._dataset.trainX[:]
+        yTr = self._dataset.trainY[:]
+        test_size = xTr.shape[0] - numLabels
+        # 数据集划分操作
+        self.trainX, self.valX, self.trainY, self.valY = train_test_split(
+            xTr, yTr, test_size=test_size, shuffle=True)
+
+
+class AugLoader(Loader, gdata.Dataset):
+    def __init__(self, batch_size, X, Y=None, shuffle=True, *args, **kwargs):
+        super().__init__(batch_size, X, Y=None, shuffle=True, *args, **kwargs)
+        self.X = nd.array(X[:])
+        if not Y is None:
+            self.Y = nd.array(Y[:])
+
+    def aug_imgs(self, imgs):
+        '''
+        对 图像做数据增强 预处理
+        dataset 需要有 type 属性（'train', 'test'）
+        '''
+        transforms_dict = {'train': transform_train, 'test': transform_test}
+        return nd.stack(*[transforms_dict[self.type](img) for img in imgs])
+
+    def __iter__(self):
+        idx = np.arange(self.nrows)
+        if self.type == 'train':
+            np.random.shuffle(idx)
+        for start in range(0, self.nrows, self.batch_size):
+            end = min(start + self.batch_size, self.nrows)
+            K = nd.array(idx[start:end])
+            if self.Y is None:
+                yield self.aug_imgs(self.X.take(K, 0))
+            else:
+                yield self.aug_imgs(self.X.take(K, 0)), self.Y.take(K, 0)

lab/dataset.py

@@ -0,0 +1,58 @@
+import numpy as np
+
+
+class Loader(dict):
+    """
+    方法
+    ========
+    L 为该类的实例
+    len(L)::返回 batch 的批数
+    iter(L)::即为数据迭代器
+
+    参数
+    =============
+    type: 'train', 'test'
+
+    Return
+    ========
+    可迭代对象（numpy 对象）
+    """
+
+    def __init__(self, batch_size, X, Y=None, shuffle=True, *args, **kwargs):
+        '''
+        X, Y 均为类 numpy, 可以是 HDF5 
+        '''
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
+        self.batch_size = batch_size
+        if shuffle:
+            self.type = 'train'
+        else:
+            self.type = 'test'
+
+        if not hasattr(X, 'take'):
+            self.X = X[:]
+        else:
+            self.X = X
+        self.nrows = len(self.X)
+        if Y is not None:
+            if not hasattr(Y, 'take'):
+                self.Y = Y[:]
+        else:
+            self.Y = None
+
+    def __iter__(self):
+        idx = np.arange(self.nrows)
+        if self.type == 'train':
+            np.random.shuffle(idx)
+
+        for start in range(0, self.nrows, self.batch_size):
+            end = min(start + self.batch_size, self.nrows)
+            K = idx[start:end].tolist()
+            if self.Y is None:
+                yield self.X.take(K, axis=0)
+            else:
+                yield self.X.take(K, axis=0), self.Y.take(K, axis=0)
+
+    def __len__(self):
+        return round(self.nrows / self.batch_size)  # 向上取整

lab/datatools.py

@@ -0,0 +1,115 @@
+import time
+
+from mxnet import metric, autograd
+from mxnet.gluon import loss as gloss, Trainer
+from gluoncv.utils import TrainingHistory  # 可视化
+
+from utils import make_dirs  # 创建多个目录
+
+
+class Tools:
+    def __init__(self, datasetName):
+        self._get_result_dir(datasetName)
+
+    def _get_result_dir(self, datasetName):
+        self.modelDir = f'models/{datasetName}'
+        self.resultDir = f'results/{datasetName}'
+        make_dirs(self.modelDir, self.resultDir)
+
+
+def evaluate_loss(data_iter, net, ctx, loss):
+    l_sum, n = 0.0, 0
+    #loss = gloss.SoftmaxCrossEntropyLoss()
+    for X, y in data_iter:
+        y = y.as_in_context(ctx).astype('float32')  # 模型的输出是 float32 类型数据
+        outputs = net(X.as_in_context(ctx))  # 模型的输出
+        l_sum += loss(outputs, y).sum().asscalar()  # 计算总损失
+        n += y.size  # 计算样本数
+    return l_sum / n  # 计算平均损失
+
+
+def test(valid_iter, net, ctx):
+    val_metric = metric.Accuracy()
+    for X, y in valid_iter:
+        X = X.as_in_context(ctx)
+        y = y.as_in_context(ctx).astype('float32')  # 模型的输出是 float32 类型数据
+        outputs = net(X)
+        val_metric.update(y, outputs)
+    return val_metric.get()
+
+
+def get_result_dirs(datasetName):
+    tools = Tools(datasetName)
+    return tools.modelDir, tools.resultDir
+
+
+def train(ctx,
+          loss,
+          trainer,
+          datasetName,
+          modelName,
+          net,
+          train_iter,
+          valid_iter,
+          num_epochs,
+          n_retrain_epoch=0):
+    '''
+    n_retrain_epoch 是从第 n_retrain_epoch 次开始训练模型
+    '''
+    train_metric = metric.Accuracy()
+    train_history = TrainingHistory(['training-error', 'validation-error'])
+    best_val_score = 0
+    modelDir, resultDir = get_result_dirs(datasetName)
+    for epoch in range(num_epochs):
+        train_l_sum, n, start = 0.0, 0, time.time()  # 计时开始
+        train_metric.reset()
+        for X, y in train_iter:
+            X = X.as_in_context(ctx)
+            y = y.as_in_context(ctx).astype('float32')  # 模型的输出是 float32 类型数据
+            with autograd.record():  # 记录梯度信息
+                outputs = net(X)  # 模型输出
+                L = loss(outputs, y)
+                l = L.mean()  # 计算总损失
+            l.backward()  # 反向传播
+            trainer.step(1)
+            train_l_sum += L.sum().asscalar()  # 计算该批量的总损失
+            train_metric.update(y, outputs)  # 计算训练精度
+            n += y.size
+        _, train_acc = train_metric.get()
+        time_s = "time {:.2f} sec".format(time.time() - start)  # 计时结束
+        valid_loss = evaluate_loss(valid_iter, net, ctx, loss)  # 计算验证集的平均损失
+        _, val_acc = test(valid_iter, net, ctx)  # 计算验证集的精度
+        epoch_s = (
+            "epoch {:d}, train loss {:.5f}, valid loss {:.5f}, train acc {:.5f}, valid acc {:.5f}, ".
+            format(n_retrain_epoch + epoch, train_l_sum / n, valid_loss,
+                   train_acc, val_acc))
+        print(epoch_s + time_s)
+        train_history.update([1 - train_acc, 1 - val_acc])  # 更新图像的纵轴
+        train_history.plot(save_path=f'{resultDir}/{modelName}_history.png')  # 实时更新图像
+        if abs(train_acc-val_acc)>.3:
+            break
+        if val_acc > best_val_score:  # 保存比较好的模型
+            best_val_score = val_acc
+            net.save_parameters('{}/{:.4f}-{}-{:d}-best.params'.format(
+                modelDir, best_val_score, modelName, n_retrain_epoch + epoch))
+
+
+def train_fine_tuning(datasetName,
+                      modelName,
+                      learning_rate,
+                      net,
+                      train_iter,
+                      valid_iter,
+                      num_epochs,
+                      n_retrain_epoch=0):
+    import d2lzh as d2l
+    ctx = d2l.try_all_gpus()[0]
+    net.collect_params().reset_ctx(ctx)
+    net.hybridize()
+    loss = gloss.SoftmaxCrossEntropyLoss()
+    trainer = Trainer(net.collect_params(), 'sgd', {
+        'learning_rate': learning_rate,
+        'wd': 0.001
+    })
+    train(ctx, loss, trainer, datasetName, modelName, net, train_iter,
+          valid_iter, num_epochs, n_retrain_epoch)

lab/gluontools.py

@@ -0,0 +1,9 @@
+import os
+
+def make_dirs(*dir_names):
+    '''
+    创建多个目录
+    '''
+    for dir_name in dir_names:
+        if not os.path.exists(dir_name):
+            os.makedirs(dir_name)