mindspore/tests/ut/python/parallel/test_reshape.py

# Copyright 2019 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from mindspore.train import Model, ParallelMode
from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits
from mindspore.nn.optim.momentum import Momentum
from mindspore import Tensor
import mindspore as ms
import numpy as np
from mindspore.ops import operations as P
import mindspore.nn as nn
from mindspore.common.parameter import Parameter
from tests.dataset_mock import MindData
from mindspore import context
from tests.ut.python.ops.test_math_ops import VirtualLoss
from mindspore.common.api import _executor
from mindspore.ops import composite as C
from mindspore.ops.operations.comm_ops import _VirtualDataset
from mindspore.ops import functional as F
from mindspore.common.parameter import ParameterTuple
from mindspore.common import dtype as mstype
from mindspore.parallel import set_algo_parameters
context.set_context(mode=context.GRAPH_MODE)
context.reset_auto_parallel_context()

class Dataset(MindData):
    def __init__(self, predict, label, length=3, input_num=2):
        super(Dataset, self).__init__(size=length)
        self.predict = predict
        self.label = label
        self.index = 0
        self.length = length
        self.input_num = input_num

    def __iter__(self):
        return self

    def __next__(self):
        if self.index >= self.length:
            raise StopIteration
        self.index += 1
        if self.input_num == 2:
            return self.predict, self.label
        else:
            return self.predict,

    def reset(self):
        self.index = 0


class ReshapeNet(nn.Cell):
    def __init__(self, strategy0, strategy1, strategy2):
        super(ReshapeNet, self).__init__()
        self.relu = P.ReLU().set_strategy(strategy0)
        self.reshape = P.Reshape().set_strategy(strategy1)
        self.matmul = P.MatMul().set_strategy(strategy2)
        self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")

    def construct(self, x):
        x = self.relu(x)
        x = self.reshape(x, (256, 25088))
        x = self.matmul(x, self.matmul_weight)
        return x


def reshape_net(strategy0, strategy1, strategy2):
    return ReshapeNet(strategy0=strategy0, strategy1=strategy1, strategy2=strategy2)


def reshape_common(parallel_mode, strategy0, strategy1, strategy2, strategy_loss):
    batch_size = 32
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2

    context.reset_auto_parallel_context()
    context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8)
    predict = Tensor(np.ones([32, 512, 7, 7]), dtype=ms.float32)
    label = Tensor(np.ones([32]), dtype=ms.int32)
    dataset = Dataset(predict, label, 2)
    net = reshape_net(strategy0, strategy1, strategy2)

    loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
    loss.softmax_cross_entropy.set_strategy(strategy_loss)
    loss.one_hot.set_strategy(((8,1), (), ()))
    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    model = Model(net, loss, opt)
    model.train(epoch_size, dataset, dataset_sink_mode=False)


def test_reshape1():
    strategy0 = ((8, 1, 1, 1), )
    strategy1 = None
    strategy2 = ((8, 1), (1, 1))
    strategy_loss = ((8, 1), (8, 1))
    reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)


def test_reshape1_strategy_1():
    strategy0 = ((8, 1, 1, 1), )
    strategy1 = ((8, 1, 1, 1), )
    strategy2 = ((8, 1), (1, 1))
    strategy_loss = ((8, 1), (8, 1))
    try:
        reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)
    except:
        pass


def test_reshape1_strategy_2():
    strategy0 = ((8, 1, 1, 1), )
    strategy1 = ((8, 1, 1, 1), )
    strategy2 = ((8, 1), (1, 1))
    strategy_loss = ((8, 1), (8, 1))
    try:
        reshape_common(ParallelMode.AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)
    except:
        pass


def test_reshape2():
    strategy0 = ((8, 1, 1, 1), )
    strategy1 = None
    strategy2 = ((8, 1), (1, 1))
    strategy_loss = ((8, 1), (8, 1))
    reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)


def test_reshape3():
    strategy0 = ((2, 1, 1, 1), )
    strategy1 = None
    strategy2 = ((8, 1), (1, 1))
    strategy_loss = ((8, 1), (8, 1))
    reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)


def test_reshape4():
    strategy0 = ((1, 1, 1, 1), )
    strategy1 = None
    strategy2 = ((8, 1), (1, 1))
    strategy_loss = ((8, 1), (8, 1))
    reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)


def test_reshape5():
    strategy0 = ((2, 1, 1, 1), )
    strategy1 = None
    strategy2 = ((1, 8), (8, 1))
    strategy_loss = ((8, 1), (8, 1))
    reshape_common(ParallelMode.SEMI_AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)


def test_reshape_auto():
    strategy0 = None
    strategy1 = None
    strategy2 = None
    strategy_loss = None
    reshape_common(ParallelMode.AUTO_PARALLEL, strategy0, strategy1, strategy2, strategy_loss)


class NetWithLoss(nn.Cell):
    def __init__(self, network):
        super(NetWithLoss, self).__init__()
        self.loss = VirtualLoss()
        self.network = network

    def construct(self, x):
        predict = self.network(x)
        return self.loss(predict)


class GradWrap(nn.Cell):
    def __init__(self, network):
        super(GradWrap, self).__init__()
        self.network = network

    def construct(self, x):
        return C.grad_all(self.network)(x)


class ReshapeNet1(nn.Cell):
    def __init__(self, strategy0):
        super(ReshapeNet1, self).__init__()
        self.virtual_dataset = _VirtualDataset()
        self.reshape = P.Reshape()
        self.matmul = P.MatMul().set_strategy(strategy0)
        self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")
        self.reshape2 = P.Reshape()

    def construct(self, x):
        x = self.virtual_dataset(x)
        x = self.reshape(x, (256, 25088))
        x = self.matmul(x, self.matmul_weight)
        x = self.reshape2(x, (256 * 256,))
        return x


class ReshapeNet2(nn.Cell):
    def __init__(self, strategy0):
        super(ReshapeNet2, self).__init__()
        self.virtual_dataset = _VirtualDataset()
        self.reshape = P.Reshape()
        self.matmul = P.MatMul().set_strategy(strategy0)
        self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")
        self.reshape2 = P.Reshape()
        self.reduce_sum = P.ReduceSum(keep_dims=True)
        self.reshape3 = P.Reshape()

    def construct(self, x):
        x = self.virtual_dataset(x)
        x = self.reshape(x, (256, 25088))
        x = self.matmul(x, self.matmul_weight)
        x = self.reshape2(x, (256 * 256,))
        x = self.reduce_sum(x, -1)
        x = self.reshape3(x, ())
        return x


class ReshapeNet3(nn.Cell):
    def __init__(self, strategy0):
        super(ReshapeNet3, self).__init__()
        self.virtual_dataset = _VirtualDataset()
        self.reshape = P.Reshape()
        self.matmul = P.MatMul().set_strategy(strategy0)
        self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")
        self.reshape2 = P.Reshape()
        self.reduce_sum = P.ReduceSum(keep_dims=False)
        self.reshape3 = P.Reshape()

    def construct(self, x):
        x = self.virtual_dataset(x)
        x = self.reshape(x, (256, 25088))
        x = self.matmul(x, self.matmul_weight)
        x = self.reshape2(x, (256 * 256,))
        x = self.reduce_sum(x, -1)
        x = self.reshape3(x, (1, 1))
        return x


class ReshapeNet4(nn.Cell):
    def __init__(self, strategy0):
        super(ReshapeNet4, self).__init__()
        self.virtual_dataset = _VirtualDataset()
        self.reshape = P.Reshape()
        self.reshape2 = P.Reshape()
        self.matmul = P.MatMul().set_strategy(strategy0)
        self.matmul_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")

    def construct(self, x):
        x = self.virtual_dataset(x)
        x = self.reshape(x, (256, 25088))
        w = self.reshape2(self.matmul_weight, (25088, 256))
        x = self.matmul(x, w)
        return x


class ReshapeNet5(nn.Cell):
    def __init__(self, strategy0):
        super(ReshapeNet5, self).__init__()
        self.virtual_dataset = _VirtualDataset()
        self.reshape = P.Reshape()
        self.matmul1 = P.MatMul().set_strategy(strategy0)
        self.matmul1_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")
        self.matmul2 = P.MatMul().set_strategy(strategy0)

    def construct(self, x):
        x = self.virtual_dataset(x)
        x = self.reshape(x, (256, 25088))
        matmul1_o = self.matmul1(x, self.matmul1_weight)
        matmul2_o = self.matmul2(matmul1_o, x)
        return matmul2_o


class ReshapeNet6(nn.Cell):
    def __init__(self, strategy0):
        super(ReshapeNet6, self).__init__()
        self.virtual_dataset = _VirtualDataset()
        self.reshape = P.Reshape()
        self.matmul1_1 = P.MatMul().set_strategy(strategy0)
        self.matmul1_2 = P.MatMul().set_strategy(strategy0)
        self.matmul1_weight = Parameter(Tensor(np.ones([25088, 256]), dtype=ms.float32), name="weight")
        self.matmul2 = P.MatMul().set_strategy(strategy0)
        self.add = P.TensorAdd()

    def construct(self, x):
        x = self.virtual_dataset(x)
        x = self.reshape(x, (256, 25088))
        matmul1_1_o = self.matmul1_1(x, self.matmul1_weight)
        matmul1_2_o = self.matmul1_2(x, self.matmul1_weight)
        matmul1_o = self.add(matmul1_1_o, matmul1_2_o)
        matmul2_o = self.matmul2(matmul1_o, x)
        return matmul2_o


def reshape_net2(backbone):
    batch_size = 16
    device_num = 16
    context.set_auto_parallel_context(device_num=device_num, global_rank=0)
    input = Tensor(np.ones([batch_size * device_num, 512, 7, 7]).astype(np.float32) * 0.01)

    net = GradWrap(NetWithLoss(backbone))
    context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
    
    _executor.compile(net, input)


def test_reshape_net1_1():
    reshape_net2(ReshapeNet1(((1, 8), (8, 1))))


def test_reshape_net1_2():
    reshape_net2(ReshapeNet1(((1, 8), (8, 2))))


def test_reshape_net2_1():
    reshape_net2(ReshapeNet2(((1, 8), (8, 1))))


def test_reshape_net2_2():
    reshape_net2(ReshapeNet2(((1, 8), (8, 2))))


def test_reshape_net3_1():
    reshape_net2(ReshapeNet3(((1, 8), (8, 1))))


def test_reshape_net3_2():
    reshape_net2(ReshapeNet3(((1, 8), (8, 2))))


def test_reshape_net4_1():
    try:
        reshape_net2(ReshapeNet4(((1, 8), (8, 1))))
    except:
        pass


def test_reshape_net4_2():
    try:
        reshape_net2(ReshapeNet4(((1, 8), (8, 2))))
    except:
        pass


def test_reshape_net5_1():
    reshape_net2(ReshapeNet5(((1, 8), (8, 1))))


def test_reshape_net5_2():
    reshape_net2(ReshapeNet5(((1, 8), (8, 2))))


def test_reshape_net6_1():
    reshape_net2(ReshapeNet6(((1, 8), (8, 1))))


def test_reshape_net6_2():
    reshape_net2(ReshapeNet6(((1, 8), (8, 2))))


class TrainOneStepCell(nn.Cell):
    """
    Network training package class.

    Append an optimizer to the training network after that the construct function
    can be called to create the backward graph.

    Args:
        network (Cell): The training network.
        optimizer (Cell): Optimizer for updating the weights.
        sens (Number): The adjust parameter. Default: 1.0.

    Examples:
        >>> net = Net()
        >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits()
        >>> optim = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
        >>> loss_net = WithLossCell(net, loss_fn)
        >>> train_net = TrainOneStepCell(loss_net, optim)
    """
    def __init__(self, network, optimizer, sens=1.0):
        super(TrainOneStepCell, self).__init__(auto_prefix=False)
        self.network = network
        self.network.add_flags(defer_inline=True)
        self.weights = ParameterTuple(network.trainable_params())
        self.optimizer = optimizer
        self.grad = C.GradOperation('grad',
                                    get_by_list=True,
                                    sens_param=True)
        self.sens = sens

    def construct(self, data):
        weights = self.weights
        loss = self.network(data)
        sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens)
        grads = self.grad(self.network, weights)(data, sens)

        return F.depend(loss, self.optimizer(grads))


def reshape_common2(parallel_mode, net):
    batch_size = 16
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2

    predict = Tensor(np.ones([batch_size, 512, 7, 7]), dtype=ms.float32)
    label = Tensor(np.ones([batch_size]), dtype=ms.int32)
    dataset = Dataset(predict, label, 2, input_num=1)
    context.reset_auto_parallel_context()
    context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=16)

    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    train_net = TrainOneStepCell(net, opt).set_train()
    model = Model(train_net)
    model.train(epoch_size, dataset, dataset_sink_mode=False)


def test_reshape_common2_0():
    reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet1(((1, 8), (8, 1))))


def test_reshape_common2_1():
    reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet1(((1, 8), (8, 2))))


def test_reshape_common2_2():
    reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet2(((1, 8), (8, 1))))


def test_reshape_common2_3():
    reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet2(((1, 8), (8, 2))))


def test_reshape_common2_4():
    reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet3(((1, 8), (8, 1))))


def test_reshape_common2_5():
    reshape_common2(ParallelMode.SEMI_AUTO_PARALLEL, ReshapeNet3(((1, 8), (8, 2))))


class BatchNormReshapeNet(nn.Cell):
    def __init__(self):
        super(BatchNormReshapeNet, self).__init__()
        self.vd = P._VirtualDataset()
        self.batch_norm = nn.BatchNorm1d(512, affine=False)
        self.reshape = P.Reshape()
        self.prelu = nn.PReLU(channel=256)

    def construct(self, x):
        x = self.vd(x)
        x = self.batch_norm(x)
        x = self.reshape(x, (512, 256))
        x = self.prelu(x)
        return x


def test_batchnorm_reshape_train():
    batch_size = 16
    device_num = 16
    context.set_auto_parallel_context(device_num=device_num, global_rank=0)
    context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
    input = Tensor(np.ones([batch_size * device_num, 512]).astype(np.float32) * 0.01)

    net = GradWrap(NetWithLoss(BatchNormReshapeNet()))
    
    _executor.compile(net, input)


def bn_with_initialize(out_channels):
    bn = nn.BatchNorm2d(out_channels, momentum=0.3, eps=1e-5).add_flags_recursive(fp32=True)
    return bn


def fc_with_initialize(input_channels, out_channels):
    return nn.Dense(input_channels, out_channels).add_flags_recursive(fp16=True)


class BNReshapeDenseBNNet(nn.Cell):
    def __init__(self):
        super(BNReshapeDenseBNNet, self).__init__()
        self.batch_norm = bn_with_initialize(2)
        self.reshape = P.Reshape()
        self.cast = P.Cast()
        self.batch_norm2 = nn.BatchNorm1d(512, affine=False)
        self.fc = fc_with_initialize(2 * 32 * 32, 512)

    def construct(self, x):
        x = self.batch_norm(x)
        x = self.reshape(x, (16, 2*32*32))
        x = self.fc(x)
        x = self.batch_norm2(x)
        return x


def test_bn_reshape_dense_bn_train():
    batch_size = 16
    device_num = 16
    context.set_auto_parallel_context(device_num=device_num, global_rank=0)
    input = Tensor(np.ones([batch_size, 2, 32, 32]).astype(np.float32) * 0.01)

    net = GradWrap(NetWithLoss(BNReshapeDenseBNNet()))
    context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
    
    _executor.compile(net, input)


class ParallelReduceMeanNet(nn.Cell):
    def __init__(self, conv_in_channel, conv_out_channel,
                 reducemean_keep_dims=False, reducemean_axis=-1, strategy=None):
        super().__init__()
        self.conv = nn.Conv2d(in_channels=conv_in_channel, out_channels=conv_out_channel,
                           kernel_size=1, stride=1, pad_mode='valid', has_bias=True,
                           weight_init='ones', bias_init='ones')
        self.reduce_mean = P.ReduceMean(keep_dims=reducemean_keep_dims)
        self.flat = nn.Flatten()
        self.reducemean_axis = reducemean_axis
        if strategy is not None:
            self.reduce_mean.set_strategy(strategy)

    def construct(self, inputs):
        x = self.conv(inputs)
        x = self.reduce_mean(x, self.reducemean_axis)
        x = self.flat(x)
        return x


class CrossEntropyLoss(nn.Cell):
    def __init__(self, reduction='mean'):
        super(CrossEntropyLoss, self).__init__()

        self.reduce_mean = P.ReduceMean()
        self.cross_entropy = SoftmaxCrossEntropyWithLogits()
        self.reduction = reduction

    def construct(self, logits, label):
        loss = self.cross_entropy(logits, label)
        if self.reduction == 'mean':
            loss = self.reduce_mean(loss, (-1,))
        return loss


def test_flatten_reshape(parallel_mode="auto_parallel"):
    batch_size = 16
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2
    context.reset_auto_parallel_context()
    context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8)
    net = ParallelReduceMeanNet(conv_in_channel=3, conv_out_channel=64, reducemean_axis=(2, 3), strategy=((4, 2, 1, 1),))
    loss = CrossEntropyLoss()
    predict = Tensor(np.ones([batch_size, 3, 32, 32]), dtype=ms.float32)
    label = Tensor(np.ones([batch_size, 64]), dtype=ms.float32)
    dataset = Dataset(predict, label, 2, input_num=2)

    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    model = Model(net, loss_fn = loss, optimizer=opt)
    model.train(epoch_size, dataset, dataset_sink_mode=False)


def test_flatten_reshape2(parallel_mode="auto_parallel"):
    batch_size = 16
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2
    context.reset_auto_parallel_context()
    context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8)
    set_algo_parameters(not_fully_use_devices=True)
    net = ParallelReduceMeanNet(conv_in_channel=3, conv_out_channel=64, reducemean_axis=(2, 3), strategy=((4, 1, 1, 1),))
    loss = CrossEntropyLoss()
    predict = Tensor(np.ones([batch_size, 3, 32, 32]), dtype=ms.float32)
    label = Tensor(np.ones([batch_size, 64]), dtype=ms.float32)
    dataset = Dataset(predict, label, 2, input_num=2)

    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    model = Model(net, loss_fn = loss, optimizer=opt)
    model.train(epoch_size, dataset, dataset_sink_mode=False)


class ParallelReshapeNet(nn.Cell):
    def __init__(self, dense_in_channel, dense_out_channel, shape, strategy=None):
        super().__init__()
        self.flat = nn.Flatten()
        self.dense = nn.Dense(in_channels=dense_in_channel,
                              out_channels=dense_out_channel,
                              weight_init='ones',
                              bias_init='ones',
                              has_bias=True)
        self.reshape = P.Reshape()
        self.shape = shape
        self.reshape.set_strategy(strategy)

    def construct(self, inputs):
        x = self.flat(inputs)
        x = self.dense(x)
        x = self.reshape(x, self.shape)
        return x


# the shape of input and output of reshape is the same
# reshape is optimized before step_parallel
def test_flatten_reshape3(parallel_mode="auto_parallel"):
    batch_size = 16
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2
    context.reset_auto_parallel_context()
    context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8)
    set_algo_parameters(not_fully_use_devices=True)
    net = ParallelReshapeNet(dense_in_channel=2048, dense_out_channel=1000, shape=(128, 1000), strategy=((16, 1),))
    loss = CrossEntropyLoss()
    predict = Tensor(np.ones([batch_size, 1, 2, 1024]), dtype=ms.float32)
    label = Tensor(np.ones([batch_size, 1000]), dtype=ms.float32)
    dataset = Dataset(predict, label, 2, input_num=2)

    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    model = Model(net, loss_fn = loss, optimizer=opt)
    model.train(epoch_size, dataset, dataset_sink_mode=False)


class CrossEntropyLoss2(nn.Cell):
    def __init__(self, reduction='mean'):
        super(CrossEntropyLoss2, self).__init__()
        self.cross_entropy = SoftmaxCrossEntropyWithLogits(reduction=reduction)

    def construct(self, logits, label):
        loss = self.cross_entropy(logits, label)
        return loss


def test_flatten_reshape4(parallel_mode="semi_auto_parallel"):
    batch_size = 16
    learning_rate = 0.1
    momentum = 0.9
    epoch_size = 2
    context.reset_auto_parallel_context()
    context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=8)
    set_algo_parameters(not_fully_use_devices=True)
    net = ParallelReduceMeanNet(conv_in_channel=3, conv_out_channel=64, reducemean_keep_dims=True, strategy=((4, 1, 1, 1),))
    loss = CrossEntropyLoss2()
    predict = Tensor(np.ones([batch_size, 3, 32, 32]), dtype=ms.float32)
    label = Tensor(np.ones([batch_size, 2048]), dtype=ms.float32)
    dataset = Dataset(predict, label, 2, input_num=2)

    opt = Momentum(net.trainable_params(), learning_rate, momentum)
    model = Model(net, loss_fn=loss, optimizer=opt)
    model.train(epoch_size, dataset, dataset_sink_mode=False)