Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 22a9c00bcd
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '22a9c00bcd'
${ noResults }
mindspore/tests/ut/python/parallel/test_allreduce_fusion.py

295 lines
13 kB
Raw Blame History 

  1. # Copyright 2020 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. 

  15. from mindspore.train import Model, ParallelMode

  16. from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits

  17. from mindspore.nn.optim.momentum import Momentum

  18. from mindspore import Tensor, context

  19. import mindspore as ms

  20. import numpy as np

  21. import mindspore.nn as nn

  22. from tests.dataset_mock import MindData

  23. from mindspore import context

  24. from mindspore.common.api import _executor

  25. from mindspore.parallel import _cost_model_context as cost_model_context

  26. 

  27. 

  28. 

  29. class Dataset(MindData):

  30.     def __init__(self, predict, label, length=3):

  31.         super(Dataset, self).__init__(size=length)

  32.         self.predict = predict

  33.         self.label = label

  34.         self.index = 0

  35.         self.length = length

  36. 

  37.     def __iter__(self):

  38.         return self

  39. 

  40.     def __next__(self):

  41.         if self.index >= self.length:

  42.             raise StopIteration

  43.         self.index += 1

  44.         return self.predict, self.label

  45. 

  46.     def reset(self):

  47.         self.index = 0

  48. 

  49. 

  50. class DenseNet1(nn.Cell):

  51.     def __init__(self, has_bias=True, activation='relu'):

  52.         super(DenseNet1, self).__init__()

  53.         self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  54.         self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  55.         self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  56.         self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  57. 

  58.     def construct(self, x):

  59.         q = self.fc1(x)

  60.         k = self.fc2(q)

  61.         v = self.fc3(k)

  62.         s = self.fc4(v)

  63.         return s

  64. 

  65. 

  66. class DenseNet2(nn.Cell):

  67.     def __init__(self, has_bias=True, activation='relu'):

  68.         super(DenseNet2, self).__init__()

  69.         self.fc1 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  70.         self.fc2 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  71.         self.fc3 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  72.         self.fc4 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  73.         self.fc5 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  74.         self.fc6 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  75.         self.fc7 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  76.         self.fc8 = nn.Dense(128, 128, has_bias=has_bias, activation=activation)

  77. 

  78.     def construct(self, x):

  79.         q = self.fc1(x)

  80.         k = self.fc2(q)

  81.         v = self.fc3(k)

  82.         s = self.fc4(v)

  83.         t = self.fc5(s)

  84.         u = self.fc6(t)

  85.         w = self.fc7(u)

  86.         z = self.fc8(w)

  87.         return z

  88. 

  89. class SimpleDMLNet(nn.Cell):

  90.     def __init__(self, net1, net2):

  91.         super(SimpleDMLNet, self).__init__()

  92.         self.backbone1 = net1

  93.         self.backbone2 = net2

  94. 

  95.     def construct(self, x):

  96.         x1 = self.backbone1(x)

  97.         x2 = self.backbone2(x)

  98.         return x1 + x2

  99. 

  100. 

  101. def train_common(net):

  102.     batch_size = 32

  103.     learning_rate = 0.1

  104.     momentum = 0.9

  105.     epoch_size = 2

  106.     device_num=4

  107.     context.reset_auto_parallel_context()

  108.     context.set_auto_parallel_context(parallel_mode=ParallelMode.SEMI_AUTO_PARALLEL, device_num=device_num, parameter_broadcast=False)

  109.     context.set_context(mode=context.GRAPH_MODE)

  110. 

  111.     predict = Tensor(np.ones([batch_size, 128]), dtype=ms.float32)

  112.     label = Tensor(np.ones([batch_size]), dtype=ms.int32)

  113.     dataset = Dataset(predict, label, 2)

  114. 

  115.     loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)

  116.     opt = Momentum(net.trainable_params(), learning_rate, momentum)

  117.     model = Model(net, loss, opt)

  118. 

  119.     model.train(epoch_size, dataset, dataset_sink_mode=False)

  120.     allreduce_fusion_dict = _executor._get_allreduce_fusion(model._train_network)

  121. 

  122.     print(allreduce_fusion_dict)

  123.     return allreduce_fusion_dict

  124. 

  125. 

  126. def test_allreduce_fusion_parameters():

  127.     cost_model_context.reset_cost_model_context()

  128.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=2)

  129.     algorithm = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_algorithm')

  130.     assert (algorithm == 2)

  131.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1)

  132.     algorithm = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_algorithm')

  133.     assert (algorithm == 1)

  134.     cost_model_context.reset_cost_model_context()

  135.     algorithm = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_algorithm')

  136.     assert (algorithm == 0)

  137. 

  138.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2)

  139.     fusion_times = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_times')

  140.     assert (fusion_times == 2)

  141. 

  142.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.2)

  143.     tail_percent = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_percent')

  144.     assert (tail_percent == 0.2)

  145.     cost_model_context.reset_cost_model_context()

  146.     tail_percent = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_percent')

  147.     assert (tail_percent == 0.1)

  148. 

  149.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_time=0.2)

  150.     tail_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_time')

  151.     assert (tail_time == 0.2)

  152.     cost_model_context.reset_cost_model_context()

  153.     tail_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_tail_time')

  154.     assert (tail_time == 0.1)

  155. 

  156.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_inherent_time=0.2)

  157.     allreduce_inherent_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_inherent_time')

  158.     assert (allreduce_inherent_time == 0.2)

  159.     cost_model_context.reset_cost_model_context()

  160.     allreduce_inherent_time = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_inherent_time')

  161.     assert (allreduce_inherent_time == 0.1)

  162. 

  163.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_bandwidth=0.2)

  164.     allreduce_bandwidth = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_bandwidth')

  165.     assert (allreduce_bandwidth == 0.2)

  166.     cost_model_context.reset_cost_model_context()

  167.     allreduce_bandwidth = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_allreduce_bandwidth')

  168.     assert (allreduce_bandwidth == 0.1)

  169. 

  170.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_computation_time_parameter=0.2)

  171.     computation_time_parameter = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_computation_time_parameter')

  172.     assert (computation_time_parameter == 0.2)

  173.     cost_model_context.reset_cost_model_context()

  174.     computation_time_parameter = cost_model_context.get_cost_model_context('costmodel_allreduce_fusion_computation_time_parameter')

  175.     assert (computation_time_parameter == 0.1)

  176. 

  177. 

  178. def test_allreduce_fusion1():

  179.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1)

  180.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2)

  181.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5)

  182.     net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))

  183.     allreduce_fusion_dict = train_common(net)

  184.     expect_dict = {'backbone2.fc8.weight': 2,

  185.                    'backbone2.fc7.weight': 2,

  186.                    'backbone2.fc6.weight': 2,

  187.                    'backbone1.fc4.weight': 2,

  188.                    'backbone1.fc3.weight': 2,

  189.                    'backbone1.fc2.weight': 2,

  190.                    'backbone2.fc5.weight': 1,

  191.                    'backbone2.fc4.weight': 1,

  192.                    'backbone2.fc3.weight': 1,

  193.                    'backbone2.fc2.weight': 1,

  194.                    'backbone2.fc1.weight': 1,

  195.                    'backbone1.fc1.weight': 1}

  196.     assert (allreduce_fusion_dict == expect_dict)

  197.     cost_model_context.reset_cost_model_context()

  198. 

  199. 

  200. # reset_cost_model_context is called, the default value of costmodel_allreduce_fusion_times is 0, step_allreduce_fusion

  201. # is bypassed.

  202. def test_allreduce_fusion2():

  203.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2)

  204.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5)

  205.     cost_model_context.reset_cost_model_context()

  206.     net = SimpleDMLNet(DenseNet1(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))

  207.     allreduce_fusion_dict = train_common(net)

  208.     expect_dict = {}

  209.     assert (allreduce_fusion_dict == expect_dict)

  210.     cost_model_context.reset_cost_model_context()

  211. 

  212. 

  213. def test_allreduce_fusion3():

  214.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1)

  215.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=3)

  216.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.3333333)

  217.     net = SimpleDMLNet(DenseNet1(has_bias=True, activation='relu'), DenseNet2(has_bias=False, activation='relu'))

  218.     allreduce_fusion_dict = train_common(net)

  219.     expect_dict = {'backbone2.fc8.weight': 3,

  220.                    'backbone2.fc7.weight': 3,

  221.                    'backbone2.fc6.weight': 2,

  222.                    'backbone2.fc5.weight': 2,

  223.                    'backbone2.fc4.weight': 2,

  224.                    'backbone2.fc3.weight': 1,

  225.                    'backbone2.fc2.weight': 1,

  226.                    'backbone2.fc1.weight': 1,

  227.                    'backbone1.fc4.bias': 3,

  228.                    'backbone1.fc4.weight': 3,

  229.                    'backbone1.fc3.bias': 3,

  230.                    'backbone1.fc3.weight': 2,

  231.                    'backbone1.fc2.bias': 2,

  232.                    'backbone1.fc2.weight': 2,

  233.                    'backbone1.fc1.bias': 2,

  234.                    'backbone1.fc1.weight': 2}

  235.     assert (allreduce_fusion_dict == expect_dict)

  236.     cost_model_context.reset_cost_model_context()

  237. 

  238. 

  239. def test_allreduce_fusion4():

  240.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=1)

  241.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_times=2)

  242.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_percent=0.5)

  243.     net = SimpleDMLNet(DenseNet2(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))

  244.     allreduce_fusion_dict = train_common(net)

  245.     expect_dict = {'backbone2.fc8.weight': 2,

  246.                    'backbone2.fc7.weight': 2,

  247.                    'backbone2.fc6.weight': 2,

  248.                    'backbone1.fc8.weight': 2,

  249.                    'backbone1.fc7.weight': 2,

  250.                    'backbone1.fc6.weight': 2,

  251.                    'backbone2.fc5.weight': 1,

  252.                    'backbone2.fc4.weight': 1,

  253.                    'backbone2.fc3.weight': 1,

  254.                    'backbone2.fc2.weight': 1,

  255.                    'backbone2.fc1.weight': 1,

  256.                    'backbone1.fc5.weight': 1,

  257.                    'backbone1.fc4.weight': 1,

  258.                    'backbone1.fc3.weight': 1,

  259.                    'backbone1.fc2.weight': 1,

  260.                    'backbone1.fc1.weight': 1}

  261. 

  262.     assert (allreduce_fusion_dict == expect_dict)

  263.     cost_model_context.reset_cost_model_context()

  264. 

  265. 

  266. def test_allreduce_fusion5():

  267.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_algorithm=2)

  268.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_tail_time=0.1)

  269.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_inherent_time=0.05)

  270.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_allreduce_bandwidth=0.000001)

  271.     cost_model_context.set_cost_model_context(costmodel_allreduce_fusion_computation_time_parameter=0.0000015)

  272.     net = SimpleDMLNet(DenseNet2(has_bias=False, activation=None), DenseNet2(has_bias=False, activation=None))

  273.     allreduce_fusion_dict = train_common(net)

  274. 

  275.     expect_dict = {'backbone2.fc8.weight': 3,

  276.                    'backbone2.fc7.weight': 3,

  277.                    'backbone2.fc6.weight': 3,

  278.                    'backbone2.fc5.weight': 2,

  279.                    'backbone2.fc4.weight': 2,

  280.                    'backbone2.fc3.weight': 2,

  281.                    'backbone2.fc2.weight': 1,

  282.                    'backbone2.fc1.weight': 1,

  283.                    'backbone1.fc8.weight': 3,

  284.                    'backbone1.fc7.weight': 3,

  285.                    'backbone1.fc6.weight': 3,

  286.                    'backbone1.fc5.weight': 2,

  287.                    'backbone1.fc4.weight': 2,

  288.                    'backbone1.fc3.weight': 2,

  289.                    'backbone1.fc2.weight': 1,

  290.                    'backbone1.fc1.weight': 1,}

  291. 

  292.     assert (allreduce_fusion_dict == expect_dict)

  293.     cost_model_context.reset_cost_model_context()