{"id":995,"date":"2023-08-09T23:05:53","date_gmt":"2023-08-09T23:05:53","guid":{"rendered":"https:\/\/www.gptmain.news\/?p=995"},"modified":"2023-08-09T23:05:53","modified_gmt":"2023-08-09T23:05:53","slug":"%d0%ba%d0%b0%d0%ba-%d1%80%d0%b0%d0%b1%d0%be%d1%82%d0%b0%d0%b5%d1%82-%d1%80%d0%b0%d1%81%d0%bf%d1%80%d0%b5%d0%b4%d0%b5%d0%bb%d0%b5%d0%bd%d0%bd%d0%be%d0%b5-%d0%be%d0%b1%d1%83%d1%87%d0%b5%d0%bd%d0%b8","status":"publish","type":"post","link":"https:\/\/gptmain.news\/?p=995","title":{"rendered":"\u041a\u0430\u043a \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442 \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0432 Pytorch: \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 \u043f\u0430\u0440\u0430\u043b\u043b\u0435\u043b\u044c\u043d\u044b\u043c\u0438 \u0434\u0430\u043d\u043d\u044b\u043c\u0438 \u0438 \u0441\u043c\u0435\u0448\u0430\u043d\u043d\u043e\u0439 \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c\u044e\n | GPTMain News"},"content":{"rendered":"
\n

\u0412 \u044d\u0442\u043e\u043c \u0443\u0440\u043e\u043a\u0435 \u043c\u044b \u043d\u0430\u0443\u0447\u0438\u043c\u0441\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c nn.parallel.DistributedDataParallel<\/code> \u0434\u043b\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043d\u0430\u0448\u0438\u0445 \u043c\u043e\u0434\u0435\u043b\u0435\u0439 \u043d\u0430 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u0445 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430\u0445. \u041c\u044b \u0432\u043e\u0437\u044c\u043c\u0435\u043c \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0439 \u043f\u0440\u0438\u043c\u0435\u0440 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0442\u043e\u0440\u0430 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439 \u0438 \u043f\u043e\u0441\u043c\u043e\u0442\u0440\u0438\u043c, \u043a\u0430\u043a \u043c\u044b \u043c\u043e\u0436\u0435\u043c \u0443\u0441\u043a\u043e\u0440\u0438\u0442\u044c \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435.<\/p>\n

\u041d\u0430\u0447\u043d\u0435\u043c \u0441 \u0438\u043c\u043f\u043e\u0440\u0442\u0430.<\/p>\n

import<\/span> torch<\/span><\/p>
import<\/span> torchvision<\/span><\/p>
import<\/span> torchvision<\/span>.<\/span>transforms <\/span>as<\/span> transforms<\/span><\/p>
import<\/span> torch<\/span>.<\/span>nn <\/span>as<\/span> nn<\/span><\/p>
import<\/span> torch<\/span>.<\/span>nn<\/span>.<\/span>functional <\/span>as<\/span> F<\/span><\/p>
import<\/span> torch<\/span>.<\/span>optim <\/span>as<\/span> optim<\/span><\/p>
import<\/span> time<\/span><\/p><\/pre>\n
\u041c\u044b \u0431\u0443\u0434\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c CIFAR10 \u0432\u043e \u0432\u0441\u0435\u0445 \u043d\u0430\u0448\u0438\u0445 \u044d\u043a\u0441\u043f\u0435\u0440\u0438\u043c\u0435\u043d\u0442\u0430\u0445 \u0441 \u0440\u0430\u0437\u043c\u0435\u0440\u043e\u043c \u043f\u0430\u043a\u0435\u0442\u0430 256.<\/p>\n
def<\/span> <\/span>create_data_loader_cifar10<\/span>(<\/span>)<\/span>:<\/span><\/p>
   transform <\/span>=<\/span> transforms<\/span>.<\/span>Compose<\/span>(<\/span><\/p>
       <\/span>[<\/span><\/p>
       transforms<\/span>.<\/span>RandomCrop<\/span>(<\/span>32<\/span>)<\/span>,<\/span><\/p>
       transforms<\/span>.<\/span>RandomHorizontalFlip<\/span>(<\/span>)<\/span>,<\/span><\/p>
       transforms<\/span>.<\/span>ToTensor<\/span>(<\/span>)<\/span>,<\/span><\/p>
       transforms<\/span>.<\/span>Normalize<\/span>(<\/span>(<\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>)<\/span>,<\/span> <\/span>(<\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>)<\/span>)<\/span>]<\/span>)<\/span><\/p>
   batch_size <\/span>=<\/span> <\/span>256<\/span><\/p>
   trainset <\/span>=<\/span> torchvision<\/span>.<\/span>datasets<\/span>.<\/span>CIFAR10<\/span>(<\/span>root<\/span>=<\/span>'.\/data'<\/span>,<\/span> train<\/span>=<\/span>True<\/span>,<\/span><\/p>
                                           download<\/span>=<\/span>True<\/span>,<\/span> transform<\/span>=<\/span>transform<\/span>)<\/span><\/p>
   trainloader <\/span>=<\/span> torch<\/span>.<\/span>utils<\/span>.<\/span>data<\/span>.<\/span>DataLoader<\/span>(<\/span>trainset<\/span>,<\/span> batch_size<\/span>=<\/span>batch_size<\/span>,<\/span><\/p>
                                           shuffle<\/span>=<\/span>True<\/span>,<\/span> num_workers<\/span>=<\/span>10<\/span>,<\/span> pin_memory<\/span>=<\/span>True<\/span>)<\/span><\/p>
   testset <\/span>=<\/span> torchvision<\/span>.<\/span>datasets<\/span>.<\/span>CIFAR10<\/span>(<\/span>root<\/span>=<\/span>'.\/data'<\/span>,<\/span> train<\/span>=<\/span>False<\/span>,<\/span><\/p>
                                       download<\/span>=<\/span>True<\/span>,<\/span> transform<\/span>=<\/span>transform<\/span>)<\/span><\/p>
   testloader <\/span>=<\/span> torch<\/span>.<\/span>utils<\/span>.<\/span>data<\/span>.<\/span>DataLoader<\/span>(<\/span>testset<\/span>,<\/span> batch_size<\/span>=<\/span>batch_size<\/span>,<\/span><\/p>
                                           shuffle<\/span>=<\/span>False<\/span>,<\/span> num_workers<\/span>=<\/span>10<\/span>)<\/span><\/p>
   <\/span>return<\/span> trainloader<\/span>,<\/span> testloader<\/span><\/p><\/pre>\n
\u0421\u043d\u0430\u0447\u0430\u043b\u0430 \u043c\u044b \u0431\u0443\u0434\u0435\u043c \u043e\u0431\u0443\u0447\u0430\u0442\u044c \u043c\u043e\u0434\u0435\u043b\u044c \u043d\u0430 \u043e\u0434\u043d\u043e\u043c \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u043c \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0435 Nvidia A100 \u0432 \u0442\u0435\u0447\u0435\u043d\u0438\u0435 1 \u044d\u043f\u043e\u0445\u0438.  \u0421\u0442\u0430\u043d\u0434\u0430\u0440\u0442\u043d\u044b\u0439 pytorch \u043c\u0430\u0442\u0435\u0440\u0438\u0430\u043b \u0437\u0434\u0435\u0441\u044c, \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u043e\u0432\u043e\u0433\u043e.  \u0423\u0447\u0435\u0431\u043d\u0438\u043a \u043e\u0441\u043d\u043e\u0432\u0430\u043d \u043d\u0430 \u043e\u0444\u0438\u0446\u0438\u0430\u043b\u044c\u043d\u043e\u043c \u0440\u0443\u043a\u043e\u0432\u043e\u0434\u0441\u0442\u0432\u0435 \u0438\u0437 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438 Pytorch.<\/p>\n
def<\/span> <\/span>train<\/span>(<\/span>net<\/span>,<\/span> trainloader<\/span>)<\/span>:<\/span><\/p>
   <\/span>print<\/span>(<\/span>\"Start training...\"<\/span>)<\/span><\/p>
   criterion <\/span>=<\/span> nn<\/span>.<\/span>CrossEntropyLoss<\/span>(<\/span>)<\/span><\/p>
   optimizer <\/span>=<\/span> optim<\/span>.<\/span>SGD<\/span>(<\/span>net<\/span>.<\/span>parameters<\/span>(<\/span>)<\/span>,<\/span> lr<\/span>=<\/span>0.001<\/span>,<\/span> momentum<\/span>=<\/span>0.9<\/span>)<\/span><\/p>
   epochs <\/span>=<\/span> <\/span>1<\/span><\/p>
   num_of_batches <\/span>=<\/span> <\/span>len<\/span>(<\/span>trainloader<\/span>)<\/span><\/p>
   <\/span>for<\/span> epoch <\/span>in<\/span> <\/span>range<\/span>(<\/span>epochs<\/span>)<\/span>:<\/span>  <\/span><\/p>
       running_loss <\/span>=<\/span> <\/span>0.0<\/span><\/p>
       <\/span>for<\/span> i<\/span>,<\/span> data <\/span>in<\/span> <\/span>enumerate<\/span>(<\/span>trainloader<\/span>,<\/span> <\/span>0<\/span>)<\/span>:<\/span><\/p>
           inputs<\/span>,<\/span> labels <\/span>=<\/span> data<\/span><\/p>
           images<\/span>,<\/span> labels <\/span>=<\/span> inputs<\/span>.<\/span>cuda<\/span>(<\/span>)<\/span>,<\/span> labels<\/span>.<\/span>cuda<\/span>(<\/span>)<\/span><\/p>
           optimizer<\/span>.<\/span>zero_grad<\/span>(<\/span>)<\/span><\/p>
           outputs <\/span>=<\/span> net<\/span>(<\/span>images<\/span>)<\/span><\/p>
           loss <\/span>=<\/span> criterion<\/span>(<\/span>outputs<\/span>,<\/span> labels<\/span>)<\/span><\/p>
           loss<\/span>.<\/span>backward<\/span>(<\/span>)<\/span><\/p>
           optimizer<\/span>.<\/span>step<\/span>(<\/span>)<\/span><\/p>
           running_loss <\/span>+=<\/span> loss<\/span>.<\/span>item<\/span>(<\/span>)<\/span><\/p>
       <\/span>print<\/span>(<\/span>f'[Epoch <\/span>{<\/span>epoch <\/span>+<\/span> <\/span>1<\/span>}<\/span>\/<\/span>{<\/span>epochs<\/span>}<\/span>] loss: <\/span>{<\/span>running_loss <\/span>\/<\/span> num_of_batches<\/span>:<\/span>.3f<\/span>}<\/span>'<\/span>)<\/span><\/p>
   <\/span>print<\/span>(<\/span>'Finished Training'<\/span>)<\/span><\/p><\/pre>\n
 test<\/code> \u0430\u043d\u0430\u043b\u043e\u0433\u0438\u0447\u043d\u043e \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u044f\u0435\u0442\u0441\u044f \u0444\u0443\u043d\u043a\u0446\u0438\u044f.  \u041e\u0441\u043d\u043e\u0432\u043d\u043e\u0439 \u0441\u043a\u0440\u0438\u043f\u0442 \u043f\u0440\u043e\u0441\u0442\u043e \u0441\u043e\u0431\u0435\u0440\u0435\u0442 \u0432\u0441\u0435 \u0432\u043c\u0435\u0441\u0442\u0435:<\/p>\n
if<\/span> __name__ <\/span>==<\/span> <\/span>'__main__'<\/span>:<\/span><\/p>
   start <\/span>=<\/span> time<\/span>.<\/span>time<\/span>(<\/span>)<\/span>  <\/span><\/p>
   PATH <\/span>=<\/span> <\/span>'.\/cifar_net.pth'<\/span><\/p>
   trainloader<\/span>,<\/span> testloader <\/span>=<\/span> create_data_loader_cifar10<\/span>(<\/span>)<\/span><\/p>
   net <\/span>=<\/span> torchvision<\/span>.<\/span>models<\/span>.<\/span>resnet50<\/span>(<\/span>False<\/span>)<\/span>.<\/span>cuda<\/span>(<\/span>)<\/span><\/p>
   start_train <\/span>=<\/span> time<\/span>.<\/span>time<\/span>(<\/span>)<\/span><\/p>
   train<\/span>(<\/span>net<\/span>,<\/span> trainloader<\/span>)<\/span><\/p>
   end_train <\/span>=<\/span> time<\/span>.<\/span>time<\/span>(<\/span>)<\/span><\/p>
   torch<\/span>.<\/span>save<\/span>(<\/span>net<\/span>.<\/span>state_dict<\/span>(<\/span>)<\/span>,<\/span> PATH<\/span>)<\/span><\/p>
   test<\/span>(<\/span>net<\/span>,<\/span> PATH<\/span>,<\/span> testloader<\/span>)<\/span><\/p>
   end <\/span>=<\/span> time<\/span>.<\/span>time<\/span>(<\/span>)<\/span><\/p>
   seconds <\/span>=<\/span> <\/span>(<\/span>end <\/span>-<\/span> start<\/span>)<\/span><\/p>
   seconds_train <\/span>=<\/span> <\/span>(<\/span>end_train <\/span>-<\/span> start_train<\/span>)<\/span><\/p>
   <\/span>print<\/span>(<\/span>f\"Total elapsed time<\/span>:<\/span> <\/span>{<\/span>seconds<\/span>:<\/span>.<\/span>2f<\/span>}<\/span> seconds<\/span>,<\/span> \\<\/span><\/p>
       Train <\/span>1<\/span> epoch <\/span>{<\/span>seconds_train<\/span>:<\/span>.<\/span>2f<\/span>}<\/span> seconds\"<\/span>)<\/span><\/p><\/pre>\n
\u041c\u044b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c resnet50 \u0434\u043b\u044f \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438 \u0441\u0435\u0442\u0438 \u043f\u0440\u0438\u043b\u0438\u0447\u043d\u043e\u0433\u043e \u0440\u0430\u0437\u043c\u0435\u0440\u0430.<\/p>\n
\u0422\u0435\u043f\u0435\u0440\u044c \u0434\u0430\u0432\u0430\u0439\u0442\u0435 \u043e\u0431\u0443\u0447\u0438\u043c \u043c\u043e\u0434\u0435\u043b\u044c:<\/p>\n
$ python -m train_1gpu<\/span><\/p><\/pre>\n
Accuracy of the network on the 10000 test images: 27 %<\/span><\/p>
Total elapsed time: 69.03 seconds,      Train 1 epoch 13.08 seconds<\/span><\/p><\/pre>\n
\u041b\u0430\u0434\u043d\u043e, \u043f\u043e\u0440\u0430 \u0437\u0430\u043d\u044f\u0442\u044c\u0441\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u0435\u0439.<\/p>\n
\n
\u041a\u043e\u0434 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u043d\u0430 GitHub.  \u0415\u0441\u043b\u0438 \u0432\u044b \u043f\u043b\u0430\u043d\u0438\u0440\u0443\u0435\u0442\u0435 \u0437\u0430\u043a\u0440\u0435\u043f\u0438\u0442\u044c \u0441\u0432\u043e\u0438 \u0437\u043d\u0430\u043d\u0438\u044f \u043e Pytorch, \u043c\u044b \u043d\u0430\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0443\u0435\u043c \u0434\u0432\u0435 \u0437\u0430\u043c\u0435\u0447\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043a\u043d\u0438\u0433\u0438: \u00ab\u0413\u043b\u0443\u0431\u043e\u043a\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 PyTorch\u00bb \u043e\u0442 Manning Publications \u0438 \u00ab\u041c\u0430\u0448\u0438\u043d\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 PyTorch \u0438 Scikit-Learn\u00bb \u0421\u0435\u0431\u0430\u0441\u0442\u044c\u044f\u043d\u0430 \u0420\u0430\u0448\u043a\u0438.  \u0412\u044b \u0432\u0441\u0435\u0433\u0434\u0430 \u043c\u043e\u0436\u0435\u0442\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434 \u0441\u043a\u0438\u0434\u043a\u0438 35% \u0431\u043b\u0435\u0439\u0441\u0430\u043c\u043c\u0435\u044021<\/strong> \u0434\u043b\u044f \u0432\u0441\u0435\u0439 \u043f\u0440\u043e\u0434\u0443\u043a\u0446\u0438\u0438 Manning’s. <\/p>\n<\/blockquote>\n
torch.nn.DataParallel: \u0431\u0435\u0437 \u0431\u043e\u043b\u0438, \u0431\u0435\u0437 \u0432\u044b\u0438\u0433\u0440\u044b\u0448\u0430<\/h2>\n
DataParallel \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u043e\u0434\u043d\u043e\u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043d\u044b\u043c, \u043c\u043d\u043e\u0433\u043e\u043f\u043e\u0442\u043e\u0447\u043d\u044b\u043c \u0438 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442 \u0442\u043e\u043b\u044c\u043a\u043e \u043d\u0430 \u043e\u0434\u043d\u043e\u0439 \u043c\u0430\u0448\u0438\u043d\u0435.  \u0414\u043b\u044f \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u0433\u043e \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430 \u043c\u044b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c \u043e\u0434\u043d\u0443 \u0438 \u0442\u0443 \u0436\u0435 \u043c\u043e\u0434\u0435\u043b\u044c \u0434\u043b\u044f \u043f\u0440\u044f\u043c\u043e\u0433\u043e \u043f\u0440\u043e\u0445\u043e\u0434\u0430.  \u041c\u044b \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u044f\u0435\u043c \u0434\u0430\u043d\u043d\u044b\u0435 \u043f\u043e \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430\u043c \u0438 \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u043c \u043f\u0440\u044f\u043c\u044b\u0435 \u043f\u0440\u043e\u0445\u043e\u0434\u044b \u0432 \u043a\u0430\u0436\u0434\u043e\u043c \u0438\u0437 \u043d\u0438\u0445.  \u041f\u043e \u0441\u0443\u0442\u0438, \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0442\u043e, \u0447\u0442\u043e \u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u0440\u0442\u0438\u0438 \u0434\u0435\u043b\u0438\u0442\u0441\u044f \u043d\u0430 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e \u0440\u0430\u0431\u043e\u0447\u0438\u0445.<\/p>\n
\u0412 \u044d\u0442\u043e\u043c \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u044d\u0442\u0430 \u0444\u0443\u043d\u043a\u0446\u0438\u043e\u043d\u0430\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u043d\u0435 \u0434\u0430\u0432\u0430\u043b\u0430 \u043d\u0438\u043a\u0430\u043a\u043e\u0439 \u0432\u044b\u0433\u043e\u0434\u044b.  \u042d\u0442\u043e \u043f\u043e\u0442\u043e\u043c\u0443, \u0447\u0442\u043e \u0441\u0438\u0441\u0442\u0435\u043c\u0430, \u043a\u043e\u0442\u043e\u0440\u0443\u044e \u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e, \u0438\u043c\u0435\u0435\u0442 \u0443\u0437\u043a\u0438\u0435 \u043c\u0435\u0441\u0442\u0430 \u0432 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0435 \u0438 \u0436\u0435\u0441\u0442\u043a\u043e\u043c \u0434\u0438\u0441\u043a\u0435.  \u0414\u0440\u0443\u0433\u0438\u0435 \u043c\u0430\u0448\u0438\u043d\u044b \u0441 \u043e\u0447\u0435\u043d\u044c \u0431\u044b\u0441\u0442\u0440\u044b\u043c\u0438 \u0434\u0438\u0441\u043a\u0430\u043c\u0438 \u0438 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u043e\u043c, \u043d\u043e \u0441 \u0442\u0440\u0443\u0434\u043e\u043c \u0441\u043f\u0440\u0430\u0432\u043b\u044f\u044e\u0449\u0438\u0435\u0441\u044f \u0441\u043e \u0441\u043a\u043e\u0440\u043e\u0441\u0442\u044c\u044e \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u0433\u043e \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430 (\u0443\u0437\u043a\u0438\u043c \u043c\u0435\u0441\u0442\u043e\u043c \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u0433\u043e \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430), \u043c\u043e\u0433\u0443\u0442 \u0438\u0437\u0432\u043b\u0435\u0447\u044c \u0432\u044b\u0433\u043e\u0434\u0443 \u0438\u0437 \u044d\u0442\u043e\u0439 \u0444\u0443\u043d\u043a\u0446\u0438\u0438.<\/p>\n
\u041d\u0430 \u043f\u0440\u0430\u043a\u0442\u0438\u043a\u0435 \u0435\u0434\u0438\u043d\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0435 \u0438\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u0435, \u043a\u043e\u0442\u043e\u0440\u043e\u0435 \u0432\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u0441\u0434\u0435\u043b\u0430\u0442\u044c \u0432 \u043a\u043e\u0434\u0435, \u044d\u0442\u043e \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0435:<\/p>\n
net <\/span>=<\/span> torchvision<\/span>.<\/span>models<\/span>.<\/span>resnet50<\/span>(<\/span>False<\/span>)<\/span><\/p>
if<\/span> torch<\/span>.<\/span>cuda<\/span>.<\/span>device_count<\/span>(<\/span>)<\/span> <\/span>><\/span> <\/span>1<\/span>:<\/span><\/p>
    <\/span>print<\/span>(<\/span>\"Let's use\"<\/span>,<\/span> torch<\/span>.<\/span>cuda<\/span>.<\/span>device_count<\/span>(<\/span>)<\/span>,<\/span> <\/span>\"GPUs!\"<\/span>)<\/span><\/p>
    net <\/span>=<\/span> nn<\/span>.<\/span>DataParallel<\/span>(<\/span>net<\/span>)<\/span><\/p><\/pre>\n
\u041a\u043e\u0433\u0434\u0430 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0448\u044c nn.DataParallel<\/code>\u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u0440\u0442\u0438\u0438 \u0434\u043e\u043b\u0436\u0435\u043d \u0431\u044b\u0442\u044c \u0434\u0435\u043b\u0438\u043c\u044b\u0439<\/strong> \u043f\u043e \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u0443 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u043e\u0432.<\/p>\n
\n
nn.DataParallel<\/code>  \u0440\u0430\u0437\u0434\u0435\u043b\u044f\u0435\u0442 \u043f\u0430\u043a\u0435\u0442 \u0438 \u043e\u0431\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u0435\u0442 \u0435\u0433\u043e \u043d\u0435\u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e \u043d\u0430 \u0432\u0441\u0435\u0445 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b\u0445 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430\u0445.  \u041f\u0440\u0438 \u043a\u0430\u0436\u0434\u043e\u043c \u043f\u0440\u044f\u043c\u043e\u043c \u043f\u0440\u043e\u0445\u043e\u0434\u0435 \u043c\u043e\u0434\u0443\u043b\u044c \u0440\u0435\u043f\u043b\u0438\u0446\u0438\u0440\u0443\u0435\u0442\u0441\u044f \u043d\u0430 \u043a\u0430\u0436\u0434\u043e\u043c \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u043c \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0435, \u0447\u0442\u043e \u043f\u0440\u0438\u0432\u043e\u0434\u0438\u0442 \u043a \u0437\u043d\u0430\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u043d\u0430\u043a\u043b\u0430\u0434\u043d\u044b\u043c \u0440\u0430\u0441\u0445\u043e\u0434\u0430\u043c.  \u041a\u0430\u0436\u0434\u0430\u044f \u0440\u0435\u043f\u043b\u0438\u043a\u0430 \u043e\u0431\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u0435\u0442 \u0447\u0430\u0441\u0442\u044c \u043f\u0430\u043a\u0435\u0442\u0430 (batch_size\/gpus).  \u0412\u043e \u0432\u0440\u0435\u043c\u044f \u043e\u0431\u0440\u0430\u0442\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0445\u043e\u0434\u0430 \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u044b \u043e\u0442 \u043a\u0430\u0436\u0434\u043e\u0439 \u0440\u0435\u043f\u043b\u0438\u043a\u0438 \u0441\u0443\u043c\u043c\u0438\u0440\u0443\u044e\u0442\u0441\u044f \u0432 \u0438\u0441\u0445\u043e\u0434\u043d\u044b\u0439 \u043c\u043e\u0434\u0443\u043b\u044c.<\/p>\n<\/blockquote>\n
\u0414\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u044f \u0432 \u043d\u0430\u0448\u0435\u0439 \u043f\u0440\u0435\u0434\u044b\u0434\u0443\u0449\u0435\u0439 \u0441\u0442\u0430\u0442\u044c\u0435 \u043e \u043f\u0430\u0440\u0430\u043b\u043b\u0435\u043b\u0438\u0437\u043c\u0435 \u0434\u0430\u043d\u043d\u044b\u0445 \u0438 \u043c\u043e\u0434\u0435\u043b\u0438.<\/p>\n
\u0425\u043e\u0440\u043e\u0448\u0435\u0439 \u043f\u0440\u0430\u043a\u0442\u0438\u043a\u043e\u0439 \u043f\u0440\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0438 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u0445 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u043e\u0432 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0437\u0430\u0440\u0430\u043d\u0435\u0435 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u0442\u044c \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u0435 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u044b, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0432\u0430\u0448 \u0441\u043a\u0440\u0438\u043f\u0442:<\/p>\n
import os<\/span><\/p>
os.environ['CUDA_VISIBLE_DEVICES'] = \"0,1\"<\/span><\/p><\/pre>\n
\u042d\u0442\u043e \u0434\u043e\u043b\u0436\u043d\u043e \u0431\u044b\u0442\u044c \u0421\u0414\u0415\u041b\u0410\u041d\u041e \u0434\u043e<\/strong> \u043b\u044e\u0431\u043e\u0439 \u0434\u0440\u0443\u0433\u043e\u0439 \u0438\u043c\u043f\u043e\u0440\u0442, \u0441\u0432\u044f\u0437\u0430\u043d\u043d\u044b\u0439 \u0441 CUDA.<\/p>\n
\u0414\u0430\u0436\u0435 \u0438\u0437 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438 Pytorch \u0432\u0438\u0434\u043d\u043e, \u0447\u0442\u043e \u044d\u0442\u043e \u043e\u0447\u0435\u043d\u044c \u043f\u043b\u043e\u0445\u0430\u044f \u0441\u0442\u0440\u0430\u0442\u0435\u0433\u0438\u044f:<\/p>\n
\n
\u0420\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0443\u0435\u0442\u0441\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c nn.DistributedDataParallel<\/code>\u0432\u043c\u0435\u0441\u0442\u043e \u044d\u0442\u043e\u0433\u043e \u043a\u043b\u0430\u0441\u0441\u0430 \u043f\u0440\u043e\u0432\u043e\u0434\u0438\u0442\u044c \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u043c\u0438 GPU, \u0434\u0430\u0436\u0435 \u0435\u0441\u043b\u0438 \u0435\u0441\u0442\u044c \u0442\u043e\u043b\u044c\u043a\u043e \u043e\u0434\u0438\u043d \u0443\u0437\u0435\u043b. <\/p>\n<\/blockquote>\n
\u041f\u0440\u0438\u0447\u0438\u043d\u0430 \u0432 \u0442\u043e\u043c, \u0447\u0442\u043e DistributedDataParallel \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442 \u043e\u0434\u0438\u043d \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u043d\u0430 \u043e\u0434\u043d\u043e\u0433\u043e \u0440\u0430\u0431\u043e\u0442\u043d\u0438\u043a\u0430<\/strong> (GPU), \u0430 DataParallel \u0438\u043d\u043a\u0430\u043f\u0441\u0443\u043b\u0438\u0440\u0443\u0435\u0442 \u0432\u0441\u044e \u043f\u0435\u0440\u0435\u0434\u0430\u0447\u0443 \u0434\u0430\u043d\u043d\u044b\u0445 \u0432 \u043e\u0434\u0438\u043d \u043f\u0440\u043e\u0446\u0435\u0441\u0441.<\/p>\n
\u0421\u043e\u0433\u043b\u0430\u0441\u043d\u043e \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u043c, \u0434\u0430\u043d\u043d\u044b\u0435 \u043c\u043e\u0433\u0443\u0442 \u043d\u0430\u0445\u043e\u0434\u0438\u0442\u044c\u0441\u044f \u043d\u0430 \u043b\u044e\u0431\u043e\u043c \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0435 \u0434\u043e \u0442\u043e\u0433\u043e, \u043a\u0430\u043a \u043e\u043d\u0438 \u0431\u0443\u0434\u0443\u0442 \u043f\u0435\u0440\u0435\u0434\u0430\u043d\u044b \u0432 \u043c\u043e\u0434\u0435\u043b\u044c.<\/p>\n
\u0412 \u043c\u043e\u0435\u043c \u044d\u043a\u0441\u043f\u0435\u0440\u0438\u043c\u0435\u043d\u0442\u0435 DataParallel \u0431\u044b\u043b \u043f\u043e\u043c\u0435\u0434\u043b\u0435\u043d\u043d\u0435\u0435<\/strong> \u0447\u0435\u043c \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u043d\u0430 \u043e\u0434\u043d\u043e\u043c \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u043c \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0435.  \u0414\u0430\u0436\u0435 \u0441 4 GPU.  \u041f\u043e\u0441\u043b\u0435 \u0443\u0432\u0435\u043b\u0438\u0447\u0435\u043d\u0438\u044f \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u0430 \u0440\u0430\u0431\u043e\u0447\u0438\u0445 \u044f \u0443\u043c\u0435\u043d\u044c\u0448\u0438\u043b \u0432\u0440\u0435\u043c\u044f, \u043d\u043e \u0432\u0441\u0435 \u0440\u0430\u0432\u043d\u043e \u0445\u0443\u0436\u0435, \u0447\u0435\u043c \u043e\u0434\u0438\u043d\u043e\u0447\u043d\u044b\u0439 GPU.  \u042f \u0438\u0437\u043c\u0435\u0440\u044f\u044e \u0438 \u0441\u043e\u043e\u0431\u0449\u0430\u044e \u0432\u0440\u0435\u043c\u044f, \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e\u0435 \u0434\u043b\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043c\u043e\u0434\u0435\u043b\u0438 \u0434\u043b\u044f \u043e\u0434\u043d\u043e\u0439 \u044d\u043f\u043e\u0445\u0438, \u0442\u043e \u0435\u0441\u0442\u044c 50\u00a0000 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439 32×32. <\/p>\n
\n
\u041f\u043e\u0441\u043b\u0435\u0434\u043d\u0435\u0435 \u0437\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435: \u0447\u0442\u043e\u0431\u044b \u0441\u0440\u0430\u0432\u043d\u0438\u0442\u044c \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0441 \u043e\u0434\u043d\u0438\u043c GPU, \u044f \u0443\u043c\u043d\u043e\u0436\u0438\u043b \u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u043a\u0435\u0442\u0430 \u043d\u0430 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e \u0440\u0430\u0431\u043e\u0447\u0438\u0445, \u0442\u043e \u0435\u0441\u0442\u044c 4 \u0434\u043b\u044f 4 GPU.  \u0412 \u043f\u0440\u043e\u0442\u0438\u0432\u043d\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435 \u044d\u0442\u043e \u0431\u043e\u043b\u0435\u0435 \u0447\u0435\u043c \u0432 2 \u0440\u0430\u0437\u0430 \u043c\u0435\u0434\u043b\u0435\u043d\u043d\u0435\u0435.<\/p>\n<\/blockquote>\n
\u042d\u0442\u043e \u043f\u043e\u0434\u0432\u043e\u0434\u0438\u0442 \u043d\u0430\u0441 \u043a \u0436\u0435\u0441\u0442\u043a\u043e\u0439 \u0442\u0435\u043c\u0435 \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0445 \u043f\u0430\u0440\u0430\u043b\u043b\u0435\u043b\u044c\u043d\u044b\u0445 \u0434\u0430\u043d\u043d\u044b\u0445.<\/p>\n
\n
\u041a\u043e\u0434 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u043d\u0430 GitHub.  \u0412\u044b \u0432\u0441\u0435\u0433\u0434\u0430 \u043c\u043e\u0436\u0435\u0442\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0430\u0442\u044c \u043d\u0430\u0448\u0443 \u0440\u0430\u0431\u043e\u0442\u0443, \u043f\u043e\u0434\u0435\u043b\u0438\u0432\u0448\u0438\u0441\u044c \u0432 \u0441\u043e\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u0435\u0442\u044f\u0445, \u0441\u0434\u0435\u043b\u0430\u0432 \u043f\u043e\u0436\u0435\u0440\u0442\u0432\u043e\u0432\u0430\u043d\u0438\u0435 \u0438 \u043a\u0443\u043f\u0438\u0432 \u043d\u0430\u0448\u0443 \u043a\u043d\u0438\u0433\u0443 \u0438 \u044d\u043b\u0435\u043a\u0442\u0440\u043e\u043d\u043d\u044b\u0439 \u043a\u0443\u0440\u0441.<\/p>\n<\/blockquote>\n
\u0420\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u0430\u044f \u043f\u0430\u0440\u0430\u043b\u043b\u0435\u043b\u044c\u043d\u0430\u044f \u043f\u0435\u0440\u0435\u0434\u0430\u0447\u0430 \u0434\u0430\u043d\u043d\u044b\u0445 Pytorch<\/h2>\n
\u0420\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u0430\u044f \u043f\u0430\u0440\u0430\u043b\u043b\u0435\u043b\u044c\u043d\u0430\u044f \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0430 \u0434\u0430\u043d\u043d\u044b\u0445 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u043c\u043d\u043e\u0433\u043e\u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u043d\u043e\u0439 \u0438 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442 \u043a\u0430\u043a \u0434\u043b\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043d\u0430 \u043e\u0434\u043d\u043e\u0439, \u0442\u0430\u043a \u0438 \u043d\u0430 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u0445 \u043c\u0430\u0448\u0438\u043d\u0430\u0445.  \u0432 \u043f\u0438\u0442\u043e\u0440\u043a\u0435, nn.parallel.DistributedDataParallel<\/code> \u0440\u0430\u0441\u043f\u0430\u0440\u0430\u043b\u043b\u0435\u043b\u0438\u0432\u0430\u0435\u0442 \u043c\u043e\u0434\u0443\u043b\u044c, \u0440\u0430\u0437\u0434\u0435\u043b\u044f\u044f \u0432\u0432\u043e\u0434 \u043c\u0435\u0436\u0434\u0443 \u0443\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u043c\u0438 \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0430\u043c\u0438.  \u042d\u0442\u043e\u0442 \u043c\u043e\u0434\u0443\u043b\u044c \u0442\u0430\u043a\u0436\u0435 \u043f\u043e\u0434\u0445\u043e\u0434\u0438\u0442 \u0434\u043b\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0440\u0430\u0431\u043e\u0442\u0435 \u0441 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u043c\u0438 \u0443\u0437\u043b\u0430\u043c\u0438 \u0438 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u043c\u0438 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c\u0438 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430\u043c\u0438.  \u0417\u0434\u0435\u0441\u044c \u044f \u044d\u043a\u0441\u043f\u0435\u0440\u0438\u043c\u0435\u043d\u0442\u0438\u0440\u043e\u0432\u0430\u043b \u0442\u043e\u043b\u044c\u043a\u043e \u0441 \u043e\u0434\u043d\u0438\u043c \u0443\u0437\u043b\u043e\u043c (1 \u043c\u0430\u0448\u0438\u043d\u0430 \u0441 4 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c\u0438 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0430\u043c\u0438).<\/p>\n
\u041e\u0441\u043d\u043e\u0432\u043d\u043e\u0435 \u043e\u0442\u043b\u0438\u0447\u0438\u0435 \u0437\u0434\u0435\u0441\u044c \u0432 \u0442\u043e\u043c, \u0447\u0442\u043e \u043a\u0430\u0436\u0434\u044b\u0439 GPU \u043e\u0431\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u0435\u0442\u0441\u044f \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u043c.  \u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u043d\u0438\u043a\u043e\u0433\u0434\u0430 \u043d\u0435 \u043f\u0435\u0440\u0435\u0434\u0430\u044e\u0442\u0441\u044f \u043c\u0435\u0436\u0434\u0443 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430\u043c\u0438, \u0442\u043e\u043b\u044c\u043a\u043e \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u044b.<\/p>\n
\n
\u041c\u043e\u0434\u0443\u043b\u044c \u0440\u0435\u043f\u043b\u0438\u0446\u0438\u0440\u0443\u0435\u0442\u0441\u044f \u043d\u0430 \u043a\u0430\u0436\u0434\u0443\u044e \u043c\u0430\u0448\u0438\u043d\u0443 \u0438 \u043a\u0430\u0436\u0434\u043e\u0435 \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u043e.  \u0412\u043e \u0432\u0440\u0435\u043c\u044f \u043f\u0440\u044f\u043c\u043e\u0433\u043e \u043f\u0440\u043e\u0445\u043e\u0434\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0440\u0430\u0431\u043e\u0447\u0438\u0439 \u043f\u0440\u043e\u0446\u0435\u0441\u0441 (GPU) \u043e\u0431\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u0435\u0442 \u0434\u0430\u043d\u043d\u044b\u0435 \u0438 \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e \u0432\u044b\u0447\u0438\u0441\u043b\u044f\u0435\u0442 \u0441\u043e\u0431\u0441\u0442\u0432\u0435\u043d\u043d\u044b\u0439 \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442.  \u0412\u043e \u0432\u0440\u0435\u043c\u044f \u043e\u0431\u0440\u0430\u0442\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0445\u043e\u0434\u0430 \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u044b \u043e\u0442 \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0443\u0437\u043b\u0430 \u0443\u0441\u0440\u0435\u0434\u043d\u044f\u044e\u0442\u0441\u044f.  \u041d\u0430\u043a\u043e\u043d\u0435\u0446, \u043a\u0430\u0436\u0434\u044b\u0439 \u0440\u0430\u0431\u043e\u0447\u0438\u0439 \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u0430 \u0438 \u043e\u0442\u043f\u0440\u0430\u0432\u043b\u044f\u0435\u0442 \u0432\u0441\u0435\u043c \u043e\u0441\u0442\u0430\u043b\u044c\u043d\u044b\u043c \u0443\u0437\u043b\u0430\u043c \u0432\u044b\u0447\u0438\u0441\u043b\u0435\u043d\u043d\u043e\u0435 \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u0430.<\/p>\n<\/blockquote>\n
\u041c\u043e\u0434\u0443\u043b\u044c \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 \u0448\u0430\u0433 all-reduce \u0434\u043b\u044f \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u043e\u0432 \u0438 \u043f\u0440\u0435\u0434\u043f\u043e\u043b\u0430\u0433\u0430\u0435\u0442, \u0447\u0442\u043e \u043e\u043d\u0438 \u0431\u0443\u0434\u0443\u0442 \u0438\u0437\u043c\u0435\u043d\u0435\u043d\u044b \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0442\u043e\u0440\u043e\u043c \u0432\u043e \u0432\u0441\u0435\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430\u0445 \u043e\u0434\u0438\u043d\u0430\u043a\u043e\u0432\u044b\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c.<\/p>\n
\u041d\u0438\u0436\u0435 \u043f\u0440\u0438\u0432\u0435\u0434\u0435\u043d\u044b \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0430\u0446\u0438\u0438 \u043f\u043e \u043f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u043e\u0432\u0430\u043d\u0438\u044e \u0432\u0430\u0448\u0435\u0433\u043e \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u044f \u0441 \u043e\u0434\u043d\u0438\u043c GPU \u0432 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u043c\u0438 GPU.<\/p>\n
\u0428\u0430\u0433 1: \u0418\u043d\u0438\u0446\u0438\u0430\u043b\u0438\u0437\u0438\u0440\u0443\u0439\u0442\u0435 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u044b \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f<\/h3>\n
def<\/span> <\/span>init_distributed<\/span>(<\/span>)<\/span>:<\/span><\/p>
    dist_url <\/span>=<\/span> <\/span>\"env:\/\/\"<\/span> <\/span><\/p>
    rank <\/span>=<\/span> <\/span>int<\/span>(<\/span>os<\/span>.<\/span>environ<\/span>[<\/span>\"RANK\"<\/span>]<\/span>)<\/span><\/p>
    world_size <\/span>=<\/span> <\/span>int<\/span>(<\/span>os<\/span>.<\/span>environ<\/span>[<\/span>'WORLD_SIZE'<\/span>]<\/span>)<\/span><\/p>
    local_rank <\/span>=<\/span> <\/span>int<\/span>(<\/span>os<\/span>.<\/span>environ<\/span>[<\/span>'LOCAL_RANK'<\/span>]<\/span>)<\/span><\/p>
    dist<\/span>.<\/span>init_process_group<\/span>(<\/span><\/p>
            backend<\/span>=<\/span>\"nccl\"<\/span>,<\/span><\/p>
            init_method<\/span>=<\/span>dist_url<\/span>,<\/span><\/p>
            world_size<\/span>=<\/span>world_size<\/span>,<\/span><\/p>
            rank<\/span>=<\/span>rank<\/span>)<\/span><\/p>
    torch<\/span>.<\/span>cuda<\/span>.<\/span>set_device<\/span>(<\/span>local_rank<\/span>)<\/span><\/p>
    dist<\/span>.<\/span>barrier<\/span>(<\/span>)<\/span><\/p><\/pre>\n
\u042d\u0442\u0430 \u0438\u043d\u0438\u0446\u0438\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442, \u043a\u043e\u0433\u0434\u0430 \u043c\u044b \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u043c \u043d\u0430\u0448 \u0441\u043a\u0440\u0438\u043f\u0442 \u0441 torch.distributed.launch<\/code> (\u041f\u0438\u0442\u043e\u0440\u0447 1.7 \u0438 1.8) \u0438\u043b\u0438  torch.run<\/code> (Pytorch 1.9+) \u0441 \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0443\u0437\u043b\u0430 (\u0437\u0434\u0435\u0441\u044c 1).<\/p>\n
\u0428\u0430\u0433 2: \u041e\u0431\u0435\u0440\u043d\u0438\u0442\u0435 \u043c\u043e\u0434\u0435\u043b\u044c \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e DDP<\/h3>\n
net <\/span>=<\/span> torchvision<\/span>.<\/span>models<\/span>.<\/span>resnet50<\/span>(<\/span>False<\/span>)<\/span>.<\/span>cuda<\/span>(<\/span>)<\/span><\/p>
net <\/span>=<\/span> nn<\/span>.<\/span>SyncBatchNorm<\/span>.<\/span>convert_sync_batchnorm<\/span>(<\/span>net<\/span>)<\/span><\/p>
local_rank <\/span>=<\/span> <\/span>int<\/span>(<\/span>os<\/span>.<\/span>environ<\/span>[<\/span>'LOCAL_RANK'<\/span>]<\/span>)<\/span><\/p>
net <\/span>=<\/span> nn<\/span>.<\/span>parallel<\/span>.<\/span>DistributedDataParallel<\/span>(<\/span>net<\/span>,<\/span> device_ids<\/span>=<\/span>[<\/span>local_rank<\/span>]<\/span>)<\/span><\/p><\/pre>\n
\u0415\u0441\u043b\u0438 \u043a\u0430\u0436\u0434\u044b\u0439 \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u044b\u0439 \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0430\u043d\u0433, tensor.cuda()<\/code> \u0438\u043b\u0438 model.cuda()<\/code> \u043c\u043e\u0436\u0435\u0442 \u0431\u044b\u0442\u044c \u0432\u044b\u0437\u0432\u0430\u043d \u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e \u0432\u043e \u0432\u0441\u0435\u043c \u0441\u043a\u0440\u0438\u043f\u0442\u0435.<\/p>\n
\u0428\u0430\u0433 3: \u0418\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0439\u0442\u0435 DistributedSampler \u0432 \u0432\u0430\u0448\u0435\u043c DataLoader<\/h3>\n
import<\/span> torch<\/span><\/p>
from<\/span> torch<\/span>.<\/span>utils<\/span>.<\/span>data<\/span>.<\/span>distributed <\/span>import<\/span> DistributedSampler<\/span><\/p>
from<\/span> torch<\/span>.<\/span>utils<\/span>.<\/span>data <\/span>import<\/span> DataLoader<\/span><\/p>
import<\/span> torch<\/span>.<\/span>nn <\/span>as<\/span> nn<\/span><\/p>
def<\/span> <\/span>create_data_loader_cifar10<\/span>(<\/span>)<\/span>:<\/span><\/p>
    transform <\/span>=<\/span> transforms<\/span>.<\/span>Compose<\/span>(<\/span><\/p>
        <\/span>[<\/span><\/p>
        transforms<\/span>.<\/span>RandomCrop<\/span>(<\/span>32<\/span>)<\/span>,<\/span><\/p>
        transforms<\/span>.<\/span>RandomHorizontalFlip<\/span>(<\/span>)<\/span>,<\/span><\/p>
        transforms<\/span>.<\/span>ToTensor<\/span>(<\/span>)<\/span>,<\/span><\/p>
        transforms<\/span>.<\/span>Normalize<\/span>(<\/span>(<\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>)<\/span>,<\/span> <\/span>(<\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>,<\/span> <\/span>0.5<\/span>)<\/span>)<\/span>]<\/span>)<\/span><\/p>
    batch_size <\/span>=<\/span> <\/span>256<\/span><\/p>
    trainset <\/span>=<\/span> torchvision<\/span>.<\/span>datasets<\/span>.<\/span>CIFAR10<\/span>(<\/span>root<\/span>=<\/span>'.\/data'<\/span>,<\/span> train<\/span>=<\/span>True<\/span>,<\/span><\/p>
                                            download<\/span>=<\/span>True<\/span>,<\/span> transform<\/span>=<\/span>transform<\/span>)<\/span>                                  <\/span><\/p>
    train_sampler <\/span>=<\/span> DistributedSampler<\/span>(<\/span>dataset<\/span>=<\/span>trainset<\/span>,<\/span> shuffle<\/span>=<\/span>True<\/span>)<\/span>                                                  <\/span><\/p>
    trainloader <\/span>=<\/span> torch<\/span>.<\/span>utils<\/span>.<\/span>data<\/span>.<\/span>DataLoader<\/span>(<\/span>trainset<\/span>,<\/span> batch_size<\/span>=<\/span>batch_size<\/span>,<\/span><\/p>
                                            sampler<\/span>=<\/span>train_sampler<\/span>,<\/span> num_workers<\/span>=<\/span>10<\/span>,<\/span> pin_memory<\/span>=<\/span>True<\/span>)<\/span><\/p>
    testset <\/span>=<\/span> torchvision<\/span>.<\/span>datasets<\/span>.<\/span>CIFAR10<\/span>(<\/span>root<\/span>=<\/span>'.\/data'<\/span>,<\/span> train<\/span>=<\/span>False<\/span>,<\/span><\/p>
                                        download<\/span>=<\/span>True<\/span>,<\/span> transform<\/span>=<\/span>transform<\/span>)<\/span><\/p>
    test_sampler <\/span>=<\/span>DistributedSampler<\/span>(<\/span>dataset<\/span>=<\/span>testset<\/span>,<\/span> shuffle<\/span>=<\/span>True<\/span>)<\/span>                                         <\/span><\/p>
    testloader <\/span>=<\/span> torch<\/span>.<\/span>utils<\/span>.<\/span>data<\/span>.<\/span>DataLoader<\/span>(<\/span>testset<\/span>,<\/span> batch_size<\/span>=<\/span>batch_size<\/span>,<\/span><\/p>
                                            shuffle<\/span>=<\/span>False<\/span>,<\/span> sampler<\/span>=<\/span>test_sampler<\/span>,<\/span> num_workers<\/span>=<\/span>10<\/span>)<\/span><\/p>
    <\/span>return<\/span> trainloader<\/span>,<\/span> testloader<\/span><\/p><\/pre>\n
\u0412 \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u043e\u043c \u0440\u0435\u0436\u0438\u043c\u0435 \u0432\u044b\u0437\u043e\u0432 data_loader.sampler.set_epoch()<\/code> \u043c\u0435\u0442\u043e\u0434 \u0432 \u043d\u0430\u0447\u0430\u043b\u0435 \u043a\u0430\u0436\u0434\u043e\u0439 \u044d\u043f\u043e\u0445\u0438 \u0434\u043e<\/strong> \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 DataLoader<\/code> \u0438\u0442\u0435\u0440\u0430\u0442\u043e\u0440 \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c \u0434\u043b\u044f \u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e\u0439 \u0440\u0430\u0431\u043e\u0442\u044b \u043f\u0435\u0440\u0435\u0442\u0430\u0441\u043e\u0432\u043a\u0438 \u0432 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u0445 \u044d\u043f\u043e\u0445\u0430\u0445.  \u0412 \u043f\u0440\u043e\u0442\u0438\u0432\u043d\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435 \u0432\u0441\u0435\u0433\u0434\u0430 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f \u043e\u0434\u0438\u043d \u0438 \u0442\u043e\u0442 \u0436\u0435 \u043f\u043e\u0440\u044f\u0434\u043e\u043a. <\/p>\n
def<\/span> <\/span>train<\/span>(<\/span>net<\/span>,<\/span> trainloader<\/span>)<\/span>:<\/span><\/p>
    <\/span>print<\/span>(<\/span>\"Start training...\"<\/span>)<\/span><\/p>
    criterion <\/span>=<\/span> nn<\/span>.<\/span>CrossEntropyLoss<\/span>(<\/span>)<\/span><\/p>
    optimizer <\/span>=<\/span> optim<\/span>.<\/span>SGD<\/span>(<\/span>net<\/span>.<\/span>parameters<\/span>(<\/span>)<\/span>,<\/span> lr<\/span>=<\/span>0.001<\/span>,<\/span> momentum<\/span>=<\/span>0.9<\/span>)<\/span><\/p>
    epochs <\/span>=<\/span> <\/span>1<\/span><\/p>
    num_of_batches <\/span>=<\/span> <\/span>len<\/span>(<\/span>trainloader<\/span>)<\/span><\/p>
    <\/span>for<\/span> epoch <\/span>in<\/span> <\/span>range<\/span>(<\/span>epochs<\/span>)<\/span>:<\/span> <\/span><\/p>
        trainloader<\/span>.<\/span>sampler<\/span>.<\/span>set_epoch<\/span>(<\/span>epoch<\/span>)<\/span><\/p><\/pre>\n
\u0412 \u0431\u043e\u043b\u0435\u0435 \u043e\u0431\u0449\u0435\u043c \u0432\u0438\u0434\u0435:<\/p>\n
for<\/span> epoch <\/span>in<\/span> <\/span>range<\/span>(<\/span>epochs<\/span>)<\/span>:<\/span><\/p>
        data_loader<\/span>.<\/span>sampler<\/span>.<\/span>set_epoch<\/span>(<\/span>epoch<\/span>)<\/span><\/p>
        train_one_epoch<\/span>(<\/span>.<\/span>.<\/span>.<\/span>)<\/span><\/p><\/pre>\n
\u041f\u0435\u0440\u0435\u0434\u043e\u0432\u043e\u0439 \u043e\u043f\u044b\u0442 \u0434\u043b\u044f DDP<\/h3>\n
\u041b\u044e\u0431\u044b\u0435 \u043c\u0435\u0442\u043e\u0434\u044b, \u0437\u0430\u0433\u0440\u0443\u0436\u0430\u044e\u0449\u0438\u0435 \u0434\u0430\u043d\u043d\u044b\u0435, \u0434\u043e\u043b\u0436\u043d\u044b \u0431\u044b\u0442\u044c \u0438\u0437\u043e\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u044b \u043e\u0442 \u043e\u0441\u043d\u043e\u0432\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430.  \u041b\u044e\u0431\u044b\u0435 \u043c\u0435\u0442\u043e\u0434\u044b, \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u044e\u0449\u0438\u0435 \u0444\u0430\u0439\u043b\u043e\u0432\u044b\u0439 \u0432\u0432\u043e\u0434-\u0432\u044b\u0432\u043e\u0434, \u0434\u043e\u043b\u0436\u043d\u044b \u0431\u044b\u0442\u044c \u0438\u0437\u043e\u043b\u0438\u0440\u043e\u0432\u0430\u043d\u044b \u043e\u0442 \u043e\u0441\u043d\u043e\u0432\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430.<\/p>\n
import<\/span> torch<\/span>.<\/span>distributed <\/span>as<\/span> dist<\/span><\/p>
import<\/span> torch <\/span><\/p>
def<\/span> <\/span>is_dist_avail_and_initialized<\/span>(<\/span>)<\/span>:<\/span><\/p>
    <\/span>if<\/span> <\/span>not<\/span> dist<\/span>.<\/span>is_available<\/span>(<\/span>)<\/span>:<\/span><\/p>
        <\/span>return<\/span> <\/span>False<\/span><\/p>
    <\/span>if<\/span> <\/span>not<\/span> dist<\/span>.<\/span>is_initialized<\/span>(<\/span>)<\/span>:<\/span><\/p>
        <\/span>return<\/span> <\/span>False<\/span><\/p>
    <\/span>return<\/span> <\/span>True<\/span><\/p>
def<\/span> <\/span>save_on_master<\/span>(<\/span>*<\/span>args<\/span>,<\/span> <\/span>**<\/span>kwargs<\/span>)<\/span>:<\/span><\/p>
    <\/span>if<\/span> is_main_process<\/span>(<\/span>)<\/span>:<\/span><\/p>
        torch<\/span>.<\/span>save<\/span>(<\/span>*<\/span>args<\/span>,<\/span> <\/span>**<\/span>kwargs<\/span>)<\/span><\/p>
def<\/span> <\/span>get_rank<\/span>(<\/span>)<\/span>:<\/span><\/p>
    <\/span>if<\/span> <\/span>not<\/span> is_dist_avail_and_initialized<\/span>(<\/span>)<\/span>:<\/span><\/p>
        <\/span>return<\/span> <\/span>0<\/span><\/p>
    <\/span>return<\/span> dist<\/span>.<\/span>get_rank<\/span>(<\/span>)<\/span><\/p>
def<\/span> <\/span>is_main_process<\/span>(<\/span>)<\/span>:<\/span><\/p>
    <\/span>return<\/span> get_rank<\/span>(<\/span>)<\/span> <\/span>==<\/span> <\/span>0<\/span><\/p><\/pre>\n
\u041d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u044d\u0442\u043e\u0439 \u0444\u0443\u043d\u043a\u0446\u0438\u0438 \u0432\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u0431\u044b\u0442\u044c \u0443\u0432\u0435\u0440\u0435\u043d\u044b, \u0447\u0442\u043e \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043a\u043e\u043c\u0430\u043d\u0434\u044b \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u044e\u0442\u0441\u044f \u0442\u043e\u043b\u044c\u043a\u043e \u0438\u0437 \u043e\u0441\u043d\u043e\u0432\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430:<\/p>\n
if<\/span> is_main_process<\/span>(<\/span>)<\/span>:<\/span><\/p><\/pre>\n
\u0417\u0430\u043f\u0443\u0441\u0442\u0438\u0442\u044c \u0441\u043a\u0440\u0438\u043f\u0442 \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e torch.distributed.launch<\/code> \u0438\u043b\u0438 torch.run<\/code><\/p>\n
$ python -m torch.distributed.launch --nproc_per_node=4 main_script.py<\/span><\/p><\/pre>\n
\u041e\u0448\u0438\u0431\u043a\u0438 \u0431\u0443\u0434\u0443\u0442.  \u041e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c\u043d\u043e \u043e\u0442\u043a\u043b\u044e\u0447\u0438\u0442\u0435 \u043b\u044e\u0431\u043e\u0439 \u043d\u0435\u0436\u0435\u043b\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0439 \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f:<\/p>\n
$ kill $(ps aux | grep main_script.py | grep -v grep | awk '{print $2}')<\/span><\/p><\/pre>\n
\u0417\u0430\u043c\u0435\u043d\u044f\u0442\u044c main_script.py<\/code> \u0441 \u0438\u043c\u0435\u043d\u0435\u043c \u0432\u0430\u0448\u0435\u0433\u043e \u0441\u043a\u0440\u0438\u043f\u0442\u0430.  \u0415\u0449\u0435 \u043e\u0434\u0438\u043d \u0431\u043e\u043b\u0435\u0435 \u043f\u0440\u043e\u0441\u0442\u043e\u0439 \u0432\u0430\u0440\u0438\u0430\u043d\u0442 $ kill -9 PID<\/code>.  \u0412 \u043f\u0440\u043e\u0442\u0438\u0432\u043d\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435 \u0432\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u043f\u0435\u0440\u0435\u0439\u0442\u0438 \u043a \u0431\u043e\u043b\u0435\u0435 \u0441\u043b\u043e\u0436\u043d\u044b\u043c \u0432\u0435\u0449\u0430\u043c, \u043d\u0430\u043f\u0440\u0438\u043c\u0435\u0440, \u0443\u0431\u0438\u0442\u044c \u0432\u0441\u0435 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u044b, \u0441\u0432\u044f\u0437\u0430\u043d\u043d\u044b\u0435 \u0441 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u043e\u043c CUDA, \u0435\u0441\u043b\u0438 \u043e\u043d\u0438 \u043d\u0435 \u043f\u043e\u043a\u0430\u0437\u0430\u043d\u044b \u0432 nvidia-smi<\/code><\/p>\n
lsof \/dev\/nvidia* | awk '{print $2}' | xargs -I {} kill {}<\/span><\/p><\/pre>\n
\u042d\u0442\u043e \u0442\u043e\u043b\u044c\u043a\u043e \u0432 \u0442\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435, \u0435\u0441\u043b\u0438 \u0432\u044b \u043d\u0435 \u043c\u043e\u0436\u0435\u0442\u0435 \u043d\u0430\u0439\u0442\u0438 PID \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430, \u0437\u0430\u043f\u0443\u0449\u0435\u043d\u043d\u043e\u0433\u043e \u0432 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u043e\u043c \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0440\u0435.<\/p>\n
\n
\u041e\u0447\u0435\u043d\u044c \u0445\u043e\u0440\u043e\u0448\u0430\u044f \u043a\u043d\u0438\u0433\u0430 \u043f\u043e \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u043e\u043c\u0443 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044e \u2014 \u00ab\u0420\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u043e\u0435 \u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e Python: \u0443\u0441\u043a\u043e\u0440\u0435\u043d\u0438\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043c\u043e\u0434\u0435\u043b\u0435\u0439 \u0438 \u043e\u0431\u0441\u043b\u0443\u0436\u0438\u0432\u0430\u043d\u0438\u0435 \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0445 \u0441\u0438\u0441\u0442\u0435\u043c\u00bb \u0413\u0443\u0430\u043d\u044c\u0445\u0443\u0430 \u0412\u0430\u043d\u0430.<\/p>\n<\/blockquote>\n
\u041e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441\u043c\u0435\u0448\u0430\u043d\u043d\u043e\u0439 \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u0438 \u0432 Pytorch<\/h2>\n
\u0421\u043c\u0435\u0448\u0430\u043d\u043d\u0430\u044f \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u044f\u0435\u0442 \u0432\u044b\u0447\u0438\u0441\u043b\u0435\u043d\u0438\u044f \u0441 \u043f\u043b\u0430\u0432\u0430\u044e\u0449\u0435\u0439 \u0437\u0430\u043f\u044f\u0442\u043e\u0439 (FP) 16 \u0438 FP 32 \u043d\u0430 \u0440\u0430\u0437\u043d\u044b\u0445 \u044d\u0442\u0430\u043f\u0430\u0445 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f.  \u0422\u0440\u0435\u043d\u0438\u0440\u043e\u0432\u043a\u0430 FP16 \u0442\u0430\u043a\u0436\u0435 \u0438\u0437\u0432\u0435\u0441\u0442\u043d\u0430 \u043a\u0430\u043a \u0442\u0440\u0435\u043d\u0438\u0440\u043e\u0432\u043a\u0430 \u0441 \u043f\u043e\u043b\u043e\u0432\u0438\u043d\u043d\u043e\u0439 \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c\u044e, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0438\u043c\u0435\u0435\u0442 \u0431\u043e\u043b\u0435\u0435 \u043d\u0438\u0437\u043a\u0443\u044e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c.  \u0410\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u0441\u043c\u0435\u0448\u0430\u043d\u043d\u0430\u044f \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c \u2014 \u044d\u0442\u043e \u0431\u0443\u043a\u0432\u0430\u043b\u044c\u043d\u043e \u043b\u0443\u0447\u0448\u0435\u0435 \u0438\u0437 \u043e\u0431\u043e\u0438\u0445 \u043c\u0438\u0440\u043e\u0432: \u0441\u043e\u043a\u0440\u0430\u0449\u0435\u043d\u043d\u043e\u0435 \u0432\u0440\u0435\u043c\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0441 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c\u044e, \u0441\u0440\u0430\u0432\u043d\u0438\u043c\u043e\u0439 \u0441 FP32.<\/p>\n
\u0412 \u0441\u043c\u0435\u0448\u0430\u043d\u043d\u043e\u0439 \u0442\u043e\u0447\u043d\u043e\u0439 \u0442\u0440\u0435\u043d\u0438\u0440\u043e\u0432\u043a\u0435 \u0432\u0441\u0435 \u0432\u044b\u0447\u0438\u0441\u043b\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043e\u043f\u0435\u0440\u0430\u0446\u0438\u0438 (\u043f\u0440\u044f\u043c\u043e\u0439 \u043f\u0440\u043e\u0445\u043e\u0434, \u043e\u0431\u0440\u0430\u0442\u043d\u044b\u0439 \u043f\u0440\u043e\u0445\u043e\u0434, \u0432\u0435\u0441\u043e\u0432\u044b\u0435 \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u044b) \u0441\u043e\u043e\u0442\u0432\u0435\u0442\u0441\u0442\u0432\u0443\u044e\u0442 \u0432\u0435\u0440\u0441\u0438\u0438 FP16.  \u0414\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u0430 \u043a\u043e\u043f\u0438\u044f \u0432\u0435\u0441\u0430 FP32, \u0430 \u0442\u0430\u043a\u0436\u0435 \u0432\u044b\u0447\u0438\u0441\u043b\u0435\u043d\u0438\u0435 \u043f\u043e\u0442\u0435\u0440\u044c \u0432 FP32 \u043f\u043e\u0441\u043b\u0435 \u043f\u0440\u044f\u043c\u043e\u0433\u043e \u043f\u0440\u043e\u0445\u043e\u0434\u0430 \u0432 FP16, \u0447\u0442\u043e\u0431\u044b \u0438\u0437\u0431\u0435\u0436\u0430\u0442\u044c \u043f\u0435\u0440\u0435\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u0438 \u043d\u0435\u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f.  \u0413\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u044b \u0432\u0435\u0441\u043e\u0432 \u0432\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u044e\u0442\u0441\u044f \u043a FP32 \u0434\u043b\u044f \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u044f \u0432\u0435\u0441\u043e\u0432 \u043c\u043e\u0434\u0435\u043b\u0438.  \u041a\u0440\u043e\u043c\u0435 \u0442\u043e\u0433\u043e, \u043f\u043e\u0442\u0435\u0440\u0438 \u0432 FP32 \u043c\u0430\u0441\u0448\u0442\u0430\u0431\u0438\u0440\u0443\u044e\u0442\u0441\u044f, \u0447\u0442\u043e\u0431\u044b \u0438\u0437\u0431\u0435\u0436\u0430\u0442\u044c \u043f\u043e\u0442\u0435\u0440\u0438 \u0437\u043d\u0430\u0447\u0438\u043c\u043e\u0441\u0442\u0438 \u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u0430 \u043f\u0435\u0440\u0435\u0434 \u043f\u0435\u0440\u0435\u0445\u043e\u0434\u043e\u043c \u043a FP16 \u0434\u043b\u044f \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u043e\u0431\u0440\u0430\u0442\u043d\u043e\u0433\u043e \u043f\u0440\u043e\u0445\u043e\u0434\u0430.  \u0412 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u043a\u043e\u043c\u043f\u0435\u043d\u0441\u0430\u0446\u0438\u0438 \u0432\u0435\u0441\u0430 FP32 \u0431\u0443\u0434\u0443\u0442 \u0443\u043c\u0435\u043d\u044c\u0448\u0435\u043d\u044b \u043d\u0430 \u043e\u0434\u0438\u043d \u0438 \u0442\u043e\u0442 \u0436\u0435 \u0441\u043a\u0430\u043b\u044f\u0440 \u043f\u0435\u0440\u0435\u0434 \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u043c \u0432\u0435\u0441\u0430. <\/p>\n