{"id":971,"date":"2023-08-09T03:21:07","date_gmt":"2023-08-09T03:21:07","guid":{"rendered":"https:\/\/www.gptmain.news\/?p=971"},"modified":"2023-08-09T03:21:07","modified_gmt":"2023-08-09T03:21:07","slug":"%d1%83%d1%87%d0%b5%d0%b1%d0%bd%d0%b8%d0%ba-byol-%d1%81%d0%b0%d0%bc%d0%be%d1%81%d1%82%d0%be%d1%8f%d1%82%d0%b5%d0%bb%d1%8c%d0%bd%d0%be%d0%b5-%d0%be%d0%b1%d1%83%d1%87%d0%b5%d0%bd%d0%b8%d0%b5-%d0%b8","status":"publish","type":"post","link":"https:\/\/gptmain.news\/?p=971","title":{"rendered":"\u0423\u0447\u0435\u0431\u043d\u0438\u043a BYOL: \u0441\u0430\u043c\u043e\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f\u043c CIFAR \u0441 \u043a\u043e\u0434\u043e\u043c \u0432 Pytorch\n | GPTMain News"},"content":{"rendered":"<div id=\"\">\n<p>\u041f\u043e\u0441\u043b\u0435 \u043f\u0440\u0435\u0437\u0435\u043d\u0442\u0430\u0446\u0438\u0438 SimCLR <strong>\u043a\u043e\u043d\u0442\u0440\u0430\u0441\u0442\u043d\u044b\u0439<\/strong> \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u043e\u0439 \u0441\u0438\u0441\u0442\u0435\u043c\u044b \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f, \u044f \u0440\u0435\u0448\u0438\u043b \u043f\u0440\u043e\u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0435\u0449\u0435 \u043e\u0434\u0438\u043d \u043f\u0435\u0447\u0430\u043b\u044c\u043d\u043e \u0438\u0437\u0432\u0435\u0441\u0442\u043d\u044b\u0439 \u043c\u0435\u0442\u043e\u0434, \u043d\u0430\u0437\u044b\u0432\u0430\u0435\u043c\u044b\u0439 BYOL.  Bootstrap Your Own Latent (BYOL) \u2014 \u044d\u0442\u043e \u043d\u043e\u0432\u044b\u0439 \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f\u043c \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439.  BYOL \u0438\u043c\u0435\u0435\u0442 \u0434\u0432\u0430 \u043e\u0441\u043d\u043e\u0432\u043d\u044b\u0445 \u043f\u0440\u0435\u0438\u043c\u0443\u0449\u0435\u0441\u0442\u0432\u0430: <\/p>\n<ul>\n<li>\n<p>\u041e\u043d \u044f\u0432\u043d\u043e \u043d\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442 \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043e\u0431\u0440\u0430\u0437\u0446\u044b.  \u0412\u043c\u0435\u0441\u0442\u043e \u044d\u0442\u043e\u0433\u043e \u043e\u043d \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u043c\u0438\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u0443\u0435\u0442 \u0441\u0445\u043e\u0434\u0441\u0442\u0432\u043e \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0439 \u043e\u0434\u043d\u043e\u0433\u043e \u0438 \u0442\u043e\u0433\u043e \u0436\u0435 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f \u0432 \u0434\u0440\u0443\u0433\u043e\u043c \u0440\u0430\u0441\u0448\u0438\u0440\u0435\u043d\u043d\u043e\u043c \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0438 (\u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u043f\u0430\u0440\u0430).  \u041e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043e\u0431\u0440\u0430\u0437\u0446\u044b \u2014 \u044d\u0442\u043e \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f \u0438\u0437 \u043f\u0430\u0440\u0442\u0438\u0438, \u043e\u0442\u043b\u0438\u0447\u043d\u043e\u0439 \u043e\u0442 \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043f\u0430\u0440\u044b.<\/p>\n<\/li>\n<li>\n<p>\u0412 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0435 \u0443\u0442\u0432\u0435\u0440\u0436\u0434\u0430\u0435\u0442\u0441\u044f, \u0447\u0442\u043e BYOL \u0442\u0440\u0435\u0431\u0443\u0435\u0442 \u043c\u0435\u043d\u044c\u0448\u0438\u0445 \u0440\u0430\u0437\u043c\u0435\u0440\u043e\u0432 \u043f\u0430\u043a\u0435\u0442\u043e\u0432, \u0447\u0442\u043e \u0434\u0435\u043b\u0430\u0435\u0442 \u0435\u0433\u043e \u043f\u0440\u0438\u0432\u043b\u0435\u043a\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u0432\u044b\u0431\u043e\u0440\u043e\u043c.<\/p>\n<\/li>\n<\/ul>\n<p>\u041d\u0438\u0436\u0435 \u0432\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u043e\u0437\u043d\u0430\u043a\u043e\u043c\u0438\u0442\u044c\u0441\u044f \u0441 \u043c\u0435\u0442\u043e\u0434\u043e\u043c.  \u0412 \u043e\u0442\u043b\u0438\u0447\u0438\u0435 \u043e\u0442 \u043e\u0440\u0438\u0433\u0438\u043d\u0430\u043b\u044c\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438, \u044f \u043d\u0430\u0437\u044b\u0432\u0430\u044e \u043e\u043d\u043b\u0430\u0439\u043d-\u0441\u0435\u0442\u0435\u0432\u043e\u0433\u043e \u0441\u0442\u0443\u0434\u0435\u043d\u0442\u0430 \u0438 \u0446\u0435\u043b\u0435\u0432\u043e\u0433\u043e \u0441\u0435\u0442\u0435\u0432\u043e\u0433\u043e \u0443\u0447\u0438\u0442\u0435\u043b\u044f. <\/p>\n<p><span class=\"gatsby-resp-image-wrapper\" style=\"position:relative;display:block;margin-left:auto;margin-right:auto;max-width:1041px\"><\/p>\n<p>    <span class=\"gatsby-resp-image-background-image\" style=\"padding-bottom:59.00000000000001%;position:relative;bottom:0;left:0;background-image:url('data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAMCAIAAADtbgqsAAAACXBIWXMAAAsTAAALEwEAmpwYAAAB50lEQVQoz3VSO2\/UQBD2\/0NUFPkDSIiSBjoaJEJDgVJBBwWJkCKiFJHCCSmIYN8dMdiYS4Qujzu\/12d714+192WbdS4KQbp83ezMtzPzzae03Qo0TQuzMiswRAXjomn7lyVulik3A9GQiuQsjzt4en7motD23SgPbNuOSrKihyKEqAkjlFPWTfS9\/e3nk8Pt+vTgx2g8N1VL1x3zm6mbAPE8Q2maYoz\/kXEtCtww3lR1N\/jy9t3mE2u4wy4Ov6uaYw0nPw3X0kz9l5\/QDPWoqkrSlvMro\/GH4dF7N5jIgLAyryHFsEPO3AYw9D0vyYHn+bCiPYGyZsm8In\/++PLT7vp0pl3q1Oc45xAi1\/XCEORFKauE6IsZaxZJJv8LggAAQClVXr2+v76xZhwPrskSNREpLOK0kK2EaLrLPly0jgcWi0huHkURhFDZP9gafN0MojOZ9iLfmJ5EwawNj6d\/zsOLqTNz0vLqPJQJXLN+7GvBCG0Ja6VgNekevXh858HdrZ03zUzVNc22RtbR2IupD4oEkTzvJftvZy54RWhNpRW6pxvP1h7eG6q77VzTVdX9PT4xDC9hKarKihNCyrK81SRlhWOUUpx1OPQckEUAhCnCS4VXmWSVOztK2SJOUJYnCZQqtreQ\/wIkCpnf0KUCKQAAAABJRU5ErkJggg==');background-size:cover;display:block\"\/><br \/>\n  <img decoding=\"async\" class=\"gatsby-resp-image-image\" alt=\"\u0431\u0435\u043b\u044c-\u043e\u0431\u0437\u043e\u0440\" title=\"\u0431\u0435\u043b\u044c-\u043e\u0431\u0437\u043e\u0440\" src=\"https:\/\/theaisummer.com\/static\/9bc12ecbc6005ca51f38120b6a7691b3\/ee9b6\/byol-overview.png\" srcset=\"\/static\/9bc12ecbc6005ca51f38120b6a7691b3\/5a46d\/byol-overview.png 300w,\/static\/9bc12ecbc6005ca51f38120b6a7691b3\/0a47e\/byol-overview.png 600w,\/static\/9bc12ecbc6005ca51f38120b6a7691b3\/ee9b6\/byol-overview.png 1041w\" sizes=\"(max-width: 1041px) 100vw, 1041px\" style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0\" loading=\"lazy\"\/><\/p>\n<p>    <\/span><br \/>\n<em>\u041e\u0431\u0437\u043e\u0440 \u043c\u0435\u0442\u043e\u0434\u0430 BYOL.  \u0418\u0441\u0442\u043e\u0447\u043d\u0438\u043a: \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442 BYOL.<\/em> <\/p>\n<p>\u041e\u043d\u043b\u0430\u0439\u043d-\u0441\u0435\u0442\u044c \u0430\u043a\u0430 \u0441\u0442\u0443\u0434\u0435\u043d\u0442: \u043f\u043e \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044e \u0441 SimCLR \u0435\u0441\u0442\u044c \u0432\u0442\u043e\u0440\u043e\u0439 MLP, \u043d\u0430\u0437\u044b\u0432\u0430\u0435\u043c\u044b\u0439 <strong>\u043f\u0440\u0435\u0434\u0441\u043a\u0430\u0437\u0430\u0442\u0435\u043b\u044c<\/strong>, \u0447\u0442\u043e \u0434\u0435\u043b\u0430\u0435\u0442 \u0432\u0435\u0441\u044c \u043c\u0435\u0442\u043e\u0434 \u0430\u0441\u0438\u043c\u043c\u0435\u0442\u0440\u0438\u0447\u043d\u044b\u043c.  \u0410\u0441\u0438\u043c\u043c\u0435\u0442\u0440\u0438\u0447\u043d\u043e\u0441\u0442\u044c \u043f\u043e \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044e \u0441 \u0447\u0435\u043c?  \u041d\u0443 \u0438 \u043a \u043c\u043e\u0434\u0435\u043b\u0438 \u0443\u0447\u0438\u0442\u0435\u043b\u044f (\u0446\u0435\u043b\u0435\u0432\u0430\u044f \u0441\u0435\u0442\u044c). <\/p>\n<p>\u041f\u043e\u0447\u0435\u043c\u0443 \u044d\u0442\u043e \u0432\u0430\u0436\u043d\u043e? <\/p>\n<p>\u041f\u043e\u0441\u043a\u043e\u043b\u044c\u043a\u0443 \u043c\u043e\u0434\u0435\u043b\u044c \u0443\u0447\u0438\u0442\u0435\u043b\u044f \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0430 <strong>\u0442\u043e\u043b\u044c\u043a\u043e<\/strong> \u0447\u0435\u0440\u0435\u0437 \u044d\u043a\u0441\u043f\u043e\u043d\u0435\u043d\u0446\u0438\u0430\u043b\u044c\u043d\u0443\u044e \u0441\u043a\u043e\u043b\u044c\u0437\u044f\u0449\u0443\u044e \u0441\u0440\u0435\u0434\u043d\u044e\u044e (EMA) \u043e\u0442 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432 \u0441\u0442\u0443\u0434\u0435\u043d\u0442\u0430.  \u0412 \u043a\u043e\u043d\u0435\u0447\u043d\u043e\u043c \u0438\u0442\u043e\u0433\u0435 \u043d\u0430 \u043a\u0430\u0436\u0434\u043e\u0439 \u0438\u0442\u0435\u0440\u0430\u0446\u0438\u0438 \u0443\u0447\u0438\u0442\u0435\u043b\u044e \u043f\u0435\u0440\u0435\u0434\u0430\u0435\u0442\u0441\u044f \u043a\u0440\u043e\u0448\u0435\u0447\u043d\u044b\u0439 \u043f\u0440\u043e\u0446\u0435\u043d\u0442 (\u043c\u0435\u043d\u0435\u0435 1%) \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432 \u0443\u0447\u0435\u043d\u0438\u043a\u0430.  \u0422\u0430\u043a\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c, <strong>\u0433\u0440\u0430\u0434\u0438\u0435\u043d\u0442\u044b \u043f\u0440\u043e\u0445\u043e\u0434\u044f\u0442 \u0442\u043e\u043b\u044c\u043a\u043e \u0447\u0435\u0440\u0435\u0437 \u0441\u0442\u0443\u0434\u0435\u043d\u0447\u0435\u0441\u043a\u0443\u044e \u0441\u0435\u0442\u044c<\/strong>.  \u042d\u0442\u043e \u043c\u043e\u0436\u0435\u0442 \u0431\u044b\u0442\u044c \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d\u043e \u043a\u0430\u043a:<\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">class<\/span><span class=\"token plain\"> <\/span><span class=\"token class-name\">EMA<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> alpha<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">super<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">alpha <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> alpha<\/span><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">update_average<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> old<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> new<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">if<\/span><span class=\"token plain\"> old <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">is<\/span><span class=\"token plain\"> <\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> new<\/span><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> old <\/span><span class=\"token operator\">*<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">alpha <\/span><span class=\"token operator\">+<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token number\">1<\/span><span class=\"token plain\"> <\/span><span class=\"token operator\">-<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">alpha<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"> <\/span><span class=\"token operator\">*<\/span><span class=\"token plain\"> new<\/span><\/p><p><span class=\"token plain\">ema <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> EMA<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token number\">0.99<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">for<\/span><span class=\"token plain\"> student_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> teacher_params <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">in<\/span><span class=\"token plain\"> <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">zip<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">student_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">parameters<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\">teacher_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">parameters<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">         old_weight<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> up_weight <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> teacher_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">data<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> student_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">data<\/span><\/p><p><span class=\"token plain\">         teacher_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">data <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> ema<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">update_average<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">old_weight<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> up_weight<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><\/p><\/pre>\n<p>\u0415\u0449\u0435 \u043e\u0434\u043d\u0438\u043c \u043a\u043b\u044e\u0447\u0435\u0432\u044b\u043c \u043e\u0442\u043b\u0438\u0447\u0438\u0435\u043c Simclr \u043e\u0442 BYOL \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0444\u0443\u043d\u043a\u0446\u0438\u044f \u043f\u043e\u0442\u0435\u0440\u044c.<\/p>\n<h2 id=\"loss-function\">\u0424\u0443\u043d\u043a\u0446\u0438\u044f \u043f\u043e\u0442\u0435\u0440\u044c<\/h2>\n<p> <strong>\u043f\u0440\u0435\u0434\u0438\u043a\u0442\u043e\u0440 MLP<\/strong> \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f <strong>\u0442\u043e\u043b\u044c\u043a\u043e<\/strong> \u043f\u0440\u0438\u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f \u043a \u0441\u0442\u0443\u0434\u0435\u043d\u0442\u0443, \u0434\u0435\u043b\u0430\u044f \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0443 <strong>\u0430\u0441\u0438\u043c\u043c\u0435\u0442\u0440\u0438\u0447\u043d\u044b\u0439<\/strong>.  \u042d\u0442\u043e \u043a\u043b\u044e\u0447\u0435\u0432\u043e\u0439 \u0432\u044b\u0431\u043e\u0440 \u043a\u043e\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u0438, \u043f\u043e\u0437\u0432\u043e\u043b\u044f\u044e\u0449\u0438\u0439 \u0438\u0437\u0431\u0435\u0436\u0430\u0442\u044c \u043a\u043e\u043b\u043b\u0430\u043f\u0441\u0430 \u0440\u0435\u0436\u0438\u043c\u0430.  \u0420\u0435\u0436\u0438\u043c \u043a\u043e\u043b\u043b\u0430\u043f\u0441\u0430 \u0437\u0434\u0435\u0441\u044c \u0431\u0443\u0434\u0435\u0442 \u0437\u0430\u043a\u043b\u044e\u0447\u0430\u0442\u044c\u0441\u044f \u0432 \u0442\u043e\u043c, \u0447\u0442\u043e\u0431\u044b \u0432\u044b\u0432\u043e\u0434\u0438\u0442\u044c \u043e\u0434\u043d\u0443 \u0438 \u0442\u0443 \u0436\u0435 \u043f\u0440\u043e\u0435\u043a\u0446\u0438\u044e \u0434\u043b\u044f \u0432\u0441\u0435\u0445 \u0432\u0445\u043e\u0434\u043d\u044b\u0445 \u0434\u0430\u043d\u043d\u044b\u0445.   <\/p>\n<p><span class=\"gatsby-resp-image-wrapper\" style=\"position:relative;display:block;margin-left:auto;margin-right:auto;max-width:1084px\"><\/p>\n<p>    <span class=\"gatsby-resp-image-background-image\" style=\"padding-bottom:35.66666666666667%;position:relative;bottom:0;left:0;background-image:url('data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAHCAIAAACHqfpvAAAACXBIWXMAAAsTAAALEwEAmpwYAAABIElEQVQY002QD4+DIAzF\/f5fb3eburlNB\/hniEqp4gC5uuUu9\/LS\/NKk5dEk\/iqETalpGKYQIuIiRMN5bczs3NY0XV23dnXEfT8+pfI+0EgCAMYgkdb2ftdl+XYFVKsKHg\/z6VSVITOGVG83DeBD8Enf90oNMW6cy9NJXi5DcR2LYjge2zST1+tUFOMp7bK8JyhLPBzE13crJQLoxBjzftzMuIJ2iN6AI88Y0OyMH8YA4KyNyxKlNIj2LzYs1mraxtnciLVr1rZBweeauCWea74IbqchbIHSvtxqrfXeJxuJbkV\/7rohy\/aUFP18lmmq8pxY5ec+y8Y8V4yPetcec5knrZP90BvNxheiZkwLYZ9PMoHm\/MNQ1xNjVmvnffynH6NviVXbB7aCAAAAAElFTkSuQmCC');background-size:cover;display:block\"\/><br \/>\n  <img decoding=\"async\" class=\"gatsby-resp-image-image\" alt=\"\u0431\u0435\u043b\u044c-\u0431\u0443\u043c\u0430\u0433\u0430-\u043e\u0431\u0437\u043e\u0440-\u0441-\u0442\u0435\u043d\u0437\u043e\u0440\u0430\u043c\u0438\" title=\"\u0431\u0435\u043b\u044c-\u0431\u0443\u043c\u0430\u0433\u0430-\u043e\u0431\u0437\u043e\u0440-\u0441-\u0442\u0435\u043d\u0437\u043e\u0440\u0430\u043c\u0438\" src=\"https:\/\/theaisummer.com\/static\/75842cc591c65f644268b688c5a1be6b\/48ca3\/byol-paper-overview-with-tensors.png\" srcset=\"\/static\/75842cc591c65f644268b688c5a1be6b\/5a46d\/byol-paper-overview-with-tensors.png 300w,\/static\/75842cc591c65f644268b688c5a1be6b\/0a47e\/byol-paper-overview-with-tensors.png 600w,\/static\/75842cc591c65f644268b688c5a1be6b\/48ca3\/byol-paper-overview-with-tensors.png 1084w\" sizes=\"(max-width: 1084px) 100vw, 1084px\" style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0\" loading=\"lazy\"\/><\/p>\n<p>    <\/span><br \/>\n<em>\u041e\u0431\u0437\u043e\u0440 \u043c\u0435\u0442\u043e\u0434\u0430 BYOL.  \u0418\u0441\u0442\u043e\u0447\u043d\u0438\u043a: \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442 BYOL.<\/em> <\/p>\n<p>\u041d\u0430\u043a\u043e\u043d\u0435\u0446, \u0430\u0432\u0442\u043e\u0440\u044b \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u043b\u0438 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0443\u044e \u0441\u0440\u0435\u0434\u043d\u0435\u043a\u0432\u0430\u0434\u0440\u0430\u0442\u0438\u0447\u043d\u0443\u044e \u043e\u0448\u0438\u0431\u043a\u0443 \u043c\u0435\u0436\u0434\u0443 L2-\u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d\u043d\u044b\u043c\u0438 \u043f\u0440\u043e\u0433\u043d\u043e\u0437\u0430\u043c\u0438 \u0438 \u0446\u0435\u043b\u0435\u0432\u044b\u043c\u0438 \u043f\u0440\u043e\u0433\u043d\u043e\u0437\u0430\u043c\u0438:<\/p>\n<div class=\"math\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><math xmlns=\"http:\/\/www.w3.org\/1998\/Math\/MathML\" display=\"block\"><semantics><mrow><msub><mi mathvariant=\"script\">\u043b<\/mi><mrow><mi>\u03b8<\/mi><mo separator=\"true\">,<\/mo><mi>\u03be<\/mi><\/mrow><\/msub><mo>\u225c<\/mo><msubsup><mrow><mo fence=\"true\">\u2225<\/mo><msub><mover accent=\"true\"><mi>\u0434<\/mi><mo>\u02c9<\/mo><\/mover><mi>\u03b8<\/mi><\/msub><mrow><mo fence=\"true\">(<\/mo><msub><mi>\u0433<\/mi><mi>\u03b8<\/mi><\/msub><mo fence=\"true\">)<\/mo><\/mrow><mo>\u2212<\/mo><msubsup><mover accent=\"true\"><mi>\u0433<\/mi><mo>\u02c9<\/mo><\/mover><mi>\u03be<\/mi><mo mathvariant=\"normal\" lspace=\"0em\" rspace=\"0em\">\u2032<\/mo><\/msubsup><mo fence=\"true\">\u2225<\/mo><\/mrow><mn>2<\/mn><mn>2<\/mn><\/msubsup><mo>&#8220;=&#8221;<\/mo><mn>2<\/mn><mo>\u2212<\/mo><mn>2<\/mn><mo>\u22c5<\/mo><mfrac><mrow><mo fence=\"true\">\u27e8<\/mo><msub><mi>\u0434<\/mi><mi>\u03b8<\/mi><\/msub><mrow><mo fence=\"true\">(<\/mo><msub><mi>\u0433<\/mi><mi>\u03b8<\/mi><\/msub><mo fence=\"true\">)<\/mo><\/mrow><mo separator=\"true\">,<\/mo><msubsup><mi>\u0433<\/mi><mi>\u03be<\/mi><mo mathvariant=\"normal\" lspace=\"0em\" rspace=\"0em\">\u2032<\/mo><\/msubsup><mo fence=\"true\">\u27e9<\/mo><\/mrow><mrow><msub><mrow><mo fence=\"true\">\u2225<\/mo><msub><mi>\u0434<\/mi><mi>\u03b8<\/mi><\/msub><mrow><mo fence=\"true\">(<\/mo><msub><mi>\u0433<\/mi><mi>\u03b8<\/mi><\/msub><mo fence=\"true\">)<\/mo><\/mrow><mo fence=\"true\">\u2225<\/mo><\/mrow><mn>2<\/mn><\/msub><mo>\u22c5<\/mo><msub><mrow><mo fence=\"true\">\u2225<\/mo><msubsup><mi>\u0433<\/mi><mi>\u03be<\/mi><mo mathvariant=\"normal\" lspace=\"0em\" rspace=\"0em\">\u2032<\/mo><\/msubsup><mo fence=\"true\">\u2225<\/mo><\/mrow><mn>2<\/mn><\/msub><\/mrow><\/mfrac><mi mathvariant=\"normal\">.<\/mi><\/mrow><annotation encoding=\"application\/x-tex\">\\mathcal{L}_{\\theta, \\xi}\\triangleq\\left\\|\\bar{q}_{\\theta}\\left(z_{\\theta}\\right)-\\bar{z}_{\\xi }^{\\prime}\\right\\|_{2}^{2}=2-2\\cdot\\frac{\\left\\langle q_{\\theta}\\left(z_{\\theta}\\right), z_{ \\xi}^{\\prime}\\right\\rangle}{\\left\\|q_{\\theta}\\left(z_{\\theta}\\right)\\right\\|_{2} \\cdot\\left\\|z_{ \\xi}^{\\prime}\\right\\|_{2}} .<\/annotation><\/semantics><\/math><\/span><span class=\"katex-html\" aria-hidden=\"true\"><span class=\"base\"><span class=\"strut\" style=\"height:1.202778em;vertical-align:-0.286108em\"\/><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathcal\">\u043b<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.3361079999999999em\"><span style=\"top:-2.5500000000000003em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right:0.04601em\">\u03be<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.286108em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><span class=\"mrel amsrm\">\u225c<\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:1.5047960000000002em;vertical-align:-0.43280799999999997em\"\/><span class=\"minner\"><span class=\"minner\"><span class=\"mopen\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.8679800000000001em\"><span style=\"top:-2.2559899999999997em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.26698em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.86798em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.35000999999999993em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.56778em\"><span style=\"top:-3em\"><span class=\"pstrut\" style=\"height:3em\"\/><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.03588em\">\u0434<\/span><\/span><\/span><span style=\"top:-3em\"><span class=\"pstrut\" style=\"height:3em\"\/><span class=\"accent-body\" style=\"left:-0.16666em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.19444em\"><span\/><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.33610799999999996em\"><span style=\"top:-2.5500000000000003em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.15em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right:0.16666666666666666em\"\/><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top:0em\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.04398em\">\u0433<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.33610799999999996em\"><span style=\"top:-2.5500000000000003em;margin-left:-0.04398em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.15em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top:0em\">)<\/span><\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.56778em\"><span style=\"top:-3em\"><span class=\"pstrut\" style=\"height:3em\"\/><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.04398em\">\u0433<\/span><\/span><\/span><span style=\"top:-3em\"><span class=\"pstrut\" style=\"height:3em\"\/><span class=\"accent-body\" style=\"left:-0.19444em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.801892em\"><span style=\"top:-2.4530000000000003em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.04601em\">\u03be<\/span><\/span><\/span><\/span><span style=\"top:-3.1130000000000004em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.383108em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.8679800000000001em\"><span style=\"top:-2.2559899999999997em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.26698em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.86798em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.35000999999999993em\"><span\/><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:1.0719880000000002em\"><span style=\"top:-2.267192em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span style=\"top:-3.3208800000000003em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.43280799999999997em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><span class=\"mrel\">&#8220;=&#8221;<\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:0.72777em;vertical-align:-0.08333em\"\/><span class=\"mord\">2<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:0.64444em;vertical-align:0em\"\/><span class=\"mord\">2<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:3.95371em;vertical-align:-1.76369em\"\/><span class=\"mord\"><span class=\"mopen nulldelimiter\"\/><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:2.19002em\"><span style=\"top:-2.11em\"><span class=\"pstrut\" style=\"height:3.173975em\"\/><span class=\"mord\"><span class=\"minner\"><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top:0em\">\u2225<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.03588em\">\u0434<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.33610799999999996em\"><span style=\"top:-2.5500000000000003em;margin-left:-0.03588em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.15em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right:0.16666666666666666em\"\/><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top:0em\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.04398em\">\u0433<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.33610799999999996em\"><span style=\"top:-2.5500000000000003em;margin-left:-0.04398em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.15em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top:0em\">)<\/span><\/span><span class=\"mclose delimcenter\" style=\"top:0em\">\u2225<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.151408em\"><span style=\"top:-2.4003em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.29969999999999997em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"minner\"><span class=\"minner\"><span class=\"mopen\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:1.173975em\"><span style=\"top:-1.955985em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.556985em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.572975em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-3.173975em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.6500149999999999em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.04398em\">\u0433<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.733692em\"><span style=\"top:-2.3986920000000005em;margin-left:-0.04398em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.04601em\">\u03be<\/span><\/span><\/span><\/span><span style=\"top:-3.0448000000000004em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.4374159999999999em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:1.173975em\"><span style=\"top:-1.955985em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.556985em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-2.572975em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><span style=\"top:-3.173975em\"><span class=\"pstrut\" style=\"height:2.606em\"\/><span class=\"delimsizinginner delim-size1\"><span>\u2225<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.6500149999999999em\"><span\/><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:-0.2486069999999999em\"><span style=\"top:-2.0002850000000003em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.6997149999999999em\"><span\/><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span style=\"top:-3.403975em\"><span class=\"pstrut\" style=\"height:3.173975em\"\/><span class=\"frac-line\" style=\"border-bottom-width:0.04em\"\/><\/span><span style=\"top:-4.213995000000001em\"><span class=\"pstrut\" style=\"height:3.173975em\"\/><span class=\"mord\"><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top:0em\"><span class=\"delimsizing size2\">\u27e8<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.03588em\">\u0434<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.33610799999999996em\"><span style=\"top:-2.5500000000000003em;margin-left:-0.03588em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.15em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right:0.16666666666666666em\"\/><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top:0em\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.04398em\">\u0433<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.33610799999999996em\"><span style=\"top:-2.5500000000000003em;margin-left:-0.04398em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.02778em\">\u03b8<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.15em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top:0em\">)<\/span><\/span><span class=\"mspace\" style=\"margin-right:0.16666666666666666em\"\/><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right:0.16666666666666666em\"\/><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right:0.04398em\">\u0433<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.751892em\"><span style=\"top:-2.4168920000000003em;margin-left:-0.04398em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right:0.04601em\">\u03be<\/span><\/span><\/span><\/span><span style=\"top:-3.063em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.4192159999999999em\"><span\/><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top:0em\"><span class=\"delimsizing size2\">\u27e9<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:1.76369em\"><span\/><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"\/><\/span><span class=\"mord\">.<\/span><\/span><\/span><\/span><\/span><\/div>\n<p>\u041f\u043e\u0442\u0435\u0440\u044f L2 \u043c\u043e\u0436\u0435\u0442 \u0431\u044b\u0442\u044c \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d\u0430 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c.  \u041d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f L2 \u043f\u0440\u0438\u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f \u0437\u0430\u0440\u0430\u043d\u0435\u0435.<\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> torch<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">functional <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">as<\/span><span class=\"token plain\"> F<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">loss_fn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">x<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> y<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   x <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> F<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">normalize<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">x<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> dim<\/span><span class=\"token operator\">=<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> p<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   y <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> F<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">normalize<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">y<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> dim<\/span><span class=\"token operator\">=<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> p<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">2<\/span><span class=\"token plain\"> <\/span><span class=\"token operator\">-<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">2<\/span><span class=\"token plain\"> <\/span><span class=\"token operator\">*<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">x <\/span><span class=\"token operator\">*<\/span><span class=\"token plain\"> y<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">sum<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">dim<\/span><span class=\"token operator\">=<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><\/p><\/pre>\n<blockquote>\n<p>\u041a\u043e\u0434 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u043d\u0430 GitHub<\/p>\n<\/blockquote>\n<h2 id=\"tracking-down-whats-happening-in-self-supervised-pretraining-knn-accuracy\">\u041e\u0442\u0441\u043b\u0435\u0436\u0438\u0432\u0430\u043d\u0438\u0435 \u0442\u043e\u0433\u043e, \u0447\u0442\u043e \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0432\u043e \u0432\u0440\u0435\u043c\u044f \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0438 \u0441 \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0435\u043c: \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c KNN<\/h2>\n<p>\u0422\u0435\u043c \u043d\u0435 \u043c\u0435\u043d\u0435\u0435, \u043f\u043e\u0442\u0435\u0440\u0438 \u0432 \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u043e\u043c \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0438 \u043d\u0435 \u044f\u0432\u043b\u044f\u044e\u0442\u0441\u044f \u043d\u0430\u0434\u0435\u0436\u043d\u044b\u043c \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u0435\u043b\u0435\u043c \u0434\u043b\u044f \u043e\u0442\u0441\u043b\u0435\u0436\u0438\u0432\u0430\u043d\u0438\u044f.  \u042f \u043e\u0431\u043d\u0430\u0440\u0443\u0436\u0438\u043b, \u0447\u0442\u043e \u043b\u0443\u0447\u0448\u0438\u0439 \u0441\u043f\u043e\u0441\u043e\u0431 \u043e\u0442\u0441\u043b\u0435\u0436\u0438\u0432\u0430\u0442\u044c, \u0447\u0442\u043e \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0432\u043e \u0432\u0440\u0435\u043c\u044f \u0442\u0440\u0435\u043d\u0438\u0440\u043e\u0432\u043a\u0438, \u2014 \u044d\u0442\u043e \u0438\u0437\u043c\u0435\u0440\u044f\u0442\u044c \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c \u039a\u039d\u039d.<\/p>\n<p>\u0412\u0430\u0436\u043d\u0435\u0439\u0448\u0435\u0435 \u043f\u0440\u0435\u0438\u043c\u0443\u0449\u0435\u0441\u0442\u0432\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f KNN \u0437\u0430\u043a\u043b\u044e\u0447\u0430\u0435\u0442\u0441\u044f \u0432 \u0442\u043e\u043c, \u0447\u0442\u043e \u043d\u0430\u043c \u043d\u0435 \u043d\u0443\u0436\u043d\u043e \u043a\u0430\u0436\u0434\u044b\u0439 \u0440\u0430\u0437 \u043e\u0431\u0443\u0447\u0430\u0442\u044c \u043b\u0438\u043d\u0435\u0439\u043d\u044b\u0439 \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0442\u043e\u0440 \u0441\u0432\u0435\u0440\u0445\u0443, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043e\u043d \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442 \u0431\u044b\u0441\u0442\u0440\u0435\u0435 \u0438 \u043f\u043e\u043b\u043d\u043e\u0441\u0442\u044c\u044e \u043d\u0435 \u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u0442\u0441\u044f.<\/p>\n<p>\u041f\u0440\u0438\u043c\u0435\u0447\u0430\u043d\u0438\u0435. \u0418\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u0435 KNN \u043f\u0440\u0438\u043c\u0435\u043d\u0438\u043c\u043e \u0442\u043e\u043b\u044c\u043a\u043e \u043a \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u0438 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439, \u043d\u043e \u0432\u044b \u043f\u043e\u043d\u044f\u043b\u0438 \u0438\u0434\u0435\u044e.  \u0414\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u044f \u0441\u043e\u0437\u0434\u0430\u043b \u043a\u043b\u0430\u0441\u0441 \u0434\u043b\u044f \u0438\u043d\u043a\u0430\u043f\u0441\u0443\u043b\u044f\u0446\u0438\u0438 \u043b\u043e\u0433\u0438\u043a\u0438 KNN \u0432 \u043d\u0430\u0448\u0435\u043c \u043a\u043e\u043d\u0442\u0435\u043a\u0441\u0442\u0435:<\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> numpy <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">as<\/span><span class=\"token plain\"> np<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> torch<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">from<\/span><span class=\"token plain\"> sklearn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">model_selection <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> cross_val_score<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">from<\/span><span class=\"token plain\"> sklearn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">neighbors <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> KNeighborsClassifier<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">from<\/span><span class=\"token plain\"> torch <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> nn<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">class<\/span><span class=\"token plain\"> <\/span><span class=\"token class-name\">KNN<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> k<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> device<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">super<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">KNN<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">k <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> k<\/span><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">device <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> device<\/span><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">model <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">to<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">device<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">eval<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">extract_features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> loader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">\"\"\"<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Infer\/Extract features from a trained model<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Args:<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           loader: train or test loader<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Returns: 3 tensors of all:  input_images, features, labels<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       \"\"\"<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       x_lst <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">[<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">]<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       features <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">[<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">]<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       label_lst <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">[<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">]<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">with<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">no_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">for<\/span><span class=\"token plain\"> input_tensor<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> label <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">in<\/span><span class=\"token plain\"> loader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               h <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">input_tensor<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">to<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">device<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">append<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">h<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               x_lst<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">append<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">input_tensor<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               label_lst<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">append<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">label<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           x_total <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">stack<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">x_lst<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           h_total <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">stack<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           label_total <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">stack<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">label_lst<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> x_total<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> h_total<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> label_total<\/span><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">knn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> labels<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> k<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">\"\"\"<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Evaluating knn accuracy in feature space.<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Calculates only top-1 accuracy (returns 0 for top-5)<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Args:<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           features: [... , dataset_size, feat_dim]<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           labels: [... , dataset_size]<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           k: nearest neighbours<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Returns: train accuracy, or train and test acc<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       \"\"\"<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       feature_dim <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">shape<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">[<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">]<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">with<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">no_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           features_np <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cpu<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">view<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> feature_dim<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">numpy<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           labels_np <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> labels<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cpu<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">view<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">numpy<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cls <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> KNeighborsClassifier<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">k<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> metric<\/span><span class=\"token operator\">=<\/span><span class=\"token string\" style=\"color:rgb(255, 121, 198)\">\"cosine\"<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">fit<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">features_np<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> labels_np<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           acc <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">eval<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> labels<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> acc<\/span><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">eval<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> labels<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">     feature_dim <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">shape<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">[<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">]<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">     features <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cpu<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">view<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> feature_dim<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">numpy<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">     labels <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> labels<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cpu<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">view<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">numpy<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">     acc <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">100<\/span><span class=\"token plain\"> <\/span><span class=\"token operator\">*<\/span><span class=\"token plain\"> np<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">mean<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">cross_val_score<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cls<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> labels<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">     <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> acc<\/span><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">_find_best_indices<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> h_query<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> h_ref<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       h_query <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> h_query <\/span><span class=\"token operator\">\/<\/span><span class=\"token plain\"> h_query<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">norm<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">dim<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">view<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       h_ref <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> h_ref <\/span><span class=\"token operator\">\/<\/span><span class=\"token plain\"> h_ref<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">norm<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">dim<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">view<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       scores <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">matmul<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">h_query<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> h_ref<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">t<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\">  <\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       score<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> indices <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> scores<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">topk<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> dim<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\">  <\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> score<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> indices<\/span><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">fit<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> train_loader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> test_loader<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">with<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">no_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           x_train<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> h_train<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> l_train <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">extract_features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">train_loader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           train_acc <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">knn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">h_train<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> l_train<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> k<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">k<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">if<\/span><span class=\"token plain\"> test_loader <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">is<\/span><span class=\"token plain\"> <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">not<\/span><span class=\"token plain\"> <\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               x_test<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> h_test<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> l_test <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">extract_features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">test_loader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               test_acc <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">eval<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">h_test<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> l_test<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">               <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> train_acc<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> test_acc<\/span><\/p><\/pre>\n<p>\u0422\u0435\u043f\u0435\u0440\u044c \u043c\u044b \u043c\u043e\u0436\u0435\u043c \u0441\u043e\u0441\u0440\u0435\u0434\u043e\u0442\u043e\u0447\u0438\u0442\u044c\u0441\u044f \u043d\u0430 \u043c\u0435\u0442\u043e\u0434\u0435 \u0438 \u043c\u043e\u0434\u0435\u043b\u0438 BYOL.<\/p>\n<h2 id=\"modify-resnet-add-mlp-projection-heads\">\u0418\u0437\u043c\u0435\u043d\u0438\u0442\u044c resnet: \u0434\u043e\u0431\u0430\u0432\u0438\u0442\u044c \u043f\u0440\u043e\u0435\u043a\u0446\u0438\u043e\u043d\u043d\u044b\u0435 \u0433\u043e\u043b\u043e\u0432\u043a\u0438 MLP<\/h2>\n<p>\u041c\u044b \u043d\u0430\u0447\u043d\u0435\u043c \u0441 \u0431\u0430\u0437\u043e\u0432\u043e\u0439 \u043c\u043e\u0434\u0435\u043b\u0438 (resnet18) \u0438 \u0438\u0437\u043c\u0435\u043d\u0438\u043c \u0435\u0435 \u0434\u043b\u044f \u0441\u0430\u043c\u043e\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f.  \u041f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u0439 \u0441\u043b\u043e\u0439, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u043e\u0431\u044b\u0447\u043d\u043e \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u044e, \u0437\u0430\u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f \u0444\u0443\u043d\u043a\u0446\u0438\u0435\u0439 \u0438\u0434\u0435\u043d\u0442\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u0438.  \u0412\u044b\u0445\u043e\u0434\u043d\u044b\u0435 \u0434\u0430\u043d\u043d\u044b\u0435 resnet18 \u0431\u0443\u0434\u0443\u0442 \u043f\u0435\u0440\u0435\u0434\u0430\u043d\u044b \u043f\u0440\u043e\u0435\u043a\u0442\u043e\u0440\u0443 MLP. <\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> copy<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> torch<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">from<\/span><span class=\"token plain\"> torch <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> nn<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">import<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">functional <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">as<\/span><span class=\"token plain\"> F<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">class<\/span><span class=\"token plain\"> <\/span><span class=\"token class-name\">MLP<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Module<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> dim<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> embedding_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">256<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> hidden_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">2048<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> batch_norm_mlp<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">super<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       norm <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">BatchNorm1d<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">hidden_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"> <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">if<\/span><span class=\"token plain\"> batch_norm_mlp <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">else<\/span><span class=\"token plain\"> nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Identity<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">net <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Sequential<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Linear<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">dim<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> hidden_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           norm<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">ReLU<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">inplace<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Linear<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">hidden_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> embedding_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">forward<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> x<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">net<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">x<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">class<\/span><span class=\"token plain\"> <\/span><span class=\"token class-name\">AddProjHead<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Module<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> in_features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> layer_name<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> hidden_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">4096<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">                embedding_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">256<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> batch_norm_mlp<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">super<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">AddProjHead<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">backbone <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> model<\/span><\/p><p><span class=\"token plain\">       <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">setattr<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">backbone<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> layer_name<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Identity<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">backbone<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">conv1 <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Conv2d<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">64<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> kernel_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> stride<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> padding<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> bias<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">backbone<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">maxpool <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Identity<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">projection <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> MLP<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">in_features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> embedding_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> hidden_size<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">hidden_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> batch_norm_mlp<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">batch_norm_mlp<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">forward<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> x<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> return_embedding<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       embedding <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">backbone<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">x<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">if<\/span><span class=\"token plain\"> return_embedding<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> embedding<\/span><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">projection<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">embedding<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><\/p><\/pre>\n<p>\u042f \u0442\u0430\u043a\u0436\u0435 \u0437\u0430\u043c\u0435\u043d\u0438\u043b \u043f\u0435\u0440\u0432\u044b\u0439 \u043a\u043e\u043d\u0432\u0435\u0440\u0441\u0438\u043e\u043d\u043d\u044b\u0439 \u0441\u043b\u043e\u0439 resnet18 \u0441 \u0441\u0432\u0435\u0440\u0442\u043a\u0438 7&#215;7 \u043d\u0430 3&#215;3, \u0442\u0430\u043a \u043a\u0430\u043a \u043c\u044b \u0438\u0433\u0440\u0430\u0435\u043c \u0441 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f\u043c\u0438 32&#215;32 (CIFAR-10).<\/p>\n<blockquote>\n<p>\u041a\u043e\u0434 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u043d\u0430 GitHub.  \u0415\u0441\u043b\u0438 \u0432\u044b \u043f\u043b\u0430\u043d\u0438\u0440\u0443\u0435\u0442\u0435 \u0437\u0430\u043a\u0440\u0435\u043f\u0438\u0442\u044c \u0441\u0432\u043e\u0438 \u0437\u043d\u0430\u043d\u0438\u044f \u043e Pytorch, \u043c\u044b \u043d\u0430\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0443\u0435\u043c \u0434\u0432\u0435 \u0437\u0430\u043c\u0435\u0447\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043a\u043d\u0438\u0433\u0438: \u00ab\u0413\u043b\u0443\u0431\u043e\u043a\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 PyTorch\u00bb \u043e\u0442 Manning Publications \u0438 \u00ab\u041c\u0430\u0448\u0438\u043d\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0441 PyTorch \u0438 Scikit-Learn\u00bb \u0421\u0435\u0431\u0430\u0441\u0442\u044c\u044f\u043d\u0430 \u0420\u0430\u0448\u043a\u0438.  \u0412\u044b \u0432\u0441\u0435\u0433\u0434\u0430 \u043c\u043e\u0436\u0435\u0442\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043a\u043e\u0434 \u0441\u043a\u0438\u0434\u043a\u0438 35% <strong>\u0431\u043b\u0435\u0439\u0441\u0430\u043c\u043c\u0435\u044021<\/strong> \u0434\u043b\u044f \u0432\u0441\u0435\u0439 \u043f\u0440\u043e\u0434\u0443\u043a\u0446\u0438\u0438 Manning&#8217;s. <\/p>\n<\/blockquote>\n<h2 id=\"the-actual-byol-method\">\u0424\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u043c\u0435\u0442\u043e\u0434 BYOL<\/h2>\n<p>\u0414\u043e \u0441\u0438\u0445 \u043f\u043e\u0440 \u044f \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u0438\u043b \u0432\u0441\u0435 \u0432\u0430\u0436\u043d\u044b\u0435 \u043a\u043e\u043c\u043f\u043e\u043d\u0435\u043d\u0442\u044b \u0434\u043b\u044f \u0434\u043e\u0441\u0442\u0438\u0436\u0435\u043d\u0438\u044f \u044d\u0442\u043e\u0439 \u0442\u043e\u0447\u043a\u0438.  \u0422\u0435\u043f\u0435\u0440\u044c \u043c\u044b \u0441\u043e\u0437\u0434\u0430\u0434\u0438\u043c \u043c\u043e\u0434\u0443\u043b\u044c BYOL \u0441 \u043d\u0430\u0448\u0438\u043c\u0438 \u043b\u044e\u0431\u0438\u043c\u044b\u043c\u0438 \u0441\u0435\u0442\u044f\u043c\u0438 \u0443\u0447\u0435\u043d\u0438\u043a\u043e\u0432 \u0438 \u0443\u0447\u0438\u0442\u0435\u043b\u0435\u0439.  \u041e\u0431\u0440\u0430\u0442\u0438\u0442\u0435 \u0432\u043d\u0438\u043c\u0430\u043d\u0438\u0435, \u0447\u0442\u043e \u0441\u0442\u0443\u0434\u0435\u043d\u0447\u0435\u0441\u043a\u0438\u0439 \u043f\u0440\u0435\u0434\u0438\u043a\u0442\u043e\u0440 MLP \u0438 \u043f\u0440\u043e\u0435\u043a\u0442\u043e\u0440 \u0438\u0434\u0435\u043d\u0442\u0438\u0447\u043d\u044b. <\/p>\n<p>\u041c\u043e\u044f \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f BYOL \u0431\u044b\u043b\u0430 \u043e\u0441\u043d\u043e\u0432\u0430\u043d\u0430 \u043d\u0430 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u0438 lucidrains.  \u042f \u0438\u0437\u043c\u0435\u043d\u0438\u043b \u0435\u0433\u043e, \u0447\u0442\u043e\u0431\u044b \u0441\u0434\u0435\u043b\u0430\u0442\u044c \u0435\u0433\u043e \u0431\u043e\u043b\u0435\u0435 \u043f\u0440\u043e\u0441\u0442\u044b\u043c \u0438 \u043f\u043e\u0438\u0433\u0440\u0430\u0442\u044c \u0441 \u043d\u0438\u043c.<\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">class<\/span><span class=\"token plain\"> <\/span><span class=\"token class-name\">BYOL<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">nn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">Module<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           net<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           batch_norm_mlp<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           layer_name<\/span><span class=\"token operator\">=<\/span><span class=\"token string\" style=\"color:rgb(255, 121, 198)\">'fc'<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           in_features<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">512<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           projection_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">256<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           projection_hidden_size<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">2048<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           moving_average_decay<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">0.99<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           use_momentum<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">\"\"\"<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       Args:<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           net: model to be trained<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           batch_norm_mlp: whether to use batchnorm1d in the mlp predictor and projector<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           in_features: the number features that are produced by the backbone net i.e. resnet<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           projection_size: the size of the output vector of the two identical MLPs<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           projection_hidden_size: the size of the hidden vector of the two identical MLPs<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           augment_fn2: apply different augmentation the second view<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           moving_average_decay: t hyperparameter to control the influence in the target network weight update<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">           use_momentum: whether to update the target network<\/span><\/p><p><span class=\"token triple-quoted-string string\" style=\"color:rgb(255, 121, 198)\">       \"\"\"<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">super<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">__init__<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">net <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> net<\/span><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_model <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> AddProjHead<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">model<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">net<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> in_features<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">in_features<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">                                        layer_name<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">layer_name<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">                                        embedding_size<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">projection_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">                                        hidden_size<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">projection_hidden_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">                                        batch_norm_mlp<\/span><span class=\"token operator\">=<\/span><span class=\"token plain\">batch_norm_mlp<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">use_momentum <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> use_momentum<\/span><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">teacher_model <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">_get_teacher<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">target_ema_updater <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> EMA<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">moving_average_decay<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_predictor <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> MLP<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">projection_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> projection_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> projection_hidden_size<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token decorator annotation punctuation\" style=\"color:rgb(248, 248, 242)\">@torch<\/span><span class=\"token decorator annotation punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token decorator annotation punctuation\" style=\"color:rgb(248, 248, 242)\">no_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">_get_teacher<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> copy<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">deepcopy<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token decorator annotation punctuation\" style=\"color:rgb(248, 248, 242)\">@torch<\/span><span class=\"token decorator annotation punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token decorator annotation punctuation\" style=\"color:rgb(248, 248, 242)\">no_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">update_moving_average<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">assert<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">use_momentum<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> <\/span><span class=\"token string\" style=\"color:rgb(255, 121, 198)\">'you do not need to update the moving average, since you have turned off momentum '<\/span><span class=\"token plain\"> \\<\/span><\/p><p><span class=\"token plain\">                                 <\/span><span class=\"token string\" style=\"color:rgb(255, 121, 198)\">'for the target encoder '<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">assert<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">teacher_model <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">is<\/span><span class=\"token plain\"> <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">not<\/span><span class=\"token plain\"> <\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> <\/span><span class=\"token string\" style=\"color:rgb(255, 121, 198)\">'target encoder has not been created yet'<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">for<\/span><span class=\"token plain\"> student_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> teacher_params <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">in<\/span><span class=\"token plain\"> <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">zip<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">parameters<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">teacher_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">parameters<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">         old_weight<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> up_weight <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> teacher_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">data<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> student_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">data<\/span><\/p><p><span class=\"token plain\">         teacher_params<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">data <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">target_ema_updater<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">update_average<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">old_weight<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> up_weight<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">forward<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           image_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> image_two<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           return_embedding<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">if<\/span><span class=\"token plain\"> return_embedding <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">or<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">image_two <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">is<\/span><span class=\"token plain\"> <\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">image_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> return_embedding<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       student_proj_one <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">image_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       student_proj_two <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">image_two<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       student_pred_one <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_predictor<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">student_proj_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       student_pred_two <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">student_predictor<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">student_proj_two<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">with<\/span><span class=\"token plain\"> torch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">no_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           teacher_proj_one <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">teacher_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">image_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">detach_<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">           teacher_proj_two <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> self<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">teacher_model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">image_two<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">detach_<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       loss_one <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> loss_fn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">student_pred_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> teacher_proj_one<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       loss_two <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> loss_fn<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">student_pred_two<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> teacher_proj_two<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\">       <\/span><\/p><p><span class=\"token plain\">       <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">loss_one <\/span><span class=\"token operator\">+<\/span><span class=\"token plain\"> loss_two<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">mean<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><\/p><\/pre>\n<p>\u0414\u043b\u044f CIFAR-10 \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c 2048 \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0441\u043a\u0440\u044b\u0442\u043e\u0433\u043e \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f \u0438 256 \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0432\u0441\u0442\u0440\u0430\u0438\u0432\u0430\u0435\u043c\u043e\u0433\u043e \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f.  \u041c\u044b \u0431\u0443\u0434\u0435\u043c \u043e\u0431\u0443\u0447\u0430\u0442\u044c resnet18, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0432\u044b\u0432\u043e\u0434\u0438\u0442 512 \u043f\u0440\u0438\u0437\u043d\u0430\u043a\u043e\u0432 \u0437\u0430 100 \u044d\u043f\u043e\u0445.  \u0427\u0430\u0441\u0442\u0438 \u043a\u043e\u0434\u0430, \u043e\u0442\u043d\u043e\u0441\u044f\u0449\u0438\u0435\u0441\u044f \u043a \u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0435 \u0438 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044e \u0434\u0430\u043d\u043d\u044b\u0445, \u043e\u043f\u0443\u0449\u0435\u043d\u044b \u0434\u043b\u044f \u0443\u0434\u043e\u0431\u0441\u0442\u0432\u0430 \u0447\u0442\u0435\u043d\u0438\u044f.  \u0412\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u043f\u043e\u0441\u043c\u043e\u0442\u0440\u0435\u0442\u044c \u0438\u0445 \u0432 \u043a\u043e\u0434\u0435.<\/p>\n<p>\u0412\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0442\u043e\u0440 \u0410\u0434\u0430\u043c\u0430 ( <span class=\"inlineMath\"><span class=\"katex\"><span class=\"katex-mathml\"><math xmlns=\"http:\/\/www.w3.org\/1998\/Math\/MathML\"><semantics><mrow><mi>\u043b<\/mi><mi>\u0440<\/mi><mo>&#8220;=&#8221;<\/mo><mn>3<\/mn><mo>*<\/mo><mn>1<\/mn><msup><mn>0<\/mn><mrow><mo>\u2212<\/mo><mn>4<\/mn><\/mrow><\/msup><\/mrow><annotation encoding=\"application\/x-tex\">\u043b\u0440=3 * 10^{-4}<\/annotation><\/semantics><\/math><\/span><span class=\"katex-html\" aria-hidden=\"true\"><span class=\"base\"><span class=\"strut\" style=\"height:0.69444em;vertical-align:0em\"\/><span class=\"mord mathnormal\" style=\"margin-right:0.01968em\">\u043b<\/span><span class=\"mord mathnormal\" style=\"margin-right:0.02778em\">\u0440<\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><span class=\"mrel\">&#8220;=&#8221;<\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:0.64444em;vertical-align:0em\"\/><span class=\"mord\">3<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><span class=\"mbin\">*<\/span><span class=\"mspace\" style=\"margin-right:0.2222222222222222em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:0.8141079999999999em;vertical-align:0em\"\/><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height:0.8141079999999999em\"><span style=\"top:-3.063em;margin-right:0.05em\"><span class=\"pstrut\" style=\"height:2.7em\"\/><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>  \u043a\u043e\u043d\u0435\u0447\u043d\u043e) \u0438\u043b\u0438 \u041b\u0410\u0420\u0421 \u0441 <span class=\"inlineMath\"><span class=\"katex\"><span class=\"katex-mathml\"><math xmlns=\"http:\/\/www.w3.org\/1998\/Math\/MathML\"><semantics><mrow><mi>\u043b<\/mi><mi>\u0440<\/mi><mo>&#8220;=&#8221;<\/mo><mn>0,1<\/mn><\/mrow><annotation encoding=\"application\/x-tex\">\u043b=0,1<\/annotation><\/semantics><\/math><\/span><span class=\"katex-html\" aria-hidden=\"true\"><span class=\"base\"><span class=\"strut\" style=\"height:0.69444em;vertical-align:0em\"\/><span class=\"mord mathnormal\" style=\"margin-right:0.01968em\">\u043b<\/span><span class=\"mord mathnormal\" style=\"margin-right:0.02778em\">\u0440<\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><span class=\"mrel\">&#8220;=&#8221;<\/span><span class=\"mspace\" style=\"margin-right:0.2777777777777778em\"\/><\/span><span class=\"base\"><span class=\"strut\" style=\"height:0.64444em;vertical-align:0em\"\/><span class=\"mord\">0<\/span><span class=\"mord\">.<\/span><span class=\"mord\">1<\/span><\/span><\/span><\/span><\/span>.  \u0421\u043e\u043e\u0431\u0449\u0435\u043d\u043d\u044b\u0435 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u043e\u0442\u043d\u043e\u0441\u044f\u0442\u0441\u044f \u043a \u0410\u0434\u0430\u043c\u0443, \u043d\u043e \u044f \u0442\u0430\u043a\u0436\u0435 \u043f\u043e\u0434\u0442\u0432\u0435\u0440\u0434\u0438\u043b, \u0447\u0442\u043e KNN \u0443\u0432\u0435\u043b\u0438\u0447\u0438\u0432\u0430\u0435\u0442\u0441\u044f \u0432 \u043f\u0435\u0440\u0432\u044b\u0435 \u044d\u043f\u043e\u0445\u0438 \u0441 LARS.<\/p>\n<p>\u0415\u0434\u0438\u043d\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0435, \u0447\u0442\u043e \u0431\u0443\u0434\u0435\u0442 \u0438\u0437\u043c\u0435\u043d\u0435\u043d\u043e \u0432 \u043a\u043e\u0434\u0435 \u043f\u043e\u0435\u0437\u0434\u0430, \u2014 \u044d\u0442\u043e \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435 EMA.<\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">training_step<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> data<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">view1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> view2<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> _ <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> data<\/span><\/p><p><span class=\"token plain\">   loss <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">view1<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cuda<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> view2<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">cuda<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> loss<\/span><\/p><p><span class=\"token plain\"\/><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">def<\/span><span class=\"token plain\"> <\/span><span class=\"token function\" style=\"color:rgb(80, 250, 123)\">train_one_epoch<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> train_dataloader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> optimizer<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">train<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   total_loss <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> <\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   num_batches <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> <\/span><span class=\"token builtin\" style=\"color:rgb(189, 147, 249)\">len<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">train_dataloader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">for<\/span><span class=\"token plain\"> data <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">in<\/span><span class=\"token plain\"> train_dataloader<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">:<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       optimizer<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">zero_grad<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       loss <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> training_step<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> data<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       loss<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">backward<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       optimizer<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">step<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">update_moving_average<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">       total_loss <\/span><span class=\"token operator\">+=<\/span><span class=\"token plain\"> loss<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">.<\/span><span class=\"token plain\">item<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><span class=\"token plain\"\/><\/p><p><span class=\"token plain\">   <\/span><span class=\"token keyword\" style=\"color:rgb(189, 147, 249);font-style:italic\">return<\/span><span class=\"token plain\"> total_loss<\/span><span class=\"token operator\">\/<\/span><span class=\"token plain\">num_batches<\/span><\/p><\/pre>\n<p>\u0414\u0430\u0432\u0430\u0439\u0442\u0435 \u043f\u0440\u044b\u0433\u0430\u0442\u044c \u043d\u0430 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b!<\/p>\n<h2 id=\"results-knn-accuracy-vs-pretraining-epochs\">\u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b: \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c KNN \u043f\u043e \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044e \u0441 \u044d\u043f\u043e\u0445\u0430\u043c\u0438 \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0438<\/h2>\n<p><span class=\"gatsby-resp-image-wrapper\" style=\"position:relative;display:block;margin-left:auto;margin-right:auto;max-width:832px\"><\/p>\n<p>    <span class=\"gatsby-resp-image-background-image\" style=\"padding-bottom:44.66666666666667%;position:relative;bottom:0;left:0;background-image:url('data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAJCAYAAAAywQxIAAAACXBIWXMAAAsTAAALEwEAmpwYAAABgElEQVQoz6WSW0sCQRTH96Fv0FoKfYaeguoh+qqC9NiroFCIPQhW+J6O4q6XdYVI17lfTmdms3wKogO\/nTk75\/I\/w0SrbHGer7LZcDgcjcdjUhQF0VoTpRSuewxR+I8JhcjAlgmy2VGy2W7Jx2b7Vux2K8bYXZQmyTUmgbUWDOKcg71pCyDxw4UAgUgpQXIasJJhAAenJSIArAIU8hTNZrNL3IR8xHqE0pZxbgWnVoudxYQAGOk7Yowr17AHixKUL4BTPUSDweDGGAPhwIGjAlWwAsv7rgYOzWv3AwS+fecJgSjsMer3+7e2TLSUc2d4EUIPE6BM+q52uIcy5qfgZDK58gq5VAaV2S+liPsLPyNny8W1kKocExz8x1BYN8LHckGLzbuWbI5PY4FdPEu\/6tJf7v1fzlL0C0rpfTQZj46eX16r6TKP0zSJCSGVdrtdm8\/ncZZlcbfbrTabzVqj0TibTqdxq9Wq4TVV8jyPe73eaafTqa7X6+MkSU7q9XrlE1qcmY6vYc8bAAAAAElFTkSuQmCC');background-size:cover;display:block\"\/><br \/>\n  <img decoding=\"async\" class=\"gatsby-resp-image-image\" alt=\"knn-byol-\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435\" title=\"knn-byol-\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435\" src=\"https:\/\/theaisummer.com\/static\/ebc3a14f95818251d3bd2b6b57055d53\/ef6b9\/knn-byol-training.png\" srcset=\"\/static\/ebc3a14f95818251d3bd2b6b57055d53\/5a46d\/knn-byol-training.png 300w,\/static\/ebc3a14f95818251d3bd2b6b57055d53\/0a47e\/knn-byol-training.png 600w,\/static\/ebc3a14f95818251d3bd2b6b57055d53\/ef6b9\/knn-byol-training.png 832w\" sizes=\"(max-width: 832px) 100vw, 832px\" style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0\" loading=\"lazy\"\/><\/p>\n<p>    <\/span><br \/>\n<em>\u0422\u043e\u0447\u043d\u043e\u0441\u0442\u044c KNN \u043a\u0430\u0436\u0434\u044b\u0435 4 \u044d\u043f\u043e\u0445\u0438.  \u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u0430\u0432\u0442\u043e\u0440\u0430<\/em> <\/p>\n<p>\u0420\u0430\u0437\u0432\u0435 \u043d\u0435 \u0443\u0434\u0438\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u043e, \u0447\u0442\u043e \u0431\u0435\u0437 \u043a\u0430\u043a\u0438\u0445-\u043b\u0438\u0431\u043e \u043c\u0435\u0442\u043e\u043a \u043c\u044b \u043c\u043e\u0436\u0435\u043c \u0434\u043e\u0441\u0442\u0438\u0447\u044c \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u0438 \u0432\u0430\u043b\u0438\u0434\u0430\u0446\u0438\u0438 \u0432 70%?  \u042f \u043d\u0430\u0448\u0435\u043b \u044d\u0442\u043e \u0443\u0434\u0438\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u043c, \u043e\u0441\u043e\u0431\u0435\u043d\u043d\u043e \u0434\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u043c\u0435\u0442\u043e\u0434\u0430, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u043a\u0430\u0436\u0435\u0442\u0441\u044f \u043c\u0435\u043d\u0435\u0435 \u0447\u0443\u0432\u0441\u0442\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u043a \u0440\u0430\u0437\u043c\u0435\u0440\u0443 \u043f\u0430\u043a\u0435\u0442\u0430. <\/p>\n<p>\u041d\u043e \u043f\u043e\u0447\u0435\u043c\u0443 \u0437\u0434\u0435\u0441\u044c \u0432\u043b\u0438\u044f\u0435\u0442 \u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u0440\u0442\u0438\u0438?  \u0420\u0430\u0437\u0432\u0435 \u044d\u0442\u043e \u043d\u0435 \u0434\u043e\u043b\u0436\u043d\u043e \u043d\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u043f\u0430\u0440\u0438\u0436?  \u041e\u0442\u043a\u0443\u0434\u0430 \u0431\u0435\u0440\u0435\u0442\u0441\u044f \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u044c \u0440\u0430\u0437\u043c\u0435\u0440\u0430 \u043f\u0430\u0440\u0442\u0438\u0438? <\/p>\n<blockquote>\n<p>\u041a\u0440\u0430\u0442\u043a\u0438\u0439 \u043e\u0442\u0432\u0435\u0442: \u041d\u0443, \u044d\u0442\u043e \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u043f\u0430\u043a\u0435\u0442\u043e\u0432 \u0432 \u0441\u043b\u043e\u044f\u0445 MLP!<\/p>\n<\/blockquote>\n<p>\u0412\u043e\u0442 \u044d\u043a\u0441\u043f\u0435\u0440\u0438\u043c\u0435\u043d\u0442\u044b, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u044f \u043f\u0440\u043e\u0432\u0435\u043b, \u0447\u0442\u043e\u0431\u044b \u043f\u0440\u043e\u0432\u0435\u0440\u0438\u0442\u044c \u044d\u0442\u043e.<\/p>\n<h2 id=\"a-note-on-batch-norm-in-mlp-networks-and-ema-momentum\">\u041f\u0440\u0438\u043c\u0435\u0447\u0430\u043d\u0438\u0435 \u043e \u043f\u0430\u043a\u0435\u0442\u043d\u043e\u0439 \u043d\u043e\u0440\u043c\u0435 \u0432 \u0441\u0435\u0442\u044f\u0445 MLP \u0438 \u043c\u043e\u043c\u0435\u043d\u0442\u0443\u043c\u0435 EMA<\/h2>\n<p>\u041c\u043d\u0435 \u0431\u044b\u043b\u043e \u043b\u044e\u0431\u043e\u043f\u044b\u0442\u043d\u043e \u043d\u0430\u0431\u043b\u044e\u0434\u0430\u0442\u044c \u0437\u0430 \u043a\u0440\u0430\u0445\u043e\u043c \u0440\u0435\u0436\u0438\u043c\u0430 \u0431\u0435\u0437 \u043f\u0430\u043a\u0435\u0442\u043d\u043e\u0439 \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438.  \u0412\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u043f\u043e\u043f\u0440\u043e\u0431\u043e\u0432\u0430\u0442\u044c \u044d\u0442\u043e \u0441\u0430\u043c\u043e\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e, \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u0438\u0432:<\/p>\n<pre class=\"prism-code language-python\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token plain\">model <\/span><span class=\"token operator\">=<\/span><span class=\"token plain\"> BYOL<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">(<\/span><span class=\"token plain\">model<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> in_features<\/span><span class=\"token operator\">=<\/span><span class=\"token number\">512<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">,<\/span><span class=\"token plain\"> batch_norm_mlp<\/span><span class=\"token operator\">=<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\" style=\"color:rgb(248, 248, 242)\">)<\/span><\/p><\/pre>\n<p>\u042f \u0437\u0430\u043c\u0435\u0442\u0438\u043b, \u0447\u0442\u043e \u0440\u0430\u0441\u0441\u0442\u043e\u044f\u043d\u0438\u0435 L2 \u0441\u0442\u0440\u0435\u043c\u0438\u0442\u0441\u044f \u043f\u043e\u0447\u0442\u0438 \u043a \u043d\u0443\u043b\u044e \u0441 \u0441\u0430\u043c\u044b\u0445 \u043f\u0435\u0440\u0432\u044b\u0445 \u044d\u043f\u043e\u0445:<\/p>\n<pre class=\"prism-code language-\" style=\"color:#F8F8F2;background-color:#282A36\"><p><span class=\"token plain\">Epoch 0: loss:0.06423207696957084<\/span><\/p><p><span class=\"token plain\">Epoch 8: loss:0.005584242034894534<\/span><\/p><p><span class=\"token plain\">Epoch 20: loss:0.005460431350347323<\/span><\/p><\/pre>\n<p>\u0423\u0431\u044b\u0442\u043e\u043a \u043f\u0440\u0438\u0431\u043b\u0438\u0436\u0430\u0435\u0442\u0441\u044f \u043a \u043d\u0443\u043b\u044e, \u0438 KNN \u043f\u0435\u0440\u0435\u0441\u0442\u0430\u0435\u0442 \u0440\u0430\u0441\u0442\u0438 (35% \u043f\u0440\u043e\u0442\u0438\u0432 60% \u0432 \u043e\u0431\u044b\u0447\u043d\u043e\u043c \u0440\u0435\u0436\u0438\u043c\u0435).  \u0412\u043e\u0442 \u043f\u043e\u0447\u0435\u043c\u0443 \u0443\u0442\u0432\u0435\u0440\u0436\u0434\u0430\u0435\u0442\u0441\u044f, \u0447\u0442\u043e BYOL <strong>\u043d\u0435\u044f\u0432\u043d\u043e<\/strong> \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442 \u0444\u043e\u0440\u043c\u0443 \u0441\u0440\u0430\u0432\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f, \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044f \u0441\u0442\u0430\u0442\u0438\u0441\u0442\u0438\u043a\u0443 \u043f\u0430\u0440\u0442\u0438\u0438 \u0432 MLP.  \u0412\u043e\u0442 \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u044c KNN:<\/p>\n<p><span class=\"gatsby-resp-image-wrapper\" style=\"position:relative;display:block;margin-left:auto;margin-right:auto;max-width:909px\"><\/p>\n<p>    <span class=\"gatsby-resp-image-background-image\" style=\"padding-bottom:60.333333333333336%;position:relative;bottom:0;left:0;background-image:url('data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAMCAIAAADtbgqsAAAACXBIWXMAAAsTAAALEwEAmpwYAAABBklEQVQoz5VS246FIAz0\/\/\/R5CQm+uDqEeTSUqA7yG726Wx0MqlSGHqhg4g451JKTAS6EEHjwm79J27Wv89g7TlA5r1X1VrVcw6pkBQpWOk\/BGKMAzMjMhYhSUyit9HESDuEgMVJPd4HlKyZ\/9jFx3EQUS7VkWjNmpySvXj+2lMZdCpRU9DUrW\/ieZ7RLJTKFNu5nB6kva4rPo5zRZxantXcGuZ9zdJyu4F64UeMpwqRVEIV1idApy4xui3uTs6RkzFm2zZYdBpiOa1J0eMHF+E+FNIGjjFvVErBWzZn85B5f71er3Ecp2lalmXAdj9aGiqmLeeMquDE8Oz73lOD7U6U2rNA5G9isMDZQyMPHwAAAABJRU5ErkJggg==');background-size:cover;display:block\"\/><br \/>\n  <img decoding=\"async\" class=\"gatsby-resp-image-image\" alt=\"\u0440\u0435\u0436\u0438\u043c-\u043a\u043e\u043b\u043b\u0430\u043f\u0441-\u0431\u0451\u043b\u044c-\u0431\u0435\u0437-\u043f\u0430\u043a\u0435\u0442\u043d\u043e\u0439 \u043d\u043e\u0440\u043c\u044b\" title=\"\u0440\u0435\u0436\u0438\u043c-\u043a\u043e\u043b\u043b\u0430\u043f\u0441-\u0431\u0451\u043b\u044c-\u0431\u0435\u0437-\u043f\u0430\u043a\u0435\u0442\u043d\u043e\u0439 \u043d\u043e\u0440\u043c\u044b\" src=\"https:\/\/theaisummer.com\/static\/ec45bee741d0ca9b9468f3d2d5f5874f\/b3ad9\/mode-collapse-byol-no-batch-norm.png\" srcset=\"\/static\/ec45bee741d0ca9b9468f3d2d5f5874f\/5a46d\/mode-collapse-byol-no-batch-norm.png 300w,\/static\/ec45bee741d0ca9b9468f3d2d5f5874f\/0a47e\/mode-collapse-byol-no-batch-norm.png 600w,\/static\/ec45bee741d0ca9b9468f3d2d5f5874f\/b3ad9\/mode-collapse-byol-no-batch-norm.png 909w\" sizes=\"(max-width: 909px) 100vw, 909px\" style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0\" loading=\"lazy\"\/><\/p>\n<p>    <\/span><br \/>\n<em>\u0421\u0432\u0435\u0440\u0442\u044b\u0432\u0430\u043d\u0438\u0435 \u0440\u0435\u0436\u0438\u043c\u0430 \u0432 BYOL \u043f\u0443\u0442\u0435\u043c \u0443\u0434\u0430\u043b\u0435\u043d\u0438\u044f \u043f\u0430\u043a\u0435\u0442\u043d\u043e\u0439 \u043d\u043e\u0440\u043c\u044b \u0432 MLP.  \u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u0430\u0432\u0442\u043e\u0440\u0430<\/em> <\/p>\n<p>\u041c\u043d\u0435 \u0445\u043e\u0440\u043e\u0448\u043e \u0438\u0437\u0432\u0435\u0441\u0442\u043d\u044b \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u044b, \u043f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u044e\u0449\u0438\u0435, \u0447\u0442\u043e \u043f\u0430\u043a\u0435\u0442\u043d\u0430\u044f \u0441\u0442\u0430\u0442\u0438\u0441\u0442\u0438\u043a\u0430 \u2014 \u043d\u0435 \u0435\u0434\u0438\u043d\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0435 \u0443\u0441\u043b\u043e\u0432\u0438\u0435 \u0434\u043b\u044f \u0440\u0430\u0431\u043e\u0442\u044b BYOL.  \u042d\u0442\u043e \u044d\u043a\u0441\u043f\u0435\u0440\u0438\u043c\u0435\u043d\u0442\u0430\u043b\u044c\u043d\u044b\u0439 \u043f\u043e\u0441\u0442, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u044f \u043d\u0435 \u0431\u0443\u0434\u0443 \u0438\u0433\u0440\u0430\u0442\u044c \u0432 \u044d\u0442\u0443 \u0438\u0433\u0440\u0443.  \u041c\u043d\u0435 \u043f\u0440\u043e\u0441\u0442\u043e \u0431\u044b\u043b\u043e \u043b\u044e\u0431\u043e\u043f\u044b\u0442\u043d\u043e \u043f\u043e\u043d\u0430\u0431\u043b\u044e\u0434\u0430\u0442\u044c \u0437\u0434\u0435\u0441\u044c \u0437\u0430 \u043a\u043e\u043b\u043b\u0430\u043f\u0441\u043e\u043c \u0440\u0435\u0436\u0438\u043c\u0430.<\/p>\n<h2 id=\"conclusion\">\u0417\u0430\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0435<\/h2>\n<p>\u0414\u043b\u044f \u0431\u043e\u043b\u0435\u0435 \u043f\u043e\u0434\u0440\u043e\u0431\u043d\u043e\u0433\u043e \u043e\u0431\u044a\u044f\u0441\u043d\u0435\u043d\u0438\u044f \u043c\u0435\u0442\u043e\u0434\u0430 \u043f\u043e\u0441\u043c\u043e\u0442\u0440\u0438\u0442\u0435 \u0432\u0438\u0434\u0435\u043e \u042f\u043d\u043d\u0438\u043a\u0430 \u043d\u0430 BYOL:<\/p>\n<div class=\"youtube-embed\" data-video_id=\"YPfUiOMYOEE\"><iframe loading=\"lazy\" title=\"BYOL: Bootstrap Your Own Latent: A New Approach to Self-Supervised Learning (Paper Explained)\" width=\"696\" height=\"392\" src=\"https:\/\/www.youtube.com\/embed\/YPfUiOMYOEE?feature=oembed&#038;enablejsapi=1\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" allowfullscreen><\/iframe><\/div>\n<p>\u0412 \u044d\u0442\u043e\u043c \u0440\u0443\u043a\u043e\u0432\u043e\u0434\u0441\u0442\u0432\u0435 \u043c\u044b \u0448\u0430\u0433 \u0437\u0430 \u0448\u0430\u0433\u043e\u043c \u0432\u043d\u0435\u0434\u0440\u0438\u043b\u0438 BYOL \u0438 \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u043e\u0431\u0443\u0447\u0438\u043b\u0438 CIFAR10.  \u041c\u044b \u043d\u0430\u0431\u043b\u044e\u0434\u0430\u0435\u043c \u0437\u043d\u0430\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0443\u0432\u0435\u043b\u0438\u0447\u0435\u043d\u0438\u0435 \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u0438 KNN \u0437\u0430 \u0441\u0447\u0435\u0442 \u0441\u043e\u043f\u043e\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0439 \u043e\u0434\u043d\u043e\u0433\u043e \u0438 \u0442\u043e\u0433\u043e \u0436\u0435 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f.  \u0421\u043b\u0443\u0447\u0430\u0439\u043d\u044b\u0439 \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0442\u043e\u0440 \u0431\u0443\u0434\u0435\u0442 \u0438\u043c\u0435\u0442\u044c 10%, \u0430 \u0441\u043e 100 \u044d\u043f\u043e\u0445\u0430\u043c\u0438 \u043c\u044b \u0434\u043e\u0441\u0442\u0438\u0433\u043d\u0435\u043c 70% \u0442\u043e\u0447\u043d\u043e\u0441\u0442\u0438 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0438 KNN \u0431\u0435\u0437 \u043a\u0430\u043a\u0438\u0445-\u043b\u0438\u0431\u043e \u043c\u0435\u0442\u043e\u043a.  \u041a\u0430\u043a \u044d\u0442\u043e \u043a\u0440\u0443\u0442\u043e?<\/p>\n<p>\u0427\u0442\u043e\u0431\u044b \u0443\u0437\u043d\u0430\u0442\u044c \u0431\u043e\u043b\u044c\u0448\u0435 \u043e\u0431 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0438 \u0441 \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0435\u043c, \u0441\u043b\u0435\u0434\u0438\u0442\u0435 \u0437\u0430 \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u044f\u043c\u0438!  \u041f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0442\u0435 \u043d\u0430\u0441, \u043f\u043e\u0434\u0435\u043b\u0438\u0432\u0448\u0438\u0441\u044c \u0432 \u0441\u043e\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u0435\u0442\u044f\u0445, \u0441\u0434\u0435\u043b\u0430\u0432 \u043f\u043e\u0436\u0435\u0440\u0442\u0432\u043e\u0432\u0430\u043d\u0438\u0435 \u0438\u043b\u0438 \u043a\u0443\u043f\u0438\u0432 \u043d\u0430\u0448\u0443 \u043a\u043d\u0438\u0433\u0443 \u00ab\u0413\u043b\u0443\u0431\u043e\u043a\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0432 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0441\u0442\u0432\u0435\u00bb.  \u0411\u044b\u043b\u043e \u0431\u044b \u0432\u0435\u0441\u044c\u043c\u0430 \u043f\u0440\u0438\u0437\u043d\u0430\u0442\u0435\u043b\u0435\u043d.<\/p>\n<div class=\"dl-prod-book-inline-banner\">\n<div class=\"dl-prod-book-inline-banner__image gatsby-image-wrapper\" style=\"position:relative;overflow:hidden\"><img decoding=\"async\" aria-hidden=\"true\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAZCAYAAAAxFw7TAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFzklEQVQ4y1WU2VOb1xnGzyfVbt1O6tST4AVjglc2s1sskgxC8GlBIAmQkMRugtlBbAJjwAKMDYbYYOyMSTJt46YzSV1PO2k7uWimnfauF532or3oTC\/7L3R6+ev7SaGeXjxzzrf9znOe9\/2OUtm1qAs3UVl2tOybmLJlvGBDy7Kina9JyZRSNWaRKVPGzCq0c5WYDJ21oE6Xp5VRhlI5TtR79WmdrkG9UykPqmQuOlONelc+OCUvn5KX3y5B\/UD0drG8Z9wrRZ0sFri8a4DPiNRlF2aRAQyPrxNff87U2gFtw6tU+keIjCWJTawzvvKY2Y0Dhha2GVnaQY\/FCQ3d5dbsfTFQgSlLzIh7pa75UBdFWW5qe1fpTuzROb\/PVX2Y0uA0ocltAiMbtE9sEZ3ZJTi6SfPQBs7uRapDs\/iH1sStwM7ZUedtAsz1425uIhDw4vU6OVdSj8ftoMxWj8\/bwPVqB5lFdgotdhx1VtwNVoJuK9fK7ehOGzeqqrHW1lNgcXDich1KK2zn4wUPP7zj4XfbLg4THkZ7mni55ObpjJsv111M9Xn40xOdtqAH3ethc9TFyzs6S+\/r7Md1\/rDVwMsFJyfzpB7HSjqo8wXJqGynJ9aC2+9nsLuZtrbmFLjR18xkvwe\/30NvxEul04tN99Ag4NsxF5FWF\/W6Tle7zndzddlySSeqMIa6HkXlRVD5HaIwKleUJ7oYxJwXSs8vNcv9AOpKi8xFl+X6klGDJimqV649AizrRSvrQZV2YyrtRCuJoRVHMZfIAvltnKrpZ2f2EYVTEr61H5PAtII2TPmtaHkBNKmBltuCKVfgRoFVxQCq\/JaoH1XWh7GAAVeFEU44pjmdPGQucR9HZBLL4h6ZvoS4bJXn4rgglFpU5QXTzgWuVOUQyjKIuvE+R3BzWT9V\/juci39IxsAqnzbO0FQywFnnAGUrh1wJJMWlRHO94w24oD0NVzVjqOpRVNUIpqrhFNTqWmR5ag9\/KMFaU4KvJp9i8cxhzumg0NZLT\/QhVyvku6IIWpFEcz3yBq7scZRtSvKZQKsZF\/AYbk+cj1pn+bN+l78PPqY3usj3yjo4W9xLpkDHJp7SPvYzcSTAUsm\/uEskxS2S4qraedTNWdEMJvu0wOP01Y\/yr+ga\/+zZpcEhkRRIPvkBsotDXL0Wxj\/5gqEP\/yKuelI7Mkv+ZqO4BlzV30E5FlB1CUx1afi9lkW+GNgkw3FbYEGOF4c5WRqmqCjM3PkAv50\/YPfVP8gqHybTcpu3JH\/zUWGVvopqWEY575KCO5fIk9yOX5B\/M8fD8Ypuvl8RJV968UVBN\/+Ww4CvP+WvyedU3JjkRPUI\/1dY5d1AuaXHXPdIwRu\/gVtkVeOczGkkp7yTr9sm4bND+M0hf5u7z4B1ireMzK2i6rFUUZVRVNX8AOXbRDUJ2LMucGkJgWvG3LnIaWlW50U3fzx4xn9+\/3O+jK4SqZrnlMSkORJoRv5G9rKAqpkQoH8L1fKQFPgI3iJjYBtTaI9vh5+SaRujtGaQYNMyDlkkuy2JCm+jWh+ljegr\/K8WKZgBaN5MSZO56UjGvcADLi39iNatz7jSucV3\/Gt8yye78Bm7WMakG7nPCUxc1o4bW06DlO9+esuNRpbiQJexRa7DmxzresC127uUDz\/h3S5x1pRMO7LLb2g1YIvfdIp0iQHSDJhrjY6t1+grP6XjyS9p2X2NY\/MLYge\/5uba57Q+ek388Cv6nv0Kz84rIvu\/IPrgc2pnDjmmL6XaLr1lo8qpSic5G9sls2+P4AeviG3\/BHviCRVy7NfM7BBc2Wf00QtCy7s4kp9gufdj3uvbJVt+Q5PkmgYmDKC0jCeZVuNKKuCs4cf0LD\/j4foMQ4tTDC8MMZ24JadOHwtLg1intzgzsifZiSPb7BtYastHMKNdjFECvzoubjY+YmL1OcH5PYpHdsjo2uKdzodcuLVNSXyf7DEBSlG0+oU3sLo5\/gubMZbW7M93hgAAAABJRU5ErkJggg==\" alt=\"\u041a\u043d\u0438\u0433\u0430 \u00ab\u0413\u043b\u0443\u0431\u043e\u043a\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0432 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0441\u0442\u0432\u0435\u00bb\" style=\"position:absolute;top:0;left:0;width:100%;height:100%;object-fit:contain;object-position:center;opacity:1;transition-delay:500ms\"\/><noscript><picture><source srcset=\"https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/69585\/deep-learning-book-cover.png 200w,&#10;https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/497c6\/deep-learning-book-cover.png 400w,&#10;https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/3c17d\/deep-learning-book-cover.png 720w\" sizes=\"(max-width: 720px) 100vw, 720px\"\/><img decoding=\"async\" loading=\"lazy\" sizes=\"(max-width: 720px) 100vw, 720px\" srcset=\"https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/69585\/deep-learning-book-cover.png 200w,&#10;https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/497c6\/deep-learning-book-cover.png 400w,&#10;https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/3c17d\/deep-learning-book-cover.png 720w\" src=\"https:\/\/theaisummer.com\/static\/502e7c498dd9d981ac44c1dcd10f9276\/3c17d\/deep-learning-book-cover.png\" alt=\"\u041a\u043d\u0438\u0433\u0430 \u00ab\u0413\u043b\u0443\u0431\u043e\u043a\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0432 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0441\u0442\u0432\u0435\u00bb\" style=\"position:absolute;top:0;left:0;opacity:1;width:100%;height:100%;object-fit:cover;object-position:center\"\/><\/picture><\/noscript><\/div>\n<div class=\"dl-prod-book-inline-banner__text\">\n<h2>\u041a\u043d\u0438\u0433\u0430 \u00ab\u0413\u043b\u0443\u0431\u043e\u043a\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0432 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0441\u0442\u0432\u0435\u00bb \ud83d\udcd6<\/h2>\n<h4>\u0423\u0437\u043d\u0430\u0439\u0442\u0435, \u043a\u0430\u043a \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c, \u043e\u0431\u0443\u0447\u0430\u0442\u044c, \u0440\u0430\u0437\u0432\u0435\u0440\u0442\u044b\u0432\u0430\u0442\u044c, \u043c\u0430\u0441\u0448\u0442\u0430\u0431\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0438 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0442\u044c \u043c\u043e\u0434\u0435\u043b\u0438 \u0433\u043b\u0443\u0431\u043e\u043a\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f.  \u0418\u0437\u0443\u0447\u0438\u0442\u0435 \u0438\u043d\u0444\u0440\u0430\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u0443 \u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0438 MLOps \u043d\u0430 \u043f\u0440\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u043f\u0440\u0438\u043c\u0435\u0440\u0430\u0445.<\/h4>\n<p>\u0423\u0437\u043d\u0430\u0442\u044c \u0431\u043e\u043b\u044c\u0448\u0435<\/p><\/div>\n<\/div>\n<p><em class=\"affiliate-disclosure\">* \u0420\u0430\u0441\u043a\u0440\u044b\u0442\u0438\u0435 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438: \u043e\u0431\u0440\u0430\u0442\u0438\u0442\u0435 \u0432\u043d\u0438\u043c\u0430\u043d\u0438\u0435, \u0447\u0442\u043e \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0438\u0437 \u043f\u0440\u0438\u0432\u0435\u0434\u0435\u043d\u043d\u044b\u0445 \u0432\u044b\u0448\u0435 \u0441\u0441\u044b\u043b\u043e\u043a \u043c\u043e\u0433\u0443\u0442 \u0431\u044b\u0442\u044c \u043f\u0430\u0440\u0442\u043d\u0435\u0440\u0441\u043a\u0438\u043c\u0438 \u0441\u0441\u044b\u043b\u043a\u0430\u043c\u0438, \u0438 \u043c\u044b \u0431\u0435\u0437 \u0434\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0445 \u0437\u0430\u0442\u0440\u0430\u0442 \u0434\u043b\u044f \u0432\u0430\u0441 \u043f\u043e\u043b\u0443\u0447\u0438\u043c \u043a\u043e\u043c\u0438\u0441\u0441\u0438\u044e, \u0435\u0441\u043b\u0438 \u0432\u044b \u0440\u0435\u0448\u0438\u0442\u0435 \u0441\u043e\u0432\u0435\u0440\u0448\u0438\u0442\u044c \u043f\u043e\u043a\u0443\u043f\u043a\u0443 \u043f\u043e\u0441\u043b\u0435 \u043f\u0435\u0440\u0435\u0445\u043e\u0434\u0430 \u043f\u043e \u0441\u0441\u044b\u043b\u043a\u0435.<\/em><\/p>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>\u041f\u043e\u0441\u043b\u0435 \u043f\u0440\u0435\u0437\u0435\u043d\u0442\u0430\u0446\u0438\u0438 SimCLR \u043a\u043e\u043d\u0442\u0440\u0430\u0441\u0442\u043d\u044b\u0439 \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u043e\u0439 \u0441\u0438\u0441\u0442\u0435\u043c\u044b \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f, \u044f \u0440\u0435\u0448\u0438\u043b \u043f\u0440\u043e\u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0435\u0449\u0435 \u043e\u0434\u0438\u043d \u043f\u0435\u0447\u0430\u043b\u044c\u043d\u043e \u0438\u0437\u0432\u0435\u0441\u0442\u043d\u044b\u0439 \u043c\u0435\u0442\u043e\u0434, \u043d\u0430\u0437\u044b\u0432\u0430\u0435\u043c\u044b\u0439 BYOL. Bootstrap Your Own Latent (BYOL) \u2014 \u044d\u0442\u043e \u043d\u043e\u0432\u044b\u0439 \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f\u043c \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439. BYOL \u0438\u043c\u0435\u0435\u0442 \u0434\u0432\u0430 \u043e\u0441\u043d\u043e\u0432\u043d\u044b\u0445 \u043f\u0440\u0435\u0438\u043c\u0443\u0449\u0435\u0441\u0442\u0432\u0430: \u041e\u043d \u044f\u0432\u043d\u043e \u043d\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442 \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043e\u0431\u0440\u0430\u0437\u0446\u044b. \u0412\u043c\u0435\u0441\u0442\u043e \u044d\u0442\u043e\u0433\u043e \u043e\u043d \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u043c\u0438\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u0443\u0435\u0442 \u0441\u0445\u043e\u0434\u0441\u0442\u0432\u043e \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0439 \u043e\u0434\u043d\u043e\u0433\u043e \u0438 \u0442\u043e\u0433\u043e \u0436\u0435 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f \u0432 \u0434\u0440\u0443\u0433\u043e\u043c [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":972,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":{"0":"post-971","1":"post","2":"type-post","3":"status-publish","4":"format-standard","5":"has-post-thumbnail","7":"category-ai-research-and-news"},"_links":{"self":[{"href":"https:\/\/gptmain.news\/index.php?rest_route=\/wp\/v2\/posts\/971","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/gptmain.news\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/gptmain.news\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/gptmain.news\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/gptmain.news\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=971"}],"version-history":[{"count":0,"href":"https:\/\/gptmain.news\/index.php?rest_route=\/wp\/v2\/posts\/971\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/gptmain.news\/index.php?rest_route=\/wp\/v2\/media\/972"}],"wp:attachment":[{"href":"https:\/\/gptmain.news\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=971"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/gptmain.news\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=971"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/gptmain.news\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=971"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}