{"id":1021,"date":"2023-08-10T02:53:01","date_gmt":"2023-08-10T02:53:01","guid":{"rendered":"https:\/\/www.gptmain.news\/?p=1021"},"modified":"2023-08-10T02:53:01","modified_gmt":"2023-08-10T02:53:01","slug":"%d1%83%d1%87%d0%b5%d0%b1%d0%bd%d0%be%d0%b5-%d0%bf%d0%be%d1%81%d0%be%d0%b1%d0%b8%d0%b5-%d0%b4%d0%bb%d1%8f-%d1%81%d0%b0%d0%bc%d0%be%d1%81%d1%82%d0%be%d1%8f%d1%82%d0%b5%d0%bb%d1%8c%d0%bd%d0%be%d0%b3","status":"publish","type":"post","link":"https:\/\/gptmain.news\/?p=1021","title":{"rendered":"\u0423\u0447\u0435\u0431\u043d\u043e\u0435 \u043f\u043e\u0441\u043e\u0431\u0438\u0435 \u0434\u043b\u044f \u0441\u0430\u043c\u043e\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f: \u0420\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f SimCLR \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e \u043c\u043e\u043b\u043d\u0438\u0438 pytorch\n | GPTMain News"},"content":{"rendered":"
\n

\u0412 \u044d\u0442\u043e\u043c \u043f\u0440\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u043e\u043c \u0440\u0443\u043a\u043e\u0432\u043e\u0434\u0441\u0442\u0432\u0435 \u043c\u044b \u043f\u0440\u0435\u0434\u043e\u0441\u0442\u0430\u0432\u0438\u043c \u0432\u0430\u043c \u043f\u043e\u0432\u0442\u043e\u0440\u043d\u0443\u044e \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044e \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u043e\u0433\u043e \u043c\u0435\u0442\u043e\u0434\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f SimCLR \u0434\u043b\u044f \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0438 \u043d\u0430\u0434\u0435\u0436\u043d\u044b\u0445 \u044d\u043a\u0441\u0442\u0440\u0430\u043a\u0442\u043e\u0440\u043e\u0432 \u0444\u0443\u043d\u043a\u0446\u0438\u0439. \u042d\u0442\u043e\u0442 \u043c\u0435\u0442\u043e\u0434 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0434\u043e\u0432\u043e\u043b\u044c\u043d\u043e \u043e\u0431\u0449\u0438\u043c \u0438 \u043c\u043e\u0436\u0435\u0442 \u043f\u0440\u0438\u043c\u0435\u043d\u044f\u0442\u044c\u0441\u044f \u043a \u043b\u044e\u0431\u043e\u043c\u0443 \u043d\u0430\u0431\u043e\u0440\u0443 \u0434\u0430\u043d\u043d\u044b\u0445 \u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0433\u043e \u0437\u0440\u0435\u043d\u0438\u044f, \u0430 \u0442\u0430\u043a\u0436\u0435 \u043a \u0440\u0430\u0437\u043b\u0438\u0447\u043d\u044b\u043c \u043f\u043e\u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u0437\u0430\u0434\u0430\u0447\u0430\u043c.<\/p>\n

\u0412 \u043f\u0440\u0435\u0434\u044b\u0434\u0443\u0449\u0435\u043c \u0443\u0440\u043e\u043a\u0435 \u044f \u043d\u0435\u043c\u043d\u043e\u0433\u043e \u0440\u0430\u0441\u0441\u043a\u0430\u0437\u0430\u043b \u043e\u0431 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0438 \u0441 \u0441\u0430\u043c\u043e\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0435\u043c. \u041f\u0440\u0438\u0448\u043b\u043e \u0432\u0440\u0435\u043c\u044f \u043f\u0440\u0438\u0441\u0442\u0443\u043f\u0438\u0442\u044c \u043a \u0432\u0430\u0448\u0435\u043c\u0443 \u043f\u0435\u0440\u0432\u043e\u043c\u0443 \u043f\u0440\u043e\u0435\u043a\u0442\u0443, \u0437\u0430\u043f\u0443\u0441\u0442\u0438\u0432 SimCLR \u043d\u0430 \u043d\u0435\u0431\u043e\u043b\u044c\u0448\u043e\u043c \u043d\u0430\u0431\u043e\u0440\u0435 \u0434\u0430\u043d\u043d\u044b\u0445 \u0441\u043e 100\u00a0000 \u043d\u0435\u043c\u0430\u0440\u043a\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0445 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439 \u043f\u043e\u0434 \u043d\u0430\u0437\u0432\u0430\u043d\u0438\u0435\u043c STL10. <\/p>\n

\n

\u041a\u043e\u0434 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u043d\u0430 Github. <\/p>\n<\/blockquote>\n

\u041c\u0435\u0442\u043e\u0434 SimCLR: \u043a\u043e\u043d\u0442\u0440\u0430\u0441\u0442\u043d\u043e\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435<\/h2>\n

\u041f\u043e\u0437\u0432\u043e\u043b\u044f\u0442\u044c \u0441<\/mi>\u044f<\/mi>\u043c<\/mi>(<\/mo>\u0442\u044b<\/mi>,<\/mo>\u0432<\/mi>)<\/mo><\/mrow>\u0441\u0438\u043c (\u0438, \u0432)<\/annotation><\/semantics><\/math><\/span>\u0441<\/span>\u044f<\/span>\u043c<\/span>(<\/span>\u0442\u044b<\/span>,<\/span>\u0432<\/span>)<\/span><\/span><\/span><\/span><\/span> \u043e\u0431\u0440\u0430\u0442\u0438\u0442\u0435 \u0432\u043d\u0438\u043c\u0430\u043d\u0438\u0435 \u043d\u0430 \u0441\u043a\u0430\u043b\u044f\u0440\u043d\u043e\u0435 \u043f\u0440\u043e\u0438\u0437\u0432\u0435\u0434\u0435\u043d\u0438\u0435 \u043c\u0435\u0436\u0434\u0443 2 \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d\u043d\u044b\u043c\u0438 \u0442\u044b<\/mi><\/mrow>\u0442\u044b<\/annotation><\/semantics><\/math><\/span>\u0442\u044b<\/span><\/span><\/span><\/span><\/span> \u0438 \u0432<\/mi><\/mrow>\u0432<\/annotation><\/semantics><\/math><\/span>\u0432<\/span><\/span><\/span><\/span><\/span> \u0432\u0435\u043a\u0442\u043e\u0440\u043e\u0432 (\u0442.\u0435. \u043a\u043e\u0441\u0438\u043d\u0443\u0441\u043d\u043e\u0435 \u0441\u0445\u043e\u0434\u0441\u0442\u0432\u043e). <\/p>\n

\u0422\u043e\u0433\u0434\u0430 \u0444\u0443\u043d\u043a\u0446\u0438\u044f \u043f\u043e\u0442\u0435\u0440\u044c \u0434\u043b\u044f \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043f\u0430\u0440\u044b \u043f\u0440\u0438\u043c\u0435\u0440\u043e\u0432 (i,j) \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u044f\u0435\u0442\u0441\u044f \u043a\u0430\u043a:<\/p>\n

\u2113<\/mi>\u044f<\/mi>,<\/mo>\u0414\u0436<\/mi><\/mrow><\/msub>“=”<\/mo>\u2212<\/mo>\u0431\u0440\u0435\u0432\u043d\u043e<\/mi>\u2061<\/mo>\u043e\u043f\u044b\u0442<\/mi>\u2061<\/mo>(<\/mo>\u0441\u0438\u043c<\/mi>\u2061<\/mo>(<\/mo>\u0433<\/mi>\u044f<\/mi><\/msub>,<\/mo>\u0433<\/mi>\u0414\u0436<\/mi><\/msub>)<\/mo><\/mrow>\/<\/mi>\u0442<\/mi>)<\/mo><\/mrow><\/mrow>\u2211<\/mo>\u043a<\/mi>“=”<\/mo>1<\/mn><\/mrow>2<\/mn>\u041d<\/mi><\/mrow><\/munderover>1<\/mn>[<\/mo>k<\/mi>\u2260<\/mo>i<\/mi>]<\/mo><\/mrow><\/msub>\u043e\u043f\u044b\u0442<\/mi>\u2061<\/mo>(<\/mo>\u0441\u0438\u043c<\/mi>\u2061<\/mo>(<\/mo>\u0433<\/mi>\u044f<\/mi><\/msub>,<\/mo>\u0433<\/mi>\u043a<\/mi><\/msub>)<\/mo><\/mrow>\/<\/mi>\u0442<\/mi>)<\/mo><\/mrow><\/mrow><\/mfrac><\/mrow>\\ell_{i, j}=-\\log \\frac{\\exp\\left(\\operatorname{sim}\\left(\\boldsymbol{z}_{i}, \\boldsymbol{z}_{j}\\right) \/ \\\u0442\u0430\u0443\\\u0441\u043f\u0440\u0430\u0432\u0430)}{\\sum_{k=1}^{2 N} \\mathbb{1}_{[k \\neq i]} \\exp \\left(\\operatorname{sim}\\left(\\boldsymbol{z}_{i}, \\boldsymbol{z}_{k}\\right) \/ \\tau\\right)}<\/annotation><\/semantics><\/math><\/span>\u2113<\/span>\u044f<\/span>,<\/span>\u0414\u0436<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>“=”<\/span><\/span>\u2212<\/span>\u0432\u043e\u0442\u0433<\/span><\/span>\u2211<\/span>\u043a<\/span>“=”<\/span>1<\/span><\/span><\/span><\/span>2<\/span>\u041d<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>1<\/span><\/span>[<\/span>k<\/span>\ue020<\/span><\/span><\/span><\/span><\/span><\/span>=<\/span><\/span>i<\/span>]<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u043e\u043f\u044b\u0442<\/span>(<\/span>\u0441<\/span>\u044f<\/span>\u043c<\/span><\/span>(<\/span>\u0433<\/span><\/span><\/span>\u044f<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>,<\/span>\u0433<\/span><\/span><\/span>\u043a<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>)<\/span><\/span>\/<\/span>\u0442<\/span>)<\/span><\/span><\/span><\/span><\/span>\u043e\u043f\u044b\u0442<\/span>(<\/span>\u0441<\/span>\u044f<\/span>\u043c<\/span><\/span>(<\/span>\u0433<\/span><\/span><\/span>\u044f<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>,<\/span>\u0433<\/span><\/span><\/span>\u0414\u0436<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>)<\/span><\/span>\/<\/span>\u0442<\/span>)<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/div>\n

\u0433\u0434\u0435 1<\/mn>[<\/mo>k<\/mi>\u2260<\/mo>i<\/mi>]<\/mo><\/mrow><\/msub>\u0435<\/mo>0<\/mn>,<\/mo>1<\/mn><\/mrow><\/mrow>\\mathbb{1}_{[k \\neq i]} \\\u0432 {0,1}<\/annotation><\/semantics><\/math><\/span>1<\/span><\/span>[<\/span>k<\/span>\ue020<\/span><\/span><\/span><\/span><\/span><\/span>=<\/span><\/span>i<\/span>]<\/span><\/span><\/span><\/span><\/span>\u200b<\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u0435<\/span><\/span>0<\/span>,<\/span>1<\/span><\/span><\/span><\/span><\/span><\/span> \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0438\u043d\u0434\u0438\u043a\u0430\u0442\u043e\u0440\u043d\u043e\u0439 \u0444\u0443\u043d\u043a\u0446\u0438\u0435\u0439, \u0432\u044b\u0447\u0438\u0441\u043b\u044f\u044e\u0449\u0435\u0439 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 1 \u0442\u043e\u0433\u0434\u0430 \u0438 \u0442\u043e\u043b\u044c\u043a\u043e \u0442\u043e\u0433\u0434\u0430, \u043a\u043e\u0433\u0434\u0430 \u043a<\/mi>!<\/mo>“=”<\/mo>\u044f<\/mi><\/mrow>\u043a != \u044f<\/annotation><\/semantics><\/math><\/span>\u043a<\/span>!<\/span>“=”<\/span><\/span>\u044f<\/span><\/span><\/span><\/span><\/span>. \u0414\u043b\u044f \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u0438\u044f \u0434\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0438 \u043e\u0431 \u044d\u0442\u043e\u043c \u043f\u0440\u043e\u0432\u0435\u0440\u044c\u0442\u0435, \u043a\u0430\u043a \u043c\u044b \u0441\u043e\u0431\u0438\u0440\u0430\u0435\u043c\u0441\u044f \u0438\u043d\u0434\u0435\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043c\u0430\u0442\u0440\u0438\u0446\u0443 \u043f\u043e\u0434\u043e\u0431\u0438\u044f, \u0447\u0442\u043e\u0431\u044b \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u0438 \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u0441\u0442\u043e\u0440\u043e\u043d\u044b.<\/p>\n

\u0442<\/mi><\/mrow>\\\u0442\u0430\u0443<\/annotation><\/semantics><\/math><\/span>\u0442<\/span><\/span><\/span><\/span><\/span> \u043e\u0431\u043e\u0437\u043d\u0430\u0447\u0430\u0435\u0442 \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u043d\u044b\u0439 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440. \u041e\u043a\u043e\u043d\u0447\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u0443\u0431\u044b\u0442\u043e\u043a \u0432\u044b\u0447\u0438\u0441\u043b\u044f\u0435\u0442\u0441\u044f \u043f\u0443\u0442\u0435\u043c \u0441\u0443\u043c\u043c\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0432\u0441\u0435\u0445 \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0445 \u043f\u0430\u0440 \u0438 \u0434\u0435\u043b\u0435\u043d\u0438\u044f \u043d\u0430 2<\/mn>\u00d7<\/mo>\u041d<\/mi>“=”<\/mo>\u0432<\/mi>\u044f<\/mi>\u0435<\/mi>\u0436<\/mi>\u0441<\/mi>\u00d7<\/mo>\u0431<\/mi>\u0430<\/mi>\u0442<\/mi>\u0441<\/mi>\u0447\u0430\u0441<\/mi>_<\/mi>\u0441<\/mi>\u044f<\/mi>\u0433<\/mi>\u0435<\/mi><\/mrow>2\\x N = \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u044b \\time \u043f\u0430\u043a\u0435\u0442\\_size<\/annotation><\/semantics><\/math><\/span>2<\/span>\u00d7<\/span><\/span>\u041d<\/span>“=”<\/span><\/span>\u0432<\/span>\u044f<\/span>\u0435<\/span>\u0436<\/span>\u0441<\/span>\u00d7<\/span><\/span>\u0431<\/span>\u0430<\/span>\u0442<\/span>\u0441<\/span>\u0447\u0430\u0441<\/span>_<\/span>\u0441<\/span>\u044f<\/span>\u0433<\/span>\u0435<\/span><\/span><\/span><\/span><\/span><\/p>\n

\u0421\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u044e\u0442 \u0440\u0430\u0437\u043b\u0438\u0447\u043d\u044b\u0435 \u0441\u043f\u043e\u0441\u043e\u0431\u044b \u0440\u0430\u0437\u0432\u0438\u0442\u0438\u044f \u043a\u043e\u043d\u0442\u0440\u0430\u0441\u0442\u043d\u044b\u0445 \u043f\u043e\u0442\u0435\u0440\u044c. \u0417\u0434\u0435\u0441\u044c \u043c\u044b \u043f\u0440\u0435\u0434\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u043c \u0432\u0430\u043c \u0432\u0430\u0436\u043d\u0443\u044e \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u044e. <\/p>\n

\u041d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f L2 \u0438 \u0432\u044b\u0447\u0438\u0441\u043b\u0435\u043d\u0438\u0435 \u043c\u0430\u0442\u0440\u0438\u0446\u044b \u043f\u043e\u0434\u043e\u0431\u0438\u044f \u043a\u043e\u0441\u0438\u043d\u0443\u0441\u043e\u0432<\/h3>\n

\u0412\u043e-\u043f\u0435\u0440\u0432\u044b\u0445, \u043a \u043f\u0440\u0438\u0437\u043d\u0430\u043a\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u043f\u0440\u0438\u043c\u0435\u043d\u0438\u0442\u044c \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044e L2, \u0438\u043d\u0430\u0447\u0435 \u044d\u0442\u043e\u0442 \u043c\u0435\u0442\u043e\u0434 \u043d\u0435 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442. \u041d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f L2 \u043e\u0437\u043d\u0430\u0447\u0430\u0435\u0442, \u0447\u0442\u043e \u0432\u0435\u043a\u0442\u043e\u0440\u044b \u043d\u043e\u0440\u043c\u0438\u0440\u043e\u0432\u0430\u043d\u044b \u0442\u0430\u043a\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c, \u0447\u0442\u043e \u0432\u0441\u0435 \u043e\u043d\u0438 \u043b\u0435\u0436\u0430\u0442 \u043d\u0430 \u043f\u043e\u0432\u0435\u0440\u0445\u043d\u043e\u0441\u0442\u0438 \u0435\u0434\u0438\u043d\u0438\u0447\u043d\u043e\u0439 (\u0433\u0438\u043f\u0435\u0440)\u0441\u0444\u0435\u0440\u044b, \u0433\u0434\u0435 \u043d\u043e\u0440\u043c\u0430 L2 \u0440\u0430\u0432\u043d\u0430 1.<\/p>\n

z_i <\/span>=<\/span> F<\/span>.<\/span>normalize<\/span>(<\/span>proj_1<\/span>,<\/span> p<\/span>=<\/span>2<\/span>,<\/span> dim<\/span>=<\/span>1<\/span>)<\/span><\/p>
z_j <\/span>=<\/span> F<\/span>.<\/span>normalize<\/span>(<\/span>proj_2<\/span>,<\/span> p<\/span>=<\/span>2<\/span>,<\/span> dim<\/span>=<\/span>1<\/span>)<\/span><\/p><\/pre>\n
\u041e\u0431\u044a\u0435\u0434\u0438\u043d\u0438\u0442\u0435 2 \u0432\u044b\u0445\u043e\u0434\u043d\u044b\u0445 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u0432 \u043f\u0430\u043a\u0435\u0442\u043d\u043e\u043c \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u0438.  \u0418\u0445 \u0444\u043e\u0440\u043c\u0430 \u0431\u0443\u0434\u0435\u0442 [<\/mo>2<\/mn>\u00d7<\/mo>b<\/mi>a<\/mi>t<\/mi>c<\/mi>h<\/mi>_<\/mi>s<\/mi>i<\/mi>z<\/mi>e<\/mi>,<\/mo>d<\/mi>i<\/mi>m<\/mi>]<\/mo><\/mrow>[2 \\times batch\\_size, dim]<\/annotation><\/semantics><\/math><\/span>[<\/span>2<\/span>\u00d7<\/span><\/span>b<\/span>a<\/span>t<\/span>c<\/span>h<\/span>_<\/span>s<\/span>i<\/span>z<\/span>e<\/span>,<\/span>d<\/span>i<\/span>m<\/span>]<\/span><\/span><\/span><\/span><\/span>.  \u0417\u0430\u0442\u0435\u043c \u0432\u044b\u0447\u0438\u0441\u043b\u0438\u0442\u0435 \u0441\u0445\u043e\u0434\u0441\u0442\u0432\u043e\/\u043b\u043e\u0433\u0438\u0442\u044b \u0432\u0441\u0435\u0445 \u043f\u0430\u0440.  \u042d\u0442\u043e \u043c\u043e\u0436\u0435\u0442 \u0431\u044b\u0442\u044c \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d\u043e \u043f\u0443\u0442\u0435\u043c \u0443\u043c\u043d\u043e\u0436\u0435\u043d\u0438\u044f \u043c\u0430\u0442\u0440\u0438\u0446 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c.  \u0412\u044b\u0445\u043e\u0434\u043d\u0430\u044f \u0444\u043e\u0440\u043c\u0430 \u0440\u0430\u0432\u043d\u0430 [<\/mo>b<\/mi>a<\/mi>t<\/mi>c<\/mi>h<\/mi>_<\/mi>s<\/mi>i<\/mi>z<\/mi>e<\/mi>\u00d7<\/mo>v<\/mi>i<\/mi>e<\/mi>w<\/mi>s<\/mi>,<\/mo>b<\/mi>a<\/mi>t<\/mi>c<\/mi>h<\/mi>_<\/mi>s<\/mi>i<\/mi>z<\/mi>e<\/mi>\u00d7<\/mo>v<\/mi>i<\/mi>e<\/mi>w<\/mi>s<\/mi>]<\/mo><\/mrow>[batch\\_size \\times views, batch\\_size \\times views]<\/annotation><\/semantics><\/math><\/span>[<\/span>b<\/span>a<\/span>t<\/span>c<\/span>h<\/span>_<\/span>s<\/span>i<\/span>z<\/span>e<\/span>\u00d7<\/span><\/span>v<\/span>i<\/span>e<\/span>w<\/span>s<\/span>,<\/span>b<\/span>a<\/span>t<\/span>c<\/span>h<\/span>_<\/span>s<\/span>i<\/span>z<\/span>e<\/span>\u00d7<\/span><\/span>v<\/span>i<\/span>e<\/span>w<\/span>s<\/span>]<\/span><\/span><\/span><\/span><\/span><\/p>\n
def<\/span> <\/span>calc_similarity_batch<\/span>(<\/span>self<\/span>,<\/span> a<\/span>,<\/span> b<\/span>)<\/span>:<\/span><\/p>
       representations <\/span>=<\/span> torch<\/span>.<\/span>cat<\/span>(<\/span>[<\/span>a<\/span>,<\/span> b<\/span>]<\/span>,<\/span> dim<\/span>=<\/span>0<\/span>)<\/span><\/p>
       <\/span>return<\/span> F<\/span>.<\/span>cosine_similarity<\/span>(<\/span>representations<\/span>.<\/span>unsqueeze<\/span>(<\/span>1<\/span>)<\/span>,<\/span> representations<\/span>.<\/span>unsqueeze<\/span>(<\/span>0<\/span>)<\/span>,<\/span> dim<\/span>=<\/span>2<\/span>)<\/span><\/p><\/pre>\n
\u0418\u043d\u0434\u0435\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435 \u043c\u0430\u0442\u0440\u0438\u0446\u044b \u0441\u0445\u043e\u0434\u0441\u0442\u0432\u0430 \u0434\u043b\u044f \u0444\u0443\u043d\u043a\u0446\u0438\u0438 \u043f\u043e\u0442\u0435\u0440\u044c SimCLR<\/h3>\n
\u0422\u0435\u043f\u0435\u0440\u044c \u043d\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u043f\u0440\u043e\u0438\u043d\u0434\u0435\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u043d\u0443\u044e \u043c\u0430\u0442\u0440\u0438\u0446\u0443 \u0440\u0430\u0437\u043c\u0435\u0440\u0430 [<\/mo>b<\/mi>a<\/mi>t<\/mi>c<\/mi>h<\/mi>_<\/mi>s<\/mi>i<\/mi>z<\/mi>e<\/mi>\u00d7<\/mo>v<\/mi>i<\/mi>e<\/mi>w<\/mi>s<\/mi>,<\/mo>b<\/mi>a<\/mi>t<\/mi>c<\/mi>h<\/mi>_<\/mi>s<\/mi>i<\/mi>z<\/mi>e<\/mi>\u00d7<\/mo>v<\/mi>i<\/mi>e<\/mi>w<\/mi>s<\/mi>]<\/mo><\/mrow>[batch\\_size \\times views, batch\\_size \\times views]<\/annotation><\/semantics><\/math><\/span>[<\/span>b<\/span>a<\/span>t<\/span>c<\/span>h<\/span>_<\/span>s<\/span>i<\/span>z<\/span>e<\/span>\u00d7<\/span><\/span>v<\/span>i<\/span>e<\/span>w<\/span>s<\/span>,<\/span>b<\/span>a<\/span>t<\/span>c<\/span>h<\/span>_<\/span>s<\/span>i<\/span>z<\/span>e<\/span>\u00d7<\/span><\/span>v<\/span>i<\/span>e<\/span>w<\/span>s<\/span>]<\/span><\/span><\/span><\/span><\/span>  \u0441\u043e\u043e\u0442\u0432\u0435\u0442\u0441\u0442\u0432\u0435\u043d\u043d\u043e. <\/p>\n
<\/p>\n
    
\n  \"simclr-\u0438\u043b\u043b\u044e\u0441\u0442\u0440\u0430\u0446\u0438\u044f-\u043f\u043e\u0442\u0435\u0440\u044f\"<\/p>\n
    <\/span>
\n\u041d\u0430\u0433\u043b\u044f\u0434\u043d\u0430\u044f \u0438\u043b\u043b\u044e\u0441\u0442\u0440\u0430\u0446\u0438\u044f SimCLR.  \u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u043e\u0442 \u0430\u0432\u0442\u043e\u0440\u0430<\/em> <\/p>\n
\u0425\u043e\u0440\u043e\u0448\u043e, \u043a\u0430\u043a, \u0447\u0435\u0440\u0442 \u0432\u043e\u0437\u044c\u043c\u0438, \u043c\u044b \u044d\u0442\u043e \u0434\u0435\u043b\u0430\u0435\u043c?  \u0423 \u043c\u0435\u043d\u044f \u0442\u0430\u043a\u043e\u0439 \u0436\u0435 \u0432\u043e\u043f\u0440\u043e\u0441.  \u0417\u0434\u0435\u0441\u044c \u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u043a\u0435\u0442\u0430 \u0441\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 2 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f, \u043d\u043e \u043c\u044b \u0445\u043e\u0442\u0438\u043c \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u0442\u044c \u0440\u0435\u0448\u0435\u043d\u0438\u0435 \u0434\u043b\u044f \u043b\u044e\u0431\u043e\u0433\u043e \u0440\u0430\u0437\u043c\u0435\u0440\u0430 \u043f\u0430\u043a\u0435\u0442\u0430.  \u0415\u0441\u043b\u0438 \u0432\u044b \u0432\u043d\u0438\u043c\u0430\u0442\u0435\u043b\u044c\u043d\u043e \u043f\u043e\u0441\u043c\u043e\u0442\u0440\u0438\u0442\u0435, \u0442\u043e \u0443\u0432\u0438\u0434\u0438\u0442\u0435, \u0447\u0442\u043e \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043f\u0430\u0440\u044b \u0441\u0434\u0432\u0438\u043d\u0443\u0442\u044b \u043e\u0442 \u0433\u043b\u0430\u0432\u043d\u043e\u0439 \u0434\u0438\u0430\u0433\u043e\u043d\u0430\u043b\u0438 \u043d\u0430 2, \u0442\u043e \u0435\u0441\u0442\u044c \u043d\u0430 \u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u0440\u0442\u0438\u0438.  \u041e\u0434\u0438\u043d \u0438\u0437 \u0441\u043f\u043e\u0441\u043e\u0431\u043e\u0432 \u0441\u0434\u0435\u043b\u0430\u0442\u044c \u044d\u0442\u043e torch.diag()<\/code>.  \u041e\u043d \u0431\u0435\u0440\u0435\u0442 \u0432\u044b\u0431\u0440\u0430\u043d\u043d\u0443\u044e \u0434\u0438\u0430\u0433\u043e\u043d\u0430\u043b\u044c \u0438\u0437 \u043c\u0430\u0442\u0440\u0438\u0446\u044b.  \u041f\u0435\u0440\u0432\u044b\u0439 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 \u2014 \u044d\u0442\u043e \u043c\u0430\u0442\u0440\u0438\u0446\u0430, \u0430 \u0432\u0442\u043e\u0440\u043e\u0439 \u0443\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0434\u0438\u0430\u0433\u043e\u043d\u0430\u043b\u044c, \u0433\u0434\u0435 \u043d\u0443\u043b\u044c \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u044b \u0433\u043b\u0430\u0432\u043d\u043e\u0439 \u0434\u0438\u0430\u0433\u043e\u043d\u0430\u043b\u0438.  \u0411\u0435\u0440\u0435\u043c \u0434\u0438\u0430\u0433\u043e\u043d\u0430\u043b\u0438, \u0441\u0434\u0432\u0438\u043d\u0443\u0442\u044b\u0435 \u043d\u0430 \u0440\u0430\u0437\u043c\u0435\u0440 \u043f\u0430\u0440\u0442\u0438\u0438.<\/p>\n
sim_ij <\/span>=<\/span> torch<\/span>.<\/span>diag<\/span>(<\/span>similarity_matrix<\/span>,<\/span> batch_size<\/span>)<\/span><\/p>
sim_ji <\/span>=<\/span> torch<\/span>.<\/span>diag<\/span>(<\/span>similarity_matrix<\/span>,<\/span> <\/span>-<\/span>batch_size<\/span>)<\/span><\/p>
positives <\/span>=<\/span> torch<\/span>.<\/span>cat<\/span>(<\/span>[<\/span>sim_ij<\/span>,<\/span> sim_ji<\/span>]<\/span>,<\/span> dim<\/span>=<\/span>0<\/span>)<\/span><\/p><\/pre>\n
\u0415\u0441\u0442\u044c \u0431<\/mi>\u0430<\/mi>\u0442<\/mi>\u0441<\/mi>\u0447\u0430\u0441<\/mi>_<\/mi>\u0441<\/mi>\u044f<\/mi>\u0433<\/mi>\u0435<\/mi>\u00d7<\/mo>\u0432<\/mi>\u044f<\/mi>\u0435<\/mi>\u0436<\/mi>\u0441<\/mi><\/mrow>\u043f\u0430\u043a\u0435\u0442\\_\u0440\u0430\u0437\u043c\u0435\u0440\\\u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u043e\u0432<\/annotation><\/semantics><\/math><\/span>\u0431<\/span>\u0430<\/span>\u0442<\/span>\u0441<\/span>\u0447\u0430\u0441<\/span>_<\/span>\u0441<\/span>\u044f<\/span>\u0433<\/span>\u0435<\/span>\u00d7<\/span><\/span>\u0432<\/span>\u044f<\/span>\u0435<\/span>\u0436<\/span>\u0441<\/span><\/span><\/span><\/span><\/span>  \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043f\u0430\u0440\u044b.  \u0414\u0440\u0443\u0433\u043e\u0439 \u043f\u0440\u0438\u043c\u0435\u0440 \u0434\u043b\u044f [6,6]  \u043c\u0430\u0442\u0440\u0438\u0446\u0430 (batch_size=3,views=2) \u0434\u043e\u043b\u0436\u043d\u0430 \u0438\u043c\u0435\u0442\u044c \u043c\u0430\u0441\u043a\u0443, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0442\u043e\u0447\u043d\u043e \u0442\u0430\u043a:<\/p>\n
[<\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>1<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>]<\/span>,<\/span><\/p>
[<\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>1<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>]<\/span>,<\/span><\/p>
[<\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>1<\/span>.<\/span>]<\/span>,<\/span><\/p>
[<\/span>1<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>]<\/span>,<\/span><\/p>
[<\/span>0<\/span>.<\/span>,<\/span> <\/span>1<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>]<\/span>,<\/span><\/p>
[<\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>1<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>,<\/span> <\/span>0<\/span>.<\/span>]<\/span><\/p><\/pre>\n
\u0414\u043b\u044f \u0437\u043d\u0430\u043c\u0435\u043d\u0430\u0442\u0435\u043b\u044f \u043d\u0430\u043c \u043d\u0443\u0436\u043d\u044b \u043a\u0430\u043a \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435, \u0442\u0430\u043a \u0438 \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0435 \u043f\u0430\u0440\u044b.  \u0422\u0430\u043a\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c, \u0431\u0438\u043d\u0430\u0440\u043d\u0430\u044f \u043c\u0430\u0441\u043a\u0430 \u0431\u0443\u0434\u0435\u0442 \u0442\u043e\u0447\u043d\u043e\u0439 \u043f\u043e\u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043d\u043e\u0439 \u0438\u043d\u0432\u0435\u0440\u0441\u0438\u0435\u0439 \u0435\u0434\u0438\u043d\u0438\u0447\u043d\u043e\u0439 \u043c\u0430\u0442\u0440\u0438\u0446\u044b.<\/p>\n
self<\/span>.<\/span>mask <\/span>=<\/span> <\/span>(<\/span>~<\/span>torch<\/span>.<\/span>eye<\/span>(<\/span>batch_size <\/span>*<\/span> <\/span>2<\/span>,<\/span> batch_size <\/span>*<\/span> <\/span>2<\/span>,<\/span> dtype<\/span>=<\/span>bool<\/span>)<\/span>)<\/span>.<\/span>float<\/span>(<\/span>)<\/span><\/p>
pos_and_negatives <\/span>=<\/span> self<\/span>.<\/span>mask <\/span>*<\/span> similarity_matrix<\/span><\/p><\/pre>\n
\u041e\u043f\u044f\u0442\u044c \u0436\u0435, \u043e\u043d\u0438 \u044f\u0432\u043b\u044f\u044e\u0442\u0441\u044f \u043a\u0430\u043a \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u043c\u0438, \u0442\u0430\u043a \u0438 \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u043c\u0438 \u0432 \u0437\u043d\u0430\u043c\u0435\u043d\u0430\u0442\u0435\u043b\u0435.<\/p>\n
\u0412\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u0440\u0430\u0437\u043e\u0431\u0440\u0430\u0442\u044c \u043e\u0441\u0442\u0430\u043b\u044c\u043d\u043e\u0435 (\u043c\u0430\u0441\u0448\u0442\u0430\u0431\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435 \u0442\u0435\u043c\u043f\u0435\u0440\u0430\u0442\u0443\u0440\u044b \u0438 \u0441\u0443\u043c\u043c\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435 \u043e\u0442\u0440\u0438\u0446\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0445 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0439 \u0438\u0437 \u0437\u043d\u0430\u043c\u0435\u043d\u0430\u0442\u0435\u043b\u044f \u0438 \u0442. \u0434.):<\/p>\n
\u0420\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u043f\u043e\u0442\u0435\u0440\u044c SimCLR<\/h3>\n
import<\/span> torch<\/span><\/p>
import<\/span> torch<\/span>.<\/span>nn <\/span>as<\/span> nn<\/span><\/p>
import<\/span> torch<\/span>.<\/span>nn<\/span>.<\/span>functional <\/span>as<\/span> F<\/span><\/p>
def<\/span> <\/span>device_as<\/span>(<\/span>t1<\/span>,<\/span> t2<\/span>)<\/span>:<\/span><\/p>
   <\/span>\"\"\"<\/span><\/p>
   Moves t1 to the device of t2<\/span><\/p>
   \"\"\"<\/span><\/p>
   <\/span>return<\/span> t1<\/span>.<\/span>to<\/span>(<\/span>t2<\/span>.<\/span>device<\/span>)<\/span><\/p>
class<\/span> <\/span>ContrastiveLoss<\/span>(<\/span>nn<\/span>.<\/span>Module<\/span>)<\/span>:<\/span><\/p>
   <\/span>\"\"\"<\/span><\/p>
   Vanilla Contrastive loss, also called InfoNceLoss as in SimCLR paper<\/span><\/p>
   \"\"\"<\/span><\/p>
   <\/span>def<\/span> <\/span>__init__<\/span>(<\/span>self<\/span>,<\/span> batch_size<\/span>,<\/span> temperature<\/span>=<\/span>0.5<\/span>)<\/span>:<\/span><\/p>
       <\/span>super<\/span>(<\/span>)<\/span>.<\/span>__init__<\/span>(<\/span>)<\/span><\/p>
       self<\/span>.<\/span>batch_size <\/span>=<\/span> batch_size<\/span><\/p>
       self<\/span>.<\/span>temperature <\/span>=<\/span> temperature<\/span><\/p>
       self<\/span>.<\/span>mask <\/span>=<\/span> <\/span>(<\/span>~<\/span>torch<\/span>.<\/span>eye<\/span>(<\/span>batch_size <\/span>*<\/span> <\/span>2<\/span>,<\/span> batch_size <\/span>*<\/span> <\/span>2<\/span>,<\/span> dtype<\/span>=<\/span>bool<\/span>)<\/span>)<\/span>.<\/span>float<\/span>(<\/span>)<\/span><\/p>
   <\/span>def<\/span> <\/span>calc_similarity_batch<\/span>(<\/span>self<\/span>,<\/span> a<\/span>,<\/span> b<\/span>)<\/span>:<\/span><\/p>
       representations <\/span>=<\/span> torch<\/span>.<\/span>cat<\/span>(<\/span>[<\/span>a<\/span>,<\/span> b<\/span>]<\/span>,<\/span> dim<\/span>=<\/span>0<\/span>)<\/span><\/p>
       <\/span>return<\/span> F<\/span>.<\/span>cosine_similarity<\/span>(<\/span>representations<\/span>.<\/span>unsqueeze<\/span>(<\/span>1<\/span>)<\/span>,<\/span> representations<\/span>.<\/span>unsqueeze<\/span>(<\/span>0<\/span>)<\/span>,<\/span> dim<\/span>=<\/span>2<\/span>)<\/span><\/p>
   <\/span>def<\/span> <\/span>forward<\/span>(<\/span>self<\/span>,<\/span> proj_1<\/span>,<\/span> proj_2<\/span>)<\/span>:<\/span><\/p>
       <\/span>\"\"\"<\/span><\/p>
       proj_1 and proj_2 are batched embeddings [batch, embedding_dim]<\/span><\/p>
       where corresponding indices are pairs<\/span><\/p>
       z_i, z_j in the SimCLR paper<\/span><\/p>
       \"\"\"<\/span><\/p>
       batch_size <\/span>=<\/span> proj_1<\/span>.<\/span>shape<\/span>[<\/span>0<\/span>]<\/span><\/p>
       z_i <\/span>=<\/span> F<\/span>.<\/span>normalize<\/span>(<\/span>proj_1<\/span>,<\/span> p<\/span>=<\/span>2<\/span>,<\/span> dim<\/span>=<\/span>1<\/span>)<\/span><\/p>
       z_j <\/span>=<\/span> F<\/span>.<\/span>normalize<\/span>(<\/span>proj_2<\/span>,<\/span> p<\/span>=<\/span>2<\/span>,<\/span> dim<\/span>=<\/span>1<\/span>)<\/span><\/p>
       similarity_matrix <\/span>=<\/span> self<\/span>.<\/span>calc_similarity_batch<\/span>(<\/span>z_i<\/span>,<\/span> z_j<\/span>)<\/span><\/p>
       sim_ij <\/span>=<\/span> torch<\/span>.<\/span>diag<\/span>(<\/span>similarity_matrix<\/span>,<\/span> batch_size<\/span>)<\/span><\/p>
       sim_ji <\/span>=<\/span> torch<\/span>.<\/span>diag<\/span>(<\/span>similarity_matrix<\/span>,<\/span> <\/span>-<\/span>batch_size<\/span>)<\/span><\/p>
       positives <\/span>=<\/span> torch<\/span>.<\/span>cat<\/span>(<\/span>[<\/span>sim_ij<\/span>,<\/span> sim_ji<\/span>]<\/span>,<\/span> dim<\/span>=<\/span>0<\/span>)<\/span><\/p>
       nominator <\/span>=<\/span> torch<\/span>.<\/span>exp<\/span>(<\/span>positives <\/span>\/<\/span> self<\/span>.<\/span>temperature<\/span>)<\/span><\/p>
       denominator <\/span>=<\/span> device_as<\/span>(<\/span>self<\/span>.<\/span>mask<\/span>,<\/span> similarity_matrix<\/span>)<\/span> <\/span>*<\/span> torch<\/span>.<\/span>exp<\/span>(<\/span>similarity_matrix <\/span>\/<\/span> self<\/span>.<\/span>temperature<\/span>)<\/span><\/p>
       all_losses <\/span>=<\/span> <\/span>-<\/span>torch<\/span>.<\/span>log<\/span>(<\/span>nominator <\/span>\/<\/span> torch<\/span>.<\/span>sum<\/span>(<\/span>denominator<\/span>,<\/span> dim<\/span>=<\/span>1<\/span>)<\/span>)<\/span><\/p>
       loss <\/span>=<\/span> torch<\/span>.<\/span>sum<\/span>(<\/span>all_losses<\/span>)<\/span> <\/span>\/<\/span> <\/span>(<\/span>2<\/span> <\/span>*<\/span> self<\/span>.<\/span>batch_size<\/span>)<\/span><\/p>
       <\/span>return<\/span> loss<\/span><\/p><\/pre>\n
\u0410\u0443\u0433\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438<\/h2>\n
\u041a\u043b\u044e\u0447\u043e\u043c \u043a \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044e \u0440\u0435\u043f\u0440\u0435\u0437\u0435\u043d\u0442\u0430\u0446\u0438\u0438 \u0441 \u0441\u0430\u043c\u043e\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0435\u043c \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0443\u0432\u0435\u043b\u0438\u0447\u0435\u043d\u0438\u0435 \u0434\u0430\u043d\u043d\u044b\u0445.  \u041e\u0431\u044b\u0447\u043d\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0439 \u043a\u043e\u043d\u0432\u0435\u0439\u0435\u0440 \u043f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u043e\u0432\u0430\u043d\u0438\u044f:<\/p>\n
    \n
  • \n
    \u041e\u0431\u0440\u0435\u0437\u043a\u0430 \u0432 \u0441\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u043c \u043c\u0430\u0441\u0448\u0442\u0430\u0431\u0435 \u043e\u0442 7% \u0434\u043e 100% \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f<\/p>\n<\/li>\n
  • \n
    \u0418\u0437\u043c\u0435\u043d\u0438\u0442\u0435 \u0440\u0430\u0437\u043c\u0435\u0440 \u0432\u0441\u0435\u0445 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439 \u0434\u043e 224 \u0438\u043b\u0438 \u0434\u0440\u0443\u0433\u0438\u0445 \u043f\u0440\u043e\u0441\u0442\u0440\u0430\u043d\u0441\u0442\u0432\u0435\u043d\u043d\u044b\u0445 \u0440\u0430\u0437\u043c\u0435\u0440\u043e\u0432.<\/p>\n<\/li>\n
  • \n
    \u041f\u0440\u0438\u043c\u0435\u043d\u0438\u0442\u044c \u0433\u043e\u0440\u0438\u0437\u043e\u043d\u0442\u0430\u043b\u044c\u043d\u043e\u0435 \u043e\u0442\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u0441 \u0432\u0435\u0440\u043e\u044f\u0442\u043d\u043e\u0441\u0442\u044c\u044e 50%<\/p>\n<\/li>\n
  • \n
    \u041f\u0440\u0438\u043c\u0435\u043d\u0438\u0442\u044c \u0441\u0438\u043b\u044c\u043d\u043e\u0435 \u0446\u0432\u0435\u0442\u043e\u0432\u043e\u0435 \u0434\u0440\u043e\u0436\u0430\u043d\u0438\u0435 \u0441 \u0432\u0435\u0440\u043e\u044f\u0442\u043d\u043e\u0441\u0442\u044c\u044e 80%<\/p>\n<\/li>\n
  • \n
    \u041f\u0440\u0438\u043c\u0435\u043d\u0438\u0442\u044c \u0440\u0430\u0437\u043c\u044b\u0442\u0438\u0435 \u043f\u043e \u0413\u0430\u0443\u0441\u0441\u0443 \u0441 \u0432\u0435\u0440\u043e\u044f\u0442\u043d\u043e\u0441\u0442\u044c\u044e 50%.  \u0420\u0430\u0437\u043c\u0435\u0440 \u044f\u0434\u0440\u0430 \u043e\u0431\u044b\u0447\u043d\u043e \u0441\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 \u043e\u043a\u043e\u043b\u043e 10% \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f \u0438\u043b\u0438 \u043c\u0435\u043d\u044c\u0448\u0435.<\/p>\n<\/li>\n
  • \n
    \u041f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u043e\u0432\u0430\u043d\u0438\u0435 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0439 RGB \u0432 \u043e\u0442\u0442\u0435\u043d\u043a\u0438 \u0441\u0435\u0440\u043e\u0433\u043e \u0441 \u0432\u0435\u0440\u043e\u044f\u0442\u043d\u043e\u0441\u0442\u044c\u044e 20%.<\/p>\n<\/li>\n
  • \n
    \u041d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u0441\u0440\u0435\u0434\u043d\u0438\u0445 \u0438 \u043e\u0442\u043a\u043b\u043e\u043d\u0435\u043d\u0438\u0439 imagenet  <\/p>\n<\/li>\n<\/ul>\n
    \u042d\u0442\u043e\u0442 \u043a\u043e\u043d\u0432\u0435\u0439\u0435\u0440 \u0431\u0443\u0434\u0435\u0442 \u043d\u0435\u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e \u043f\u0440\u0438\u043c\u0435\u043d\u044f\u0442\u044c\u0441\u044f \u043a \u043a\u0430\u0436\u0434\u043e\u043c\u0443 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044e \u0434\u0432\u0430\u0436\u0434\u044b \u0438 \u0441\u043e\u0437\u0434\u0430\u0441\u0442 \u0434\u0432\u0430 \u0440\u0430\u0437\u043d\u044b\u0445 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0431\u0443\u0434\u0443\u0442 \u043f\u0435\u0440\u0435\u0434\u0430\u043d\u044b \u0432 \u043c\u043e\u0434\u0435\u043b\u044c \u043c\u0430\u0433\u0438\u0441\u0442\u0440\u0430\u043b\u0438.  \u0412 \u044d\u0442\u043e\u0439 \u0437\u0430\u043f\u0438\u0441\u043d\u043e\u0439 \u043a\u043d\u0438\u0436\u043a\u0435 \u043c\u044b \u0431\u0443\u0434\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0441\u0442\u0430\u043d\u0434\u0430\u0440\u0442\u043d\u044b\u0439 \u0444\u0430\u0439\u043b resnet18.<\/p>\n
    import<\/span> torch<\/span><\/p>
    import<\/span> torchvision<\/span>.<\/span>transforms <\/span>as<\/span> T<\/span><\/p>
    class<\/span> <\/span>Augment<\/span>:<\/span><\/p>
       <\/span>\"\"\"<\/span><\/p>
       A stochastic data augmentation module<\/span><\/p>
       Transforms any given data example randomly<\/span><\/p>
       resulting in two correlated views of the same example,<\/span><\/p>
       denoted x \u0303i and x \u0303j, which we consider as a positive pair.<\/span><\/p>
       \"\"\"<\/span><\/p>
       <\/span>def<\/span> <\/span>__init__<\/span>(<\/span>self<\/span>,<\/span> img_size<\/span>,<\/span> s<\/span>=<\/span>1<\/span>)<\/span>:<\/span><\/p>
           color_jitter <\/span>=<\/span> T<\/span>.<\/span>ColorJitter<\/span>(<\/span><\/p>
               <\/span>0.8<\/span> <\/span>*<\/span> s<\/span>,<\/span> <\/span>0.8<\/span> <\/span>*<\/span> s<\/span>,<\/span> <\/span>0.8<\/span> <\/span>*<\/span> s<\/span>,<\/span> <\/span>0.2<\/span> <\/span>*<\/span> s<\/span><\/p>
           <\/span>)<\/span><\/p>
           blur <\/span>=<\/span> T<\/span>.<\/span>GaussianBlur<\/span>(<\/span>(<\/span>3<\/span>,<\/span> <\/span>3<\/span>)<\/span>,<\/span> <\/span>(<\/span>0.1<\/span>,<\/span> <\/span>2.0<\/span>)<\/span>)<\/span><\/p>
           self<\/span>.<\/span>train_transform <\/span>=<\/span> torch<\/span>.<\/span>nn<\/span>.<\/span>Sequential<\/span>(<\/span><\/p>
               T<\/span>.<\/span>RandomResizedCrop<\/span>(<\/span>size<\/span>=<\/span>img_size<\/span>)<\/span>,<\/span><\/p>
               T<\/span>.<\/span>RandomHorizontalFlip<\/span>(<\/span>p<\/span>=<\/span>0.5<\/span>)<\/span>,<\/span>  <\/span><\/p>
               T<\/span>.<\/span>RandomApply<\/span>(<\/span>[<\/span>color_jitter<\/span>]<\/span>,<\/span> p<\/span>=<\/span>0.8<\/span>)<\/span>,<\/span><\/p>
               T<\/span>.<\/span>RandomApply<\/span>(<\/span>[<\/span>blur<\/span>]<\/span>,<\/span> p<\/span>=<\/span>0.5<\/span>)<\/span>,<\/span><\/p>
               T<\/span>.<\/span>RandomGrayscale<\/span>(<\/span>p<\/span>=<\/span>0.2<\/span>)<\/span>,<\/span><\/p>
               T<\/span>.<\/span>Normalize<\/span>(<\/span>mean<\/span>=<\/span>[<\/span>0.485<\/span>,<\/span> <\/span>0.456<\/span>,<\/span> <\/span>0.406<\/span>]<\/span>,<\/span> std<\/span>=<\/span>[<\/span>0.229<\/span>,<\/span> <\/span>0.224<\/span>,<\/span> <\/span>0.225<\/span>]<\/span>)<\/span><\/p>
           <\/span>)<\/span><\/p>
       <\/span>def<\/span> <\/span>__call__<\/span>(<\/span>self<\/span>,<\/span> x<\/span>)<\/span>:<\/span><\/p>
           <\/span>return<\/span> self<\/span>.<\/span>train_transform<\/span>(<\/span>x<\/span>)<\/span>,<\/span> self<\/span>.<\/span>train_transform<\/span>(<\/span>x<\/span>)<\/span><\/p><\/pre>\n
    \u041d\u0438\u0436\u0435 \u043f\u0440\u0438\u0432\u0435\u0434\u0435\u043d\u044b 4 \u0440\u0430\u0437\u043d\u044b\u0445 \u0432\u0438\u0434\u0430 \u043e\u0434\u043d\u043e\u0433\u043e \u0438 \u0442\u043e\u0433\u043e \u0436\u0435 \u0438\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u044f \u0441 \u043f\u0440\u0438\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u043c \u043e\u0434\u043d\u043e\u0433\u043e \u0438 \u0442\u043e\u0433\u043e \u0436\u0435 \u0441\u0442\u043e\u0445\u0430\u0441\u0442\u0438\u0447\u0435\u0441\u043a\u043e\u0433\u043e \u043f\u0430\u0439\u043f\u043b\u0430\u0439\u043d\u0430:<\/p>\n
    <\/p>\n
        
    \n  \"\u0430\u0443\u0433\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438-stl10-simclr-\u043a\u043e\u043d\u0432\u0435\u0439\u0435\u0440\"<\/p>\n
        <\/span>
    \n4 \u0440\u0430\u0437\u043d\u044b\u0445 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u043e\u0434\u043d\u043e\u0433\u043e \u0438 \u0442\u043e\u0433\u043e \u0436\u0435 \u043f\u0430\u0439\u043f\u043b\u0430\u0439\u043d\u0430.  \u0418\u0437\u043e\u0431\u0440\u0430\u0436\u0435\u043d\u0438\u0435 \u0430\u0432\u0442\u043e\u0440\u0430<\/em> <\/p>\n
    \u0427\u0442\u043e\u0431\u044b \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0438\u0445, \u0432\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u043e\u0442\u043c\u0435\u043d\u0438\u0442\u044c \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044e \u0441\u0440\u0435\u0434\u043d\u0435\u0433\u043e \u0441\u0442\u0430\u043d\u0434\u0430\u0440\u0442\u043d\u043e\u0433\u043e \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u0438 \u043f\u043e\u043c\u0435\u0441\u0442\u0438\u0442\u044c \u0446\u0432\u0435\u0442\u043e\u0432\u044b\u0435 \u043a\u0430\u043d\u0430\u043b\u044b \u0432 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0435\u0435 \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u0435:<\/p>\n
    def<\/span> <\/span>imshow<\/span>(<\/span>img<\/span>)<\/span>:<\/span><\/p>
       <\/span>\"\"\"<\/span><\/p>
       shows an imagenet-normalized image on the screen<\/span><\/p>
       \"\"\"<\/span><\/p>
       mean <\/span>=<\/span> torch<\/span>.<\/span>tensor<\/span>(<\/span>[<\/span>0.485<\/span>,<\/span> <\/span>0.456<\/span>,<\/span> <\/span>0.406<\/span>]<\/span>,<\/span> dtype<\/span>=<\/span>torch<\/span>.<\/span>float32<\/span>)<\/span><\/p>
       std <\/span>=<\/span> torch<\/span>.<\/span>tensor<\/span>(<\/span>[<\/span>0.229<\/span>,<\/span> <\/span>0.224<\/span>,<\/span> <\/span>0.225<\/span>]<\/span>,<\/span> dtype<\/span>=<\/span>torch<\/span>.<\/span>float32<\/span>)<\/span><\/p>
       unnormalize <\/span>=<\/span> T<\/span>.<\/span>Normalize<\/span>(<\/span>(<\/span>-<\/span>mean <\/span>\/<\/span> std<\/span>)<\/span>.<\/span>tolist<\/span>(<\/span>)<\/span>,<\/span> <\/span>(<\/span>1.0<\/span> <\/span>\/<\/span> std<\/span>)<\/span>.<\/span>tolist<\/span>(<\/span>)<\/span>)<\/span><\/p>
       npimg <\/span>=<\/span> unnormalize<\/span>(<\/span>img<\/span>)<\/span>.<\/span>numpy<\/span>(<\/span>)<\/span><\/p>
       plt<\/span>.<\/span>imshow<\/span>(<\/span>np<\/span>.<\/span>transpose<\/span>(<\/span>npimg<\/span>,<\/span> <\/span>(<\/span>1<\/span>,<\/span> <\/span>2<\/span>,<\/span> <\/span>0<\/span>)<\/span>)<\/span>)<\/span><\/p>
       plt<\/span>.<\/span>show<\/span>(<\/span>)<\/span><\/p>
    dataset <\/span>=<\/span> STL10<\/span>(<\/span>\".\/\"<\/span>,<\/span> split<\/span>=<\/span>'train'<\/span>,<\/span> transform<\/span>=<\/span>Augment<\/span>(<\/span>96<\/span>)<\/span>,<\/span> download<\/span>=<\/span>True<\/span>)<\/span><\/p>
    imshow<\/span>(<\/span>dataset<\/span>[<\/span>99<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>0<\/span>]<\/span>)<\/span><\/p>
    imshow<\/span>(<\/span>dataset<\/span>[<\/span>99<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>0<\/span>]<\/span>)<\/span><\/p>
    imshow<\/span>(<\/span>dataset<\/span>[<\/span>99<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>0<\/span>]<\/span>)<\/span><\/p>
    imshow<\/span>(<\/span>dataset<\/span>[<\/span>99<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>0<\/span>]<\/span>)<\/span><\/p><\/pre>\n
    \u0418\u0437\u043c\u0435\u043d\u0438\u0442\u0435 Resnet18 \u0438 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u0442\u0435 \u0433\u0440\u0443\u043f\u043f\u044b \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432<\/h2>\n
    \u041e\u0434\u043d\u0438\u043c \u0438\u0437 \u0432\u0430\u0436\u043d\u044b\u0445 \u0448\u0430\u0433\u043e\u0432 \u0434\u043b\u044f \u0437\u0430\u043f\u0443\u0441\u043a\u0430 simclr \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0443\u0434\u0430\u043b\u0435\u043d\u0438\u0435 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0435\u0433\u043e \u043f\u043e\u043b\u043d\u043e\u0441\u0432\u044f\u0437\u043d\u043e\u0433\u043e \u0441\u043b\u043e\u044f.  \u041c\u044b \u0437\u0430\u043c\u0435\u043d\u0438\u043c \u0435\u0433\u043e \u0444\u0443\u043d\u043a\u0446\u0438\u0435\u0439 \u0442\u043e\u0436\u0434\u0435\u0441\u0442\u0432\u0430.  \u0417\u0430\u0442\u0435\u043c \u043d\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u0434\u043e\u0431\u0430\u0432\u0438\u0442\u044c \u043f\u0440\u043e\u0435\u043a\u0446\u0438\u043e\u043d\u043d\u0443\u044e \u0433\u043e\u043b\u043e\u0432\u043a\u0443 (\u0435\u0449\u0435 \u043e\u0434\u0438\u043d MLP), \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f \u0442\u043e\u043b\u044c\u043a\u043e \u043d\u0430 \u044d\u0442\u0430\u043f\u0435 \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0438 \u0441 \u0441\u0430\u043c\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0435\u043c.  \u0414\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u043d\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u0437\u043d\u0430\u0442\u044c \u0440\u0430\u0437\u043c\u0435\u0440\u043d\u043e\u0441\u0442\u044c \u0444\u0443\u043d\u043a\u0446\u0438\u0439 \u043d\u0430\u0448\u0435\u0439 \u043c\u043e\u0434\u0435\u043b\u0438.  \u0412 \u0447\u0430\u0441\u0442\u043d\u043e\u0441\u0442\u0438, resnet18 \u0432\u044b\u0432\u043e\u0434\u0438\u0442 512-\u043c\u0435\u0440\u043d\u044b\u0439 \u0432\u0435\u043a\u0442\u043e\u0440, \u0430 resnet50 \u0432\u044b\u0432\u043e\u0434\u0438\u0442 2048-\u043c\u0435\u0440\u043d\u044b\u0439 \u0432\u0435\u043a\u0442\u043e\u0440.  \u041f\u0440\u043e\u0435\u043a\u0446\u0438\u044f MLP \u043f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u0443\u0435\u0442 \u0435\u0433\u043e \u0432 \u0440\u0430\u0437\u043c\u0435\u0440 \u0432\u0435\u043a\u0442\u043e\u0440\u0430 \u0432\u0441\u0442\u0440\u0430\u0438\u0432\u0430\u043d\u0438\u044f, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0441\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 128, \u0441\u043e\u0433\u043b\u0430\u0441\u043d\u043e \u043e\u0444\u0438\u0446\u0438\u0430\u043b\u044c\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0435.<\/p>\n
    \u0414\u043b\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0446\u0438\u0438 \u043c\u043e\u0434\u0435\u043b\u0435\u0439 SSL \u043c\u044b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c \u0442\u044f\u0436\u0435\u043b\u044b\u0435 \u043c\u0435\u0442\u043e\u0434\u044b \u0440\u0435\u0433\u0443\u043b\u044f\u0440\u0438\u0437\u0430\u0446\u0438\u0438, \u0442\u0430\u043a\u0438\u0435 \u043a\u0430\u043a \u0441\u043d\u0438\u0436\u0435\u043d\u0438\u0435 \u0432\u0435\u0441\u0430.  \u0427\u0442\u043e\u0431\u044b \u0438\u0437\u0431\u0435\u0436\u0430\u0442\u044c \u0443\u0445\u0443\u0434\u0448\u0435\u043d\u0438\u044f \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438, \u043d\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u0438\u0441\u043a\u043b\u044e\u0447\u0438\u0442\u044c \u0441\u043d\u0438\u0436\u0435\u043d\u0438\u0435 \u0432\u0435\u0441\u0430 \u0438\u0437 \u0441\u043b\u043e\u0435\u0432 \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u043f\u0430\u0440\u0442\u0438\u0438. <\/p>\n
    import<\/span> pytorch_lightning <\/span>as<\/span> pl<\/span><\/p>
    import<\/span> torch<\/span><\/p>
    import<\/span> torch<\/span>.<\/span>nn<\/span>.<\/span>functional <\/span>as<\/span> F<\/span><\/p>
    from<\/span> pl_bolts<\/span>.<\/span>optimizers<\/span>.<\/span>lr_scheduler <\/span>import<\/span> LinearWarmupCosineAnnealingLR<\/span><\/p>
    from<\/span> torch<\/span>.<\/span>optim <\/span>import<\/span> SGD<\/span>,<\/span> Adam<\/span><\/p>
    class<\/span> <\/span>AddProjection<\/span>(<\/span>nn<\/span>.<\/span>Module<\/span>)<\/span>:<\/span><\/p>
       <\/span>def<\/span> <\/span>__init__<\/span>(<\/span>self<\/span>,<\/span> config<\/span>,<\/span> model<\/span>=<\/span>None<\/span>,<\/span> mlp_dim<\/span>=<\/span>512<\/span>)<\/span>:<\/span><\/p>
           <\/span>super<\/span>(<\/span>AddProjection<\/span>,<\/span> self<\/span>)<\/span>.<\/span>__init__<\/span>(<\/span>)<\/span><\/p>
           embedding_size <\/span>=<\/span> config<\/span>.<\/span>embedding_size<\/span><\/p>
           self<\/span>.<\/span>backbone <\/span>=<\/span> default<\/span>(<\/span>model<\/span>,<\/span> models<\/span>.<\/span>resnet18<\/span>(<\/span>pretrained<\/span>=<\/span>False<\/span>,<\/span> num_classes<\/span>=<\/span>config<\/span>.<\/span>embedding_size<\/span>)<\/span>)<\/span><\/p>
           mlp_dim <\/span>=<\/span> default<\/span>(<\/span>mlp_dim<\/span>,<\/span> self<\/span>.<\/span>backbone<\/span>.<\/span>fc<\/span>.<\/span>in_features<\/span>)<\/span><\/p>
           <\/span>print<\/span>(<\/span>'Dim MLP input:'<\/span>,<\/span>mlp_dim<\/span>)<\/span><\/p>
           self<\/span>.<\/span>backbone<\/span>.<\/span>fc <\/span>=<\/span> nn<\/span>.<\/span>Identity<\/span>(<\/span>)<\/span><\/p>
           self<\/span>.<\/span>projection <\/span>=<\/span> nn<\/span>.<\/span>Sequential<\/span>(<\/span><\/p>
               nn<\/span>.<\/span>Linear<\/span>(<\/span>in_features<\/span>=<\/span>mlp_dim<\/span>,<\/span> out_features<\/span>=<\/span>mlp_dim<\/span>)<\/span>,<\/span><\/p>
               nn<\/span>.<\/span>BatchNorm1d<\/span>(<\/span>mlp_dim<\/span>)<\/span>,<\/span><\/p>
               nn<\/span>.<\/span>ReLU<\/span>(<\/span>)<\/span>,<\/span><\/p>
               nn<\/span>.<\/span>Linear<\/span>(<\/span>in_features<\/span>=<\/span>mlp_dim<\/span>,<\/span> out_features<\/span>=<\/span>embedding_size<\/span>)<\/span>,<\/span><\/p>
               nn<\/span>.<\/span>BatchNorm1d<\/span>(<\/span>embedding_size<\/span>)<\/span>,<\/span><\/p>
           <\/span>)<\/span><\/p>
       <\/span>def<\/span> <\/span>forward<\/span>(<\/span>self<\/span>,<\/span> x<\/span>,<\/span> return_embedding<\/span>=<\/span>False<\/span>)<\/span>:<\/span><\/p>
           embedding <\/span>=<\/span> self<\/span>.<\/span>backbone<\/span>(<\/span>x<\/span>)<\/span><\/p>
           <\/span>if<\/span> return_embedding<\/span>:<\/span><\/p>
               <\/span>return<\/span> embedding<\/span><\/p>
           <\/span>return<\/span> self<\/span>.<\/span>projection<\/span>(<\/span>embedding<\/span>)<\/span><\/p><\/pre>\n
    \u0421\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u0448\u0430\u0433\u043e\u043c \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0440\u0430\u0437\u0434\u0435\u043b\u0435\u043d\u0438\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432 \u043c\u043e\u0434\u0435\u043b\u0435\u0439 \u043d\u0430 2 \u0433\u0440\u0443\u043f\u043f\u044b. <\/p>\n
    \u0426\u0435\u043b\u044c\u044e \u0432\u0442\u043e\u0440\u043e\u0439 \u0433\u0440\u0443\u043f\u043f\u044b \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0443\u0434\u0430\u043b\u0435\u043d\u0438\u0435 \u043f\u043e\u0442\u0435\u0440\u0438 \u0432\u0435\u0441\u0430 \u0438\u0437 \u0441\u043b\u043e\u0435\u0432 \u043d\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u043f\u0430\u0440\u0442\u0438\u0438.  \u0412 \u0441\u043b\u0443\u0447\u0430\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0442\u043e\u0440\u0430 LARS \u0432\u0430\u043c \u0442\u0430\u043a\u0436\u0435 \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e \u0443\u0434\u0430\u043b\u0438\u0442\u044c \u0443\u043c\u0435\u043d\u044c\u0448\u0435\u043d\u0438\u0435 \u0432\u0435\u0441\u0430 \u0438\u0437 \u0441\u043c\u0435\u0449\u0435\u043d\u0438\u0439.  \u041e\u0434\u043d\u0438\u043c \u0438\u0437 \u0441\u043f\u043e\u0441\u043e\u0431\u043e\u0432 \u0434\u043e\u0441\u0442\u0438\u0436\u0435\u043d\u0438\u044f \u044d\u0442\u043e\u0433\u043e \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0430\u044f \u0444\u0443\u043d\u043a\u0446\u0438\u044f: <\/p>\n
    def<\/span> <\/span>define_param_groups<\/span>(<\/span>model<\/span>,<\/span> weight_decay<\/span>,<\/span> optimizer_name<\/span>)<\/span>:<\/span><\/p>
       <\/span>def<\/span> <\/span>exclude_from_wd_and_adaptation<\/span>(<\/span>name<\/span>)<\/span>:<\/span><\/p>
           <\/span>if<\/span> <\/span>'bn'<\/span> <\/span>in<\/span> name<\/span>:<\/span><\/p>
               <\/span>return<\/span> <\/span>True<\/span><\/p>
           <\/span>if<\/span> optimizer_name <\/span>==<\/span> <\/span>'lars'<\/span> <\/span>and<\/span> <\/span>'bias'<\/span> <\/span>in<\/span> name<\/span>:<\/span><\/p>
               <\/span>return<\/span> <\/span>True<\/span><\/p>
       param_groups <\/span>=<\/span> <\/span>[<\/span><\/p>
           <\/span>{<\/span><\/p>
               <\/span>'params'<\/span>:<\/span> <\/span>[<\/span>p <\/span>for<\/span> name<\/span>,<\/span> p <\/span>in<\/span> model<\/span>.<\/span>named_parameters<\/span>(<\/span>)<\/span> <\/span>if<\/span> <\/span>not<\/span> exclude_from_wd_and_adaptation<\/span>(<\/span>name<\/span>)<\/span>]<\/span>,<\/span><\/p>
               <\/span>'weight_decay'<\/span>:<\/span> weight_decay<\/span>,<\/span><\/p>
               <\/span>'layer_adaptation'<\/span>:<\/span> <\/span>True<\/span>,<\/span><\/p>
           <\/span>}<\/span>,<\/span><\/p>
           <\/span>{<\/span><\/p>
               <\/span>'params'<\/span>:<\/span> <\/span>[<\/span>p <\/span>for<\/span> name<\/span>,<\/span> p <\/span>in<\/span> model<\/span>.<\/span>named_parameters<\/span>(<\/span>)<\/span> <\/span>if<\/span> exclude_from_wd_and_adaptation<\/span>(<\/span>name<\/span>)<\/span>]<\/span>,<\/span><\/p>
               <\/span>'weight_decay'<\/span>:<\/span> <\/span>0<\/span>.<\/span>,<\/span><\/p>
               <\/span>'layer_adaptation'<\/span>:<\/span> <\/span>False<\/span>,<\/span><\/p>
           <\/span>}<\/span>,<\/span><\/p>
       <\/span>]<\/span><\/p>
       <\/span>return<\/span> param_groups<\/span><\/p><\/pre>\n
    \u042f \u043d\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0430\u0442\u043e\u0440 LARS \u0432 \u044d\u0442\u043e\u043c \u0440\u0443\u043a\u043e\u0432\u043e\u0434\u0441\u0442\u0432\u0435, \u043d\u043e \u0435\u0441\u043b\u0438 \u0432\u044b \u043f\u043b\u0430\u043d\u0438\u0440\u0443\u0435\u0442\u0435 \u0435\u0433\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c, \u0432\u043e\u0442 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f, \u043a\u043e\u0442\u043e\u0440\u0443\u044e \u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0441\u043f\u0440\u0430\u0432\u043a\u0438.<\/p>\n
    \u041b\u043e\u0433\u0438\u043a\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f SimCLR<\/h2>\n
    \u0417\u0434\u0435\u0441\u044c \u043c\u044b \u0440\u0435\u0430\u043b\u0438\u0437\u0443\u0435\u043c \u0432\u0441\u044e \u043e\u0431\u0443\u0447\u0430\u044e\u0449\u0443\u044e \u043b\u043e\u0433\u0438\u043a\u0443 SimCLR.  \u0412\u043e\u0437\u044c\u043c\u0438\u0442\u0435 2 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f, \u043f\u0435\u0440\u0435\u0448\u043b\u0438\u0442\u0435 \u0438\u0445, \u0447\u0442\u043e\u0431\u044b \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c \u043f\u0440\u043e\u0433\u043d\u043e\u0437\u044b \u0432\u0441\u0442\u0440\u0430\u0438\u0432\u0430\u043d\u0438\u044f, \u0438 \u0440\u0430\u0441\u0441\u0447\u0438\u0442\u0430\u0439\u0442\u0435 \u043f\u043e\u0442\u0435\u0440\u0438 SimCLR.<\/p>\n