{"id":1617,"date":"2023-08-17T03:04:09","date_gmt":"2023-08-17T03:04:09","guid":{"rendered":"https:\/\/www.gptmain.news\/?p=1617"},"modified":"2023-08-17T03:04:09","modified_gmt":"2023-08-17T03:04:09","slug":"%d0%ba%d0%bb%d0%b0%d1%81%d1%82%d0%b5%d1%80%d0%b8%d0%b7%d0%b0%d1%86%d0%b8%d1%8f-%d0%b4%d0%be%d0%ba%d1%83%d0%bc%d0%b5%d0%bd%d1%82%d0%be%d0%b2-%d0%b8%d0%b8-%d0%bb%d0%b5%d1%82%d0%be-gptmain-news","status":"publish","type":"post","link":"https:\/\/gptmain.news\/?p=1617","title":{"rendered":"\u041a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u044f \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u043e\u0432 | \u0418\u0418 \u041b\u0435\u0442\u043e\n | GPTMain News"},"content":{"rendered":"
\n

\u041a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u044f \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u043e\u0432 \u2014 \u044d\u0442\u043e \u0437\u0430\u0434\u0430\u0447\u0430 \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u0438 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u043e\u0432 \u043f\u043e \u0440\u0430\u0437\u043d\u044b\u043c \u0433\u0440\u0443\u043f\u043f\u0430\u043c \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u0438\u0445 \u0442\u0435\u043a\u0441\u0442\u043e\u0432\u043e\u0433\u043e \u0438 \u0441\u0435\u043c\u0430\u043d\u0442\u0438\u0447\u0435\u0441\u043a\u043e\u0433\u043e \u043a\u043e\u043d\u0442\u0435\u043a\u0441\u0442\u0430. \u042d\u0442\u043e \u043d\u0435\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u0438\u0440\u0443\u0435\u043c\u044b\u0439 \u043c\u0435\u0442\u043e\u0434, \u043f\u043e\u0441\u043a\u043e\u043b\u044c\u043a\u0443 \u0443 \u043d\u0430\u0441 \u043d\u0435\u0442 \u043c\u0435\u0442\u043e\u043a \u0434\u043b\u044f \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u043e\u0432, \u0438 \u043e\u043d \u043f\u0440\u0438\u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f \u0432 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u043e\u043d\u043d\u043e\u043c \u043f\u043e\u0438\u0441\u043a\u0435 \u0438 \u043f\u043e\u0438\u0441\u043a\u043e\u0432\u044b\u0445 \u0441\u0438\u0441\u0442\u0435\u043c\u0430\u0445.<\/p>\n

\u0414\u0430\u0432\u0430\u0439\u0442\u0435 \u043d\u0430\u0447\u043d\u0435\u043c…<\/p>\n

\u0427\u0442\u043e\u0431\u044b \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u0446\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u044b \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u0438\u0445 \u0441\u043e\u0434\u0435\u0440\u0436\u0430\u043d\u0438\u044f, \u044f \u0440\u0435\u0448\u0438\u043b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c \u041a-\u0441\u0440\u0435\u0434\u043d\u0438\u0445. \u0418\u0437-\u0437\u0430 \u0442\u043e\u0433\u043e, \u0447\u0442\u043e \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u044b \u043d\u0435 \u043f\u043e\u043c\u0435\u0447\u0435\u043d\u044b, \u044d\u0442\u043e \u044f\u0432\u043d\u043e \u043f\u0440\u043e\u0431\u043b\u0435\u043c\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0431\u0435\u0437 \u0443\u0447\u0438\u0442\u0435\u043b\u044f, \u0438 \u043e\u0434\u043d\u0438\u043c \u0438\u0437 \u043b\u0443\u0447\u0448\u0438\u0445 \u0440\u0435\u0448\u0435\u043d\u0438\u0439 \u0434\u043e\u043b\u0436\u043d\u043e \u0431\u044b\u0442\u044c K-Means. \u041a\u043e\u043d\u0435\u0447\u043d\u043e, \u043c\u044b \u043c\u043e\u0436\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0434\u0440\u0443\u0433\u043e\u0439 \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c, \u043d\u0430\u043f\u0440\u0438\u043c\u0435\u0440 \u0441\u043c\u0435\u0448\u0430\u043d\u043d\u044b\u0435 \u043c\u043e\u0434\u0435\u043b\u0438 \u0413\u0430\u0443\u0441\u0441\u0430, \u0438\u043b\u0438 \u0434\u0430\u0436\u0435 \u043c\u0435\u0442\u043e\u0434\u044b \u0433\u043b\u0443\u0431\u043e\u043a\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f, \u0442\u0430\u043a\u0438\u0435 \u043a\u0430\u043a \u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440\u044b. \u042f \u0431\u0443\u0434\u0443 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c python \u0441 \u0431\u043b\u043e\u043a\u043d\u043e\u0442\u043e\u043c Jupyter, \u0447\u0442\u043e\u0431\u044b \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0438\u0442\u044c \u043a\u043e\u0434 \u0438 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0441 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0435\u0439.<\/p>\n

\u042f \u0440\u0430\u0437\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u044e \u043a\u043e\u0434 \u0432 \u0441\u0440\u0435\u0434\u0435 Anaconda \u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0435 \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u0438:<\/p>\n

\u041f\u0430\u043d\u0434\u044b \u0434\u043b\u044f \u043f\u0435\u0440\u0435\u0434\u0430\u0447\u0438 \u0434\u0430\u043d\u043d\u044b\u0445<\/p>\n

Sklearn \u0434\u043b\u044f \u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0433\u043e \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0438 \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438<\/p>\n

Matplotlib \u0434\u043b\u044f \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u044f \u0433\u0440\u0430\u0444\u0438\u043a\u043e\u0432<\/p>\n

Ntlk \u0434\u043b\u044f \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u043e\u0432 \u0435\u0441\u0442\u0435\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0433\u043e \u044f\u0437\u044b\u043a\u0430<\/p>\n

BeautifulSoup \u0434\u043b\u044f \u0430\u043d\u0430\u043b\u0438\u0437\u0430 \u0442\u0435\u043a\u0441\u0442\u0430 \u0438\u0437 XML-\u0444\u0430\u0439\u043b\u0430 \u0438 \u0438\u0437\u0431\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u043e\u0442 \u043a\u0430\u0442\u0435\u0433\u043e\u0440\u0438\u0439<\/p>\n

\u0410\u043d\u0430\u043b\u0438\u0437 \u0434\u0430\u043d\u043d\u044b\u0445<\/h2>\n

\u0424\u0443\u043d\u043a\u0446\u0438\u044f parseXML \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442 xml.etree.ElementTree \u0434\u043b\u044f \u0430\u043d\u0430\u043b\u0438\u0437\u0430 \u0434\u0430\u043d\u043d\u044b\u0445. \u042f \u0440\u0435\u0448\u0438\u043b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0434\u043b\u044f \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u0438 \u0442\u043e\u043b\u044c\u043a\u043e \u043d\u0430\u0437\u0432\u0430\u043d\u0438\u0435 \u0438 \u043e\u043f\u0438\u0441\u0430\u043d\u0438\u0435 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043e\u0432, \u043d\u0430\u0438\u0431\u043e\u043b\u0435\u0435 \u0440\u0435\u043b\u0435\u0432\u0430\u043d\u0442\u043d\u044b\u0445 \u0441\u0435\u043c\u0430\u0441\u0438\u043e\u043b\u043e\u0433\u0438\u0438. \u0418\u0437-\u0437\u0430 \u0442\u043e\u0433\u043e, \u0447\u0442\u043e \u043e\u043f\u0438\u0441\u0430\u043d\u0438\u0435 \u043d\u0435 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f raw tex, \u043c\u044b \u0438\u0437\u0432\u043b\u0435\u043a\u0430\u0435\u043c \u0442\u0435\u043a\u0441\u0442 \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u043e\u0439 BeautifulSoup, \u043a\u0430\u043a \u044f \u0443\u0436\u0435 \u0443\u043f\u043e\u043c\u0438\u043d\u0430\u043b. \u0422\u0430\u043a\u0436\u0435 \u043c\u044b \u043e\u0442\u0431\u0440\u0430\u0441\u044b\u0432\u0430\u0435\u043c \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u044b \u0441 \u043e\u0447\u0435\u043d\u044c \u043c\u0430\u043b\u0435\u043d\u044c\u043a\u0438\u043c \u043e\u043f\u0438\u0441\u0430\u043d\u0438\u0435\u043c, \u043f\u043e\u0442\u043e\u043c\u0443 \u0447\u0442\u043e \u043e\u043d\u0438 \u0432\u043b\u0438\u044f\u044e\u0442 \u043d\u0430 \u043e\u043a\u043e\u043d\u0447\u0430\u0442\u0435\u043b\u044c\u043d\u0443\u044e \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u044e. \u041c\u043e\u0436\u043d\u043e \u0441\u0447\u0438\u0442\u0430\u0442\u044c, \u0447\u0442\u043e \u0432\u0441\u0435 \u043e\u043d\u0438 \u043f\u0440\u0438\u043d\u0430\u0434\u043b\u0435\u0436\u0430\u0442 \u0434\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u043c\u0443 \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0443. \u041a\u043e\u043d\u0435\u0447\u043d\u043e, \u0435\u0441\u0442\u044c \u0441\u043f\u043e\u0441\u043e\u0431\u044b \u0438\u0445 \u0432\u043a\u043b\u044e\u0447\u0438\u0442\u044c, \u043d\u043e \u044f \u043f\u043e\u043a\u0430 \u0438\u043c\u0438 \u043d\u0435 \u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0441\u044c.<\/p>\n

import<\/span> xml<\/span>.<\/span>etree<\/span>.<\/span>ElementTree <\/span>as<\/span> ET<\/span><\/p>
import<\/span> pandas <\/span>as<\/span> pd<\/span><\/p>
import<\/span> nltk<\/span><\/p>
from<\/span> sklearn<\/span>.<\/span>cluster <\/span>import<\/span> KMeans<\/span><\/p>
from<\/span> sklearn<\/span>.<\/span>externals <\/span>import<\/span> joblib<\/span><\/p>
from<\/span> sklearn<\/span>.<\/span>feature_extraction<\/span>.<\/span>text <\/span>import<\/span> TfidfVectorizer<\/span><\/p>
from<\/span> sklearn<\/span>.<\/span>metrics<\/span>.<\/span>pairwise <\/span>import<\/span> cosine_similarity<\/span><\/p>
nltk<\/span>.<\/span>download<\/span>(<\/span>'punkt'<\/span>)<\/span><\/p>
from<\/span> bs4 <\/span>import<\/span> BeautifulSoup<\/span><\/p>
from<\/span> nltk <\/span>import<\/span> SnowballStemmer<\/span><\/p>
import<\/span> re<\/span><\/p>
def<\/span> <\/span>parseXML<\/span>(<\/span>xmlfile<\/span>)<\/span>:<\/span><\/p>
    tree <\/span>=<\/span> ET<\/span>.<\/span>parse<\/span>(<\/span>xmlfile<\/span>)<\/span><\/p>
    root <\/span>=<\/span> tree<\/span>.<\/span>getroot<\/span>(<\/span>)<\/span><\/p>
    titles<\/span>=<\/span>[<\/span>]<\/span><\/p>
    descriptions<\/span>=<\/span>[<\/span>]<\/span><\/p>
    <\/span>for<\/span> item <\/span>in<\/span> root<\/span>.<\/span>findall<\/span>(<\/span>'.\/channel\/item'<\/span>)<\/span>:<\/span><\/p>
        <\/span>for<\/span> child <\/span>in<\/span> item<\/span>:<\/span><\/p>
            <\/span>if<\/span>(<\/span>child<\/span>.<\/span>tag<\/span>==<\/span>'title'<\/span> <\/span>)<\/span>:<\/span><\/p>
                titles<\/span>.<\/span>append<\/span>(<\/span>child<\/span>.<\/span>text<\/span>)<\/span><\/p>
            <\/span>if<\/span> <\/span>(<\/span>child<\/span>.<\/span>tag <\/span>==<\/span> <\/span>'description'<\/span> <\/span>)<\/span>:<\/span><\/p>
                soup <\/span>=<\/span> BeautifulSoup<\/span>(<\/span>str<\/span>(<\/span>child<\/span>.<\/span>text<\/span>)<\/span>.<\/span>encode<\/span>(<\/span>'utf8'<\/span>,<\/span>'ignore'<\/span>)<\/span>,<\/span> <\/span>\"lxml\"<\/span>)<\/span><\/p>
                strtext<\/span>=<\/span>soup<\/span>.<\/span>text<\/span>.<\/span>replace<\/span>(<\/span>u'\\xa0'<\/span>,<\/span> <\/span>u' '<\/span>)<\/span>.<\/span>replace<\/span>(<\/span>'\\n'<\/span>,<\/span>' '<\/span>)<\/span><\/p>
                descriptions<\/span>.<\/span>append<\/span>(<\/span>strtext<\/span>)<\/span><\/p>
    <\/span>return<\/span> titles<\/span>,<\/span>descriptions<\/span><\/p>
bef_titles<\/span>,<\/span>bef_descriptions <\/span>=<\/span> parseXML<\/span>(<\/span>'data.source.rss-feeds.xml'<\/span>)<\/span><\/p>
print<\/span>(<\/span>'Count of items before dropping:'<\/span> <\/span>,<\/span>len<\/span>(<\/span>bef_titles<\/span>)<\/span>)<\/span><\/p>
titles<\/span>=<\/span>[<\/span>]<\/span><\/p>
descriptions<\/span>=<\/span>[<\/span>]<\/span><\/p>
for<\/span> i <\/span>in<\/span> <\/span>range<\/span>(<\/span>len<\/span>(<\/span>bef_titles<\/span>)<\/span>)<\/span>:<\/span><\/p>
    <\/span>if<\/span> <\/span>(<\/span> <\/span>len<\/span>(<\/span>bef_descriptions<\/span>[<\/span>i<\/span>]<\/span>)<\/span> <\/span>><\/span> <\/span>500<\/span>)<\/span>:<\/span><\/p>
        titles<\/span>.<\/span>append<\/span>(<\/span>bef_titles<\/span>[<\/span>i<\/span>]<\/span>)<\/span><\/p>
        descriptions<\/span>.<\/span>append<\/span>(<\/span>bef_descriptions<\/span>[<\/span>i<\/span>]<\/span>)<\/span><\/p>
print<\/span>(<\/span>'Count of items after:'<\/span> <\/span>,<\/span>len<\/span>(<\/span>titles<\/span>)<\/span>)<\/span><\/p><\/pre>\n
[nltk_data] Downloading package punkt to<\/span><\/p>
[nltk_data]     C:\\Users\\sergi\\AppData\\Roaming\\nltk_data...<\/span><\/p>
[nltk_data]   Package punkt is already up-to-date!<\/span><\/p>
Count of items before dropping: 1662<\/span><\/p>
Count of items after: 1130<\/span><\/p><\/pre>\n
\u0422\u043e\u043a\u0435\u043d\u0438\u0437\u0430\u0446\u0438\u044f \u0438 \u0441\u0442\u0435\u043c\u043c\u0438\u043d\u0433<\/h2>\n
\u0421\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u0448\u0430\u0433\u043e\u043c \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0440\u0430\u0437\u0431\u0438\u0435\u043d\u0438\u0435 \u0442\u0435\u043a\u0441\u0442\u0430 \u043d\u0430 \u0441\u043b\u043e\u0432\u0430, \u0443\u0434\u0430\u043b\u0435\u043d\u0438\u0435 \u0432\u0441\u0435\u0445 \u043c\u043e\u0440\u0444\u043e\u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u0430\u0444\u0444\u0438\u043a\u0441\u043e\u0432 \u0438 \u0443\u0434\u0430\u043b\u0435\u043d\u0438\u0435 \u043e\u0431\u0449\u0438\u0445 \u0441\u043b\u043e\u0432, \u0442\u0430\u043a\u0438\u0445 \u043a\u0430\u043a \u0430\u0440\u0442\u0438\u043a\u043b\u0438 \u0438 \u043f\u0440\u0435\u0434\u043b\u043e\u0433\u0438.  \u042d\u0442\u043e \u043c\u043e\u0436\u043d\u043e \u0441\u0434\u0435\u043b\u0430\u0442\u044c \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e \u0432\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u044b\u0445 \u0444\u0443\u043d\u043a\u0446\u0438\u0439 ntlk.  \u0412 \u043a\u043e\u043d\u0446\u0435 \u043a\u043e\u043d\u0446\u043e\u0432, \u043c\u044b \u043f\u043e\u043b\u0443\u0447\u0430\u0435\u043c \u0434\u0432\u0430 \u0440\u0430\u0437\u043d\u044b\u0445 \u0441\u043b\u043e\u0432\u0430\u0440\u044f (\u043e\u0434\u0438\u043d \u0442\u043e\u043a\u0435\u043d\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0439 \u0438 \u0431\u0430\u0437\u043e\u0432\u044b\u0439 \u0438 \u043e\u0434\u0438\u043d \u0442\u043e\u043b\u044c\u043a\u043e \u0442\u043e\u043a\u0435\u043d\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0439), \u0438 \u043c\u044b \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u044f\u0435\u043c \u0438\u0445 \u0432 \u043a\u0430\u0434\u0440 \u0434\u0430\u043d\u043d\u044b\u0445 pandas.<\/p>\n
def<\/span> <\/span>tokenize_and_stem<\/span>(<\/span>text<\/span>)<\/span>:<\/span><\/p>
    tokens <\/span>=<\/span> <\/span>[<\/span>word <\/span>for<\/span> sent <\/span>in<\/span> nltk<\/span>.<\/span>sent_tokenize<\/span>(<\/span>text<\/span>)<\/span> <\/span>for<\/span> word <\/span>in<\/span> nltk<\/span>.<\/span>word_tokenize<\/span>(<\/span>sent<\/span>)<\/span>]<\/span><\/p>
    filtered_tokens <\/span>=<\/span> <\/span>[<\/span>]<\/span><\/p>
    <\/span>for<\/span> token <\/span>in<\/span> tokens<\/span>:<\/span><\/p>
        <\/span>if<\/span> re<\/span>.<\/span>search<\/span>(<\/span>'[a-zA-Z]'<\/span>,<\/span> token<\/span>)<\/span>:<\/span><\/p>
            filtered_tokens<\/span>.<\/span>append<\/span>(<\/span>token<\/span>)<\/span><\/p>
    stems <\/span>=<\/span> <\/span>[<\/span>stemmer<\/span>.<\/span>stem<\/span>(<\/span>t<\/span>)<\/span> <\/span>for<\/span> t <\/span>in<\/span> filtered_tokens<\/span>]<\/span><\/p>
    <\/span>return<\/span> stems<\/span><\/p>
def<\/span> <\/span>tokenize_only<\/span>(<\/span>text<\/span>)<\/span>:<\/span><\/p>
    tokens <\/span>=<\/span> <\/span>[<\/span>word<\/span>.<\/span>lower<\/span>(<\/span>)<\/span> <\/span>for<\/span> sent <\/span>in<\/span> nltk<\/span>.<\/span>sent_tokenize<\/span>(<\/span>text<\/span>)<\/span> <\/span>for<\/span> word <\/span>in<\/span> nltk<\/span>.<\/span>word_tokenize<\/span>(<\/span>sent<\/span>)<\/span>]<\/span><\/p>
    filtered_tokens <\/span>=<\/span> <\/span>[<\/span>]<\/span><\/p>
    <\/span>for<\/span> token <\/span>in<\/span> tokens<\/span>:<\/span><\/p>
        <\/span>if<\/span> re<\/span>.<\/span>search<\/span>(<\/span>'[a-zA-Z]'<\/span>,<\/span> token<\/span>)<\/span>:<\/span><\/p>
            filtered_tokens<\/span>.<\/span>append<\/span>(<\/span>token<\/span>)<\/span><\/p>
    <\/span>return<\/span> filtered_tokens<\/span><\/p><\/pre>\n
stemmer <\/span>=<\/span> SnowballStemmer<\/span>(<\/span>\"english\"<\/span>)<\/span><\/p>
totalvocab_stemmed <\/span>=<\/span> <\/span>[<\/span>]<\/span><\/p>
totalvocab_tokenized <\/span>=<\/span> <\/span>[<\/span>]<\/span><\/p>
for<\/span> i <\/span>in<\/span> descriptions<\/span>:<\/span><\/p>
    allwords_stemmed <\/span>=<\/span> tokenize_and_stem<\/span>(<\/span>i<\/span>)<\/span><\/p>
    totalvocab_stemmed<\/span>.<\/span>extend<\/span>(<\/span>allwords_stemmed<\/span>)<\/span><\/p>
    allwords_tokenized <\/span>=<\/span> tokenize_only<\/span>(<\/span>i<\/span>)<\/span><\/p>
    totalvocab_tokenized<\/span>.<\/span>extend<\/span>(<\/span>allwords_tokenized<\/span>)<\/span><\/p>
vocab_frame <\/span>=<\/span> pd<\/span>.<\/span>DataFrame<\/span>(<\/span>{<\/span>'words'<\/span>:<\/span> totalvocab_tokenized<\/span>}<\/span>,<\/span> index<\/span>=<\/span>totalvocab_stemmed<\/span>)<\/span><\/p>
print<\/span>(<\/span>'there are '<\/span> <\/span>+<\/span> <\/span>str<\/span>(<\/span>vocab_frame<\/span>.<\/span>shape<\/span>[<\/span>0<\/span>]<\/span>)<\/span> <\/span>+<\/span> <\/span>' items in vocab_frame'<\/span>)<\/span><\/p><\/pre>\n
there are 481437 items in vocab_frame<\/span><\/p><\/pre>\n
\u0412\u0435\u043a\u0442\u043e\u0440\u0438\u0437\u0430\u0446\u0438\u044f \u0438 \u0441\u0442\u0435\u043c\u043c\u0438\u043d\u0433<\/h2>\n
\u041f\u0440\u0435\u0436\u0434\u0435 \u0447\u0435\u043c \u043c\u044b \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u043c \u0434\u0430\u043d\u043d\u044b\u0435 \u0432 \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c K-\u0441\u0440\u0435\u0434\u043d\u0438\u0445, \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e \u0438\u0445 \u0432\u0435\u043a\u0442\u043e\u0440\u0438\u0437\u043e\u0432\u0430\u0442\u044c.  \u0421\u0430\u043c\u044b\u0439 \u043f\u043e\u043f\u0443\u043b\u044f\u0440\u043d\u044b\u0439 \u043c\u0435\u0442\u043e\u0434 \u2014 Tdidf Vectorizer, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0441\u043e\u0437\u0434\u0430\u0435\u0442 \u043c\u0430\u0442\u0440\u0438\u0446\u0443 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u0447\u0430\u0441\u0442\u043e\u0442\u044b \u0441\u043b\u043e\u0432 \u0432 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0445, \u0438 \u0438\u043c\u0435\u043d\u043d\u043e \u0435\u0433\u043e \u043c\u044b \u0441\u043e\u0431\u0438\u0440\u0430\u0435\u043c\u0441\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c.  \u041f\u043e \u0441\u0443\u0442\u0438, \u044d\u0442\u043e \u043f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442, \u043d\u0430\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u0432\u0430\u0436\u043d\u043e \u0441\u043b\u043e\u0432\u043e \u0434\u043b\u044f \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430. \u0421\u0442\u043e\u0438\u0442 \u043e\u0442\u043c\u0435\u0442\u0438\u0442\u044c, \u0447\u0442\u043e \u0432 \u0431\u0443\u0434\u0443\u0449\u0435\u0439 \u0440\u0430\u0431\u043e\u0442\u0435 word2vec \u0438 doc2vec \u043c\u043e\u0433\u0443\u0442 \u0431\u044b\u0442\u044c \u0433\u043e\u0440\u0430\u0437\u0434\u043e \u0431\u043e\u043b\u0435\u0435 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u044b\u043c\u0438 \u0434\u043b\u044f \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u043e\u0442\u043d\u043e\u0448\u0435\u043d\u0438\u0439 \u043c\u0435\u0436\u0434\u0443 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u0430\u043c\u0438.<\/p>\n
tfidf_vectorizer <\/span>=<\/span> TfidfVectorizer<\/span>(<\/span>max_df<\/span>=<\/span>0.8<\/span>,<\/span> max_features<\/span>=<\/span>200000<\/span>,<\/span>min_df<\/span>=<\/span>0.2<\/span>,<\/span> stop_words<\/span>=<\/span>'english'<\/span>,<\/span><\/p>
                                 use_idf<\/span>=<\/span>True<\/span>,<\/span> tokenizer<\/span>=<\/span>tokenize_and_stem<\/span>,<\/span> ngram_range<\/span>=<\/span>(<\/span>1<\/span>,<\/span>3<\/span>)<\/span>)<\/span><\/p>
tfidf_matrix <\/span>=<\/span> tfidf_vectorizer<\/span>.<\/span>fit_transform<\/span>(<\/span>descriptions<\/span>)<\/span><\/p>
print<\/span>(<\/span>'Td idf Matrix shape: '<\/span>,<\/span>tfidf_matrix<\/span>.<\/span>shape<\/span>)<\/span><\/p>
terms <\/span>=<\/span> tfidf_vectorizer<\/span>.<\/span>get_feature_names<\/span>(<\/span>)<\/span><\/p>
dist <\/span>=<\/span> <\/span>1<\/span> <\/span>-<\/span> cosine_similarity<\/span>(<\/span>tfidf_matrix<\/span>)<\/span><\/p><\/pre>\n
Td idf Matrix shape:  (1130, 74)<\/span><\/p><\/pre>\n
\u041a \u043e\u0437\u043d\u0430\u0447\u0430\u0435\u0442<\/h2>\n
\u0417\u0434\u0435\u0441\u044c \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0444\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u0430\u044f \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u044f, \u0433\u0434\u0435 K \u043e\u0437\u043d\u0430\u0447\u0430\u0435\u0442 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 5 \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u043e\u0432 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u043c\u0430\u0442\u0440\u0438\u0446\u044b Td-idf.  \u041c\u044b \u043c\u043e\u0436\u0435\u043c \u043b\u0435\u0433\u043a\u043e \u043f\u0440\u0435\u0434\u0441\u043a\u0430\u0437\u0430\u0442\u044c, \u0447\u0442\u043e \u044d\u0442\u043e \u043d\u0435 \u0431\u0443\u0434\u0435\u0442 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u043c \u0440\u0435\u0448\u0435\u043d\u0438\u0435\u043c, \u043f\u043e\u0442\u043e\u043c\u0443 \u0447\u0442\u043e \u043e\u043d\u043e \u0443\u0447\u0438\u0442\u044b\u0432\u0430\u0435\u0442 \u0442\u043e\u043b\u044c\u043a\u043e \u0447\u0430\u0441\u0442\u043e\u0442\u0443 \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0441\u043b\u043e\u0432\u0430 \u0432 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0435.<\/p>\n
num_clusters <\/span>=<\/span> <\/span>5<\/span><\/p>
km <\/span>=<\/span> KMeans<\/span>(<\/span>n_clusters<\/span>=<\/span>num_clusters<\/span>)<\/span><\/p>
km<\/span>.<\/span>fit<\/span>(<\/span>tfidf_matrix<\/span>)<\/span><\/p>
clusters <\/span>=<\/span> km<\/span>.<\/span>labels_<\/span>.<\/span>tolist<\/span>(<\/span>)<\/span><\/p><\/pre>\n
\u0427\u0442\u043e\u0431\u044b \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u0438\u0442\u044c \u043a\u043b\u0430\u0441\u0442\u0435\u0440, \u044f \u0441\u043e\u0437\u0434\u0430\u044e \u043a\u0430\u0434\u0440 \u0434\u0430\u043d\u043d\u044b\u0445 pandas, \u0438\u043d\u0434\u0435\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0439 \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430\u043c\u0438.  \u041d\u0438\u0436\u0435 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u044b \u043f\u0435\u0440\u0432\u044b\u0435 6 \u0441\u043b\u043e\u0432 \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430.  \u041c\u044b \u0437\u0430\u043c\u0435\u0447\u0430\u0435\u043c, \u0447\u0442\u043e \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u044f \u0434\u0430\u043b\u0435\u043a\u0430 \u043e\u0442 \u0441\u043e\u0432\u0435\u0440\u0448\u0435\u043d\u0441\u0442\u0432\u0430, \u043f\u043e\u0441\u043a\u043e\u043b\u044c\u043a\u0443 \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0441\u043b\u043e\u0432\u0430 \u0432\u0445\u043e\u0434\u044f\u0442 \u0431\u043e\u043b\u0435\u0435 \u0447\u0435\u043c \u0432 \u043e\u0434\u043d\u0443 \u0433\u0440\u0443\u043f\u043f\u0443.  \u0422\u0430\u043a\u0436\u0435 \u043d\u0435\u0442 \u0447\u0435\u0442\u043a\u043e\u0433\u043e \u0440\u0430\u0437\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f \u0441\u043c\u044b\u0441\u043b\u043e\u0432\u043e\u0433\u043e \u0441\u043e\u0434\u0435\u0440\u0436\u0430\u043d\u0438\u044f \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u043e\u0432.  \u041c\u044b \u043b\u0435\u0433\u043a\u043e \u043c\u043e\u0436\u0435\u043c \u0432\u0438\u0434\u0435\u0442\u044c, \u0447\u0442\u043e \u0442\u0435\u0440\u043c\u0438\u043d\u044b, \u0441\u0432\u044f\u0437\u0430\u043d\u043d\u044b\u0435 \u0441 \u0440\u0430\u0431\u043e\u0442\u043e\u0439, \u0432\u043a\u043b\u044e\u0447\u0430\u044e\u0442 \u0431\u043e\u043b\u0435\u0435 \u043e\u0434\u043d\u043e\u0433\u043e \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430.<\/p>\n
items <\/span>=<\/span> <\/span>{<\/span> <\/span>'title'<\/span>:<\/span> titles<\/span>,<\/span> <\/span>'description'<\/span>:<\/span> descriptions<\/span>}<\/span><\/p>
frame <\/span>=<\/span> pd<\/span>.<\/span>DataFrame<\/span>(<\/span>items<\/span>,<\/span> index <\/span>=<\/span> <\/span>[<\/span>clusters<\/span>]<\/span> <\/span>,<\/span> columns <\/span>=<\/span> <\/span>[<\/span> <\/span>'title'<\/span>,<\/span>'cluster'<\/span>]<\/span>)<\/span><\/p>
print<\/span>(<\/span>\"Top terms per cluster:\"<\/span>)<\/span><\/p>
order_centroids <\/span>=<\/span> km<\/span>.<\/span>cluster_centers_<\/span>.<\/span>argsort<\/span>(<\/span>)<\/span>[<\/span>:<\/span>,<\/span> <\/span>:<\/span>:<\/span>-<\/span>1<\/span>]<\/span><\/p>
for<\/span> i <\/span>in<\/span> <\/span>range<\/span>(<\/span>num_clusters<\/span>)<\/span>:<\/span><\/p>
    <\/span>print<\/span>(<\/span>\"Cluster %d words:\"<\/span> <\/span>%<\/span> i<\/span>,<\/span> end<\/span>=<\/span>''<\/span>)<\/span><\/p>
    <\/span>for<\/span> ind <\/span>in<\/span> order_centroids<\/span>[<\/span>i<\/span>,<\/span> <\/span>:<\/span>6<\/span>]<\/span>:<\/span>  <\/span><\/p>
        <\/span>print<\/span>(<\/span>' %s'<\/span> <\/span>%<\/span> vocab_frame<\/span>.<\/span>ix<\/span>[<\/span>terms<\/span>[<\/span>ind<\/span>]<\/span>.<\/span>split<\/span>(<\/span>' '<\/span>)<\/span>]<\/span>.<\/span>values<\/span>.<\/span>tolist<\/span>(<\/span>)<\/span>[<\/span>0<\/span>]<\/span>[<\/span>0<\/span>]<\/span>,<\/span> end<\/span>=<\/span>','<\/span>)<\/span><\/p>
    <\/span>print<\/span>(<\/span>)<\/span><\/p><\/pre>\n
Top terms per cluster:<\/span><\/p>
Cluster 0 words: labour, employability, european, social, work, eu,<\/span><\/p>
Cluster 1 words: occupational, sectors, skill, employability, services, workers,<\/span><\/p>
Cluster 2 words: skill, job, labour, develop, market, cedefop,<\/span><\/p>
Cluster 3 words: education, training, learning, vocational, education, cedefop,<\/span><\/p>
Cluster 4 words: rates, unemployment, area, employability, increasingly, stated,<\/span><\/p><\/pre>\n
\u0412\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f<\/h2>\n
\u0427\u0442\u043e\u0431\u044b \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0438\u0437\u0430\u0446\u0438\u044e, \u043c\u044b \u0434\u043e\u043b\u0436\u043d\u044b \u0441\u043d\u0430\u0447\u0430\u043b\u0430 \u0443\u043c\u0435\u043d\u044c\u0448\u0438\u0442\u044c \u0438\u0445 \u0440\u0430\u0437\u043c\u0435\u0440\u043d\u043e\u0441\u0442\u044c.  \u041c\u044b \u0434\u043e\u0441\u0442\u0438\u0433\u043b\u0438 \u044d\u0442\u043e\u0433\u043e \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e t-SNE (t-Distributed Stochastic Neighbor Embedding) \u0438\u0437 \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438 sklearn.manifold.  \u0414\u0440\u0443\u0433\u043e\u0439 \u0441\u043f\u043e\u0441\u043e\u0431 \u2014 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c PCA \u0438\u043b\u0438 Multi-Demiensional Scaling (MDS).<\/p>\n
\u0413\u0440\u0430\u0444\u0438\u043a\u0430 \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442\u0441\u044f \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438 matplotlib.<\/p>\n
import<\/span> os  <\/span><\/p>
import<\/span> matplotlib<\/span>.<\/span>pyplot <\/span>as<\/span> plt<\/span><\/p>
import<\/span> matplotlib <\/span>as<\/span> mpl<\/span><\/p>
from<\/span> sklearn<\/span>.<\/span>manifold <\/span>import<\/span> TSNE<\/span><\/p>
tsne <\/span>=<\/span> TSNE<\/span>(<\/span>n_components<\/span>=<\/span>2<\/span>,<\/span> verbose<\/span>=<\/span>1<\/span>,<\/span> perplexity<\/span>=<\/span>40<\/span>,<\/span> n_iter<\/span>=<\/span>300<\/span>)<\/span><\/p>
pos <\/span>=<\/span> tsne<\/span>.<\/span>fit_transform<\/span>(<\/span>dist<\/span>)<\/span><\/p>
xs<\/span>,<\/span> ys <\/span>=<\/span> pos<\/span>[<\/span>:<\/span>,<\/span> <\/span>0<\/span>]<\/span>,<\/span> pos<\/span>[<\/span>:<\/span>,<\/span> <\/span>1<\/span>]<\/span><\/p>
cluster_colors <\/span>=<\/span> <\/span>{<\/span>0<\/span>:<\/span> <\/span>'#1b9e77'<\/span>,<\/span> <\/span>1<\/span>:<\/span> <\/span>'#d95f02'<\/span>,<\/span> <\/span>2<\/span>:<\/span> <\/span>'#7570b3'<\/span>,<\/span> <\/span>3<\/span>:<\/span> <\/span>'#e7298a'<\/span>,<\/span> <\/span>4<\/span>:<\/span> <\/span>'#66a61e'<\/span>}<\/span><\/p>
cluster_names <\/span>=<\/span> <\/span>{<\/span>0<\/span>:<\/span> <\/span>'A'<\/span>,<\/span>1<\/span>:<\/span> <\/span>'B'<\/span>,<\/span>  <\/span>2<\/span>:<\/span> <\/span>'C'<\/span>,<\/span> <\/span>3<\/span>:<\/span> <\/span>'D'<\/span>,<\/span>  <\/span>4<\/span>:<\/span> <\/span>'E'<\/span>}<\/span><\/p><\/pre>\n
[t-SNE] Computing pairwise distances...<\/span><\/p>
[t-SNE] Computing 121 nearest neighbors...<\/span><\/p>
[t-SNE] Computed conditional probabilities for sample 1000 \/ 1130<\/span><\/p>
[t-SNE] Computed conditional probabilities for sample 1130 \/ 1130<\/span><\/p>
[t-SNE] Mean sigma: 1.785805<\/span><\/p>
[t-SNE] KL divergence after 100 iterations with early exaggeration: 0.947952<\/span><\/p>
[t-SNE] Error after 125 iterations: 0.947952<\/span><\/p><\/pre>\n
%<\/span>matplotlib inline<\/span><\/p>
df <\/span>=<\/span> pd<\/span>.<\/span>DataFrame<\/span>(<\/span>dict<\/span>(<\/span>x<\/span>=<\/span>xs<\/span>,<\/span> y<\/span>=<\/span>ys<\/span>,<\/span> label<\/span>=<\/span>clusters<\/span>,<\/span> title<\/span>=<\/span>titles<\/span>)<\/span>)<\/span><\/p>
groups <\/span>=<\/span> df<\/span>.<\/span>groupby<\/span>(<\/span>'label'<\/span>)<\/span><\/p>
fig<\/span>,<\/span> ax <\/span>=<\/span> plt<\/span>.<\/span>subplots<\/span>(<\/span>figsize<\/span>=<\/span>(<\/span>16<\/span>,<\/span>8<\/span>)<\/span> <\/span>)<\/span><\/p>
for<\/span> name<\/span>,<\/span> group <\/span>in<\/span> groups<\/span>:<\/span><\/p>
    ax<\/span>.<\/span>plot<\/span>(<\/span>group<\/span>.<\/span>x<\/span>,<\/span> group<\/span>.<\/span>y<\/span>,<\/span> marker<\/span>=<\/span>'o'<\/span>,<\/span> linestyle<\/span>=<\/span>''<\/span>,<\/span> ms<\/span>=<\/span>12<\/span>,<\/span><\/p>
            label<\/span>=<\/span>cluster_names<\/span>[<\/span>name<\/span>]<\/span>,<\/span> color<\/span>=<\/span>cluster_colors<\/span>[<\/span>name<\/span>]<\/span>,<\/span> mec<\/span>=<\/span>'none'<\/span>)<\/span><\/p>
ax<\/span>.<\/span>legend<\/span>(<\/span>numpoints<\/span>=<\/span>1<\/span>)<\/span><\/p>
plt<\/span>.<\/span>show<\/span>(<\/span>)<\/span><\/p><\/pre>\n
<\/p>\n
    
\n  \"\u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442<\/p>\n
    <\/span><\/p>\n
\u041c\u044b \u0432\u0438\u0434\u0438\u043c, \u0447\u0442\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u043d\u0435 \u0442\u0430\u043a \u043f\u043b\u043e\u0445\u0438, \u043a\u0430\u043a \u043c\u044b \u0434\u0443\u043c\u0430\u043b\u0438 \u0438\u0437\u043d\u0430\u0447\u0430\u043b\u044c\u043d\u043e.  \u0425\u043e\u0442\u044f \u0435\u0441\u0442\u044c \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u043e\u0435 \u0447\u0430\u0441\u0442\u0438\u0447\u043d\u043e\u0435 \u0441\u043e\u0432\u043f\u0430\u0434\u0435\u043d\u0438\u0435, \u0433\u0440\u0443\u043f\u043f\u044b \u0432\u0435\u0441\u044c\u043c\u0430 \u0440\u0430\u0437\u043b\u0438\u0447\u0430\u044e\u0442\u0441\u044f.  \u041e\u0434\u043d\u0430\u043a\u043e \u043d\u0435\u0442 \u043d\u0438\u043a\u0430\u043a\u0438\u0445 \u0441\u043e\u043c\u043d\u0435\u043d\u0438\u0439 \u0432 \u0442\u043e\u043c, \u0447\u0442\u043e \u043c\u044b \u043c\u043e\u0436\u0435\u043c \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0438\u0445 \u0433\u043e\u0440\u0430\u0437\u0434\u043e \u0434\u0430\u043b\u044c\u0448\u0435.<\/p>\n
\u0421\u043b\u0435\u0434\u0443\u0435\u0442 \u043e\u0442\u043c\u0435\u0442\u0438\u0442\u044c, \u0447\u0442\u043e \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u044b \u0441 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u043c\u0438 \u0441\u043b\u043e\u0432\u0430\u043c\u0438 \u043d\u0435 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u044b \u043d\u0430 \u0433\u0440\u0430\u0444\u0438\u043a\u0435.  \u042f \u0442\u0430\u043a\u0436\u0435 \u0437\u0430\u043c\u0435\u0442\u0438\u043b, \u0447\u0442\u043e \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u044b \u043d\u0430\u043f\u0438\u0441\u0430\u043d\u044b \u043d\u0430 \u044f\u0437\u044b\u043a\u0435, \u043e\u0442\u043b\u0438\u0447\u043d\u043e\u043c \u043e\u0442 \u0430\u043d\u0433\u043b\u0438\u0439\u0441\u043a\u043e\u0433\u043e.  \u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u043c\u044b \u043d\u0435 \u043e\u0431\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u0435\u043c \u0438\u0445, \u0438 \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0438\u0445 \u043a\u043b\u0430\u0441\u0441\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u044f \u0444\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u0438 \u0441\u043b\u0443\u0447\u0430\u0439\u043d\u0430.  \u041d\u0430 \u0434\u0438\u0430\u0433\u0440\u0430\u043c\u043c\u0435 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043d\u0435\u0443\u043c\u0435\u0441\u0442\u043d\u044b\u0445 \u0442\u043e\u0447\u0435\u043a.<\/p>\n
\u041a\u0440\u043e\u043c\u0435 \u0442\u043e\u0433\u043e, \u043f\u0440\u0435\u0434\u0441\u0442\u043e\u0438\u0442 \u0435\u0449\u0435 \u043c\u043d\u043e\u0433\u043e \u0440\u0430\u0431\u043e\u0442\u044b \u043f\u043e \u043e\u0447\u0438\u0441\u0442\u043a\u0435 \u0438 \u043f\u0440\u0435\u0434\u0432\u0430\u0440\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0439 \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0435 \u0434\u0430\u043d\u043d\u044b\u0445.<\/p>\n
\u041e\u0434\u0438\u043d \u0438\u0437 \u0441\u043f\u043e\u0441\u043e\u0431\u043e\u0432 \u2014 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0432\u0435\u043a\u0442\u043e\u0440\u0438\u0437\u0430\u0446\u0438\u0438 tdidf, \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0434\u043b\u044f \u0432\u0435\u043a\u0442\u043e\u0440\u0438\u0437\u0430\u0446\u0438\u0438 doc2vec.  \u0418\u043b\u0438 \u043c\u044b \u043c\u043e\u0436\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0434\u0440\u0443\u0433\u043e\u0439 \u043c\u0435\u0442\u043e\u0434, \u0442\u0430\u043a\u043e\u0439 \u043a\u0430\u043a Affinity Propagation, Spectra Clustering \u0438\u043b\u0438 \u0431\u043e\u043b\u0435\u0435 \u0441\u043e\u0432\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0435 \u043c\u0435\u0442\u043e\u0434\u044b, \u0442\u0430\u043a\u0438\u0435 \u043a\u0430\u043a HDBSCAN \u0438 Variational Autoencoders, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u044f \u0445\u043e\u0442\u0435\u043b \u0431\u044b \u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0430\u0442\u044c.<\/p>\n
PS: \u0427\u0442\u043e\u0431\u044b \u0437\u0430\u043f\u0443\u0441\u0442\u0438\u0442\u044c \u043a\u043e\u0434, \u0432\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u0441\u0434\u0435\u043b\u0430\u0442\u044c \u044d\u0442\u043e \u043f\u0440\u044f\u043c\u043e \u0438\u0437 jupyter, \u0435\u0441\u043b\u0438 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043b\u0435\u043d\u044b \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u044b\u0435 \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u0438, \u0438\u043b\u0438 \u0432\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u044d\u043a\u0441\u043f\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0435\u0433\u043e \u0432 \u0432\u0438\u0434\u0435 \u0444\u0430\u0439\u043b\u0430 .py \u0438 \u0437\u0430\u043f\u0443\u0441\u0442\u0438\u0442\u044c \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e ide \u0438\u043b\u0438 \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u0447\u0435\u0440\u0435\u0437 \u043a\u043e\u043d\u0441\u043e\u043b\u044c.<\/p>\n
\n
\"\u041a\u043d\u0438\u0433\u0430