{"id":11860,"date":"2024-08-11T20:19:58","date_gmt":"2024-08-11T12:19:58","guid":{"rendered":"https:\/\/17aitech.com\/?p=11860"},"modified":"2024-12-04T17:58:32","modified_gmt":"2024-12-04T09:58:32","slug":"%e3%80%90%e8%af%be%e7%a8%8b%e6%80%bb%e7%bb%93%e3%80%91day20%ef%bc%9atransformer%e6%ba%90%e7%a0%81%e6%b7%b1%e5%85%a5%e7%90%86%e8%a7%a3-%e8%ae%ad%e7%bb%83%e8%bf%87%e7%a8%8b","status":"publish","type":"post","link":"https:\/\/17aitech.com\/?p=11860","title":{"rendered":"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_85 ez-toc-wrap-left-text counter-hierarchy ez-toc-counter ez-toc-light-blue ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u6587\u7ae0\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E5%89%8D%E8%A8%80\" >\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E8%AE%AD%E7%BB%83%E6%B5%81%E7%A8%8B\" >\u8bad\u7ec3\u6d41\u7a0b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E4%BB%A3%E7%A0%81%E5%88%86%E6%9E%90%E7%90%86%E8%A7%A3\" >\u4ee3\u7801\u5206\u6790\u7406\u89e3<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E6%95%B0%E6%8D%AE%E5%AF%B9%E9%BD%90_collate_fn\" >\u6570\u636e\u5bf9\u9f50 collate_fn()<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E5%BC%80%E5%A7%8B%E8%AE%AD%E7%BB%83_run_epoch\" >\u5f00\u59cb\u8bad\u7ec3 run_epoch()<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/17aitech.com\/?p=11860\/#EncoderDecoder%E7%9A%84forward%E5%87%BD%E6%95%B0\" >EncoderDecoder\u7684forward()\u51fd\u6570<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/17aitech.com\/?p=11860\/#embeddingsforward\" >embeddings.forward()<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/17aitech.com\/?p=11860\/#Positional_Encoding\" >Positional Encoding<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/17aitech.com\/?p=11860\/#Encoder\" >Encoder<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/17aitech.com\/?p=11860\/#attention%E5%87%BD%E6%95%B0\" >attention\u51fd\u6570<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/17aitech.com\/?p=11860\/#MultiHeadedAttentionforward\" >MultiHeadedAttention.forward()<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/17aitech.com\/?p=11860\/#Decoder\" >Decoder<\/a><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E6%95%B0%E6%8D%AE%E5%BD%A2%E7%8A%B6%E6%A2%B3%E7%90%86\" >\u6570\u636e\u5f62\u72b6\u68b3\u7406<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"https:\/\/17aitech.com\/?p=11860\/#1_%E5%88%9D%E5%A7%8B%E8%BE%93%E5%85%A5_src%E3%80%81src_mask%E3%80%81tgt%E3%80%81tgt_mask%E3%80%81tgt_y\" >1. \u521d\u59cb\u8f93\u5165 src\u3001src_mask\u3001tgt\u3001tgt_mask\u3001tgt_y<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-15\" href=\"https:\/\/17aitech.com\/?p=11860\/#2_%E7%BB%8F%E8%BF%87%E5%B5%8C%E5%85%A5%E5%B1%82_Embeddings\" >2. \u7ecf\u8fc7\u5d4c\u5165\u5c42 (Embeddings)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-16\" href=\"https:\/\/17aitech.com\/?p=11860\/#3_%E4%BD%8D%E7%BD%AE%E7%BC%96%E7%A0%81_PositionalEncoding\" >3. \u4f4d\u7f6e\u7f16\u7801 (PositionalEncoding)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-17\" href=\"https:\/\/17aitech.com\/?p=11860\/#4_%E7%BC%96%E7%A0%81%E5%99%A8%E5%B1%82%EF%BC%88Encoder%EF%BC%89\" >4. \u7f16\u7801\u5668\u5c42\uff08Encoder\uff09<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-18\" href=\"https:\/\/17aitech.com\/?p=11860\/#41_Add_Norm\" >4.1 Add&amp;Norm<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-19\" href=\"https:\/\/17aitech.com\/?p=11860\/#41_%E8%87%AA%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%EF%BC%88MultiHeadedAttention%EF%BC%89\" >4.1 \u81ea\u6ce8\u610f\u529b\u673a\u5236\uff08MultiHeadedAttention\uff09<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-20\" href=\"https:\/\/17aitech.com\/?p=11860\/#5_%E8%A7%A3%E7%A0%81%E5%99%A8%E5%B1%82%EF%BC%88Decoder%EF%BC%89\" >5. \u89e3\u7801\u5668\u5c42\uff08Decoder\uff09<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-21\" href=\"https:\/\/17aitech.com\/?p=11860\/#51_%E7%9B%AE%E6%A0%87%E5%B5%8C%E5%85%A5Embeddings\" >5.1 \u76ee\u6807\u5d4c\u5165(Embeddings)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-22\" href=\"https:\/\/17aitech.com\/?p=11860\/#52_%E4%BD%8D%E7%BD%AE%E7%BC%96%E7%A0%81PositionalEncoding\" >5.2 \u4f4d\u7f6e\u7f16\u7801(PositionalEncoding)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-23\" href=\"https:\/\/17aitech.com\/?p=11860\/#53_%E8%87%AA%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%EF%BC%88Masked_MultiHeadedAttention%EF%BC%89\" >5.3 \u81ea\u6ce8\u610f\u529b\u673a\u5236\uff08Masked MultiHeadedAttention\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-24\" href=\"https:\/\/17aitech.com\/?p=11860\/#54_%E8%9E%8D%E5%90%88%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%EF%BC%88MultiHeadedAttention%EF%BC%89\" >5.4 \u878d\u5408\u6ce8\u610f\u529b\u673a\u5236\uff08MultiHeadedAttention\uff09<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-25\" href=\"https:\/\/17aitech.com\/?p=11860\/#6_%E8%AE%A1%E7%AE%97%E6%8D%9F%E5%A4%B1\" >6. \u8ba1\u7b97\u635f\u5931<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-26\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E5%86%85%E5%AE%B9%E5%B0%8F%E7%BB%93\" >\u5185\u5bb9\u5c0f\u7ed3<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-27\" href=\"https:\/\/17aitech.com\/?p=11860\/#%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99\" >\u53c2\u8003\u8d44\u6599<\/a><\/li><\/ul><\/nav><\/div>\n<h2><span class=\"ez-toc-section\" id=\"%E5%89%8D%E8%A8%80\"><\/span>\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u5728\u4e0a\u4e00\u7ae0<a href=\"https:\/\/17aitech.com\/?p=10297\">\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day19\uff08\u4e0b\uff09\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3<\/a>\u603b\u7ed3\u4e2d\uff0c\u6211\u4eec\u5bf9Transformer\u67b6\u6784\u4ee5\u53ca\u521d\u59cb\u5316\u90e8\u5206\u505a\u4e86\u68b3\u7406\uff0c\u672c\u7ae0\u6211\u4eec\u5c06\u5bf9Transformer\u8bad\u7ec3\u8fc7\u7a0b\u8fdb\u884c\u4ee3\u7801\u5206\u6790\u7406\u89e3\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E8%AE%AD%E7%BB%83%E6%B5%81%E7%A8%8B\"><\/span>\u8bad\u7ec3\u6d41\u7a0b<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p><a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/\u8bad\u7ec3\u8fc7\u7a0b\u65f6\u5e8f\u56fe.png\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/\u8bad\u7ec3\u8fc7\u7a0b\u65f6\u5e8f\u56fe.png\" alt=\"\" \/><\/a><\/p>\n<ul>\n<li>\u8bad\u7ec3\u8fc7\u7a0b\u4e3b\u8981\u7531\u56db\u4e2a\u4e3b\u8981\u90e8\u5206\u7ec4\u6210\uff1a<\/li>\n<li>\u7b2c\u4e00\u90e8\u5206\uff1a\u52a0\u8f7d\u6570\u636e\u96c6\u3002\u901a\u8fc7get_dataloader()\u52a0\u8f7d\u6570\u636e\u96c6\uff0c\u8fd9\u4e00\u8fc7\u7a0b\u4e0eSeq2Seq\u7c7b\u4f3c\uff0c\u8fd9\u91cc\u4e0d\u518d\u8d58\u8ff0\u3002<\/li>\n<li>\u7b2c\u4e8c\u90e8\u5206\uff1a<strong>\u6570\u636e\u5bf9\u9f50<\/strong>\u3002\u8c03\u7528collate_fn()\u51fd\u6570\uff0c\u5bf9\u6570\u636e\u96c6\u6574\u7406\u5e76\u5bf9\u9f50\u3002<\/li>\n<li>\u7b2c\u4e09\u90e8\u5206\uff1a<strong>\u5f00\u59cb\u8bad\u7ec3<\/strong>\u3002\u8c03\u7528run_epoch()\u51fd\u6570\uff0c\u5faa\u73af\u904d\u5386dataloader\u4e2d\u7684\u6570\u636e\u96c6\u5e76\u8fdb\u884c\u8bad\u7ec3\u3002<\/li>\n<li>\u7b2c\u56db\u90e8\u5206\uff1a<strong>\u524d\u5411\u4f20\u64ad<\/strong>\u3002\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\uff0c\u8c03\u7528EncoderDecoder\u7684forward()\u51fd\u6570\u8fdb\u884c\u524d\u5411\u4f20\u64ad\u3002<\/li>\n<\/ul>\n<h2><span class=\"ez-toc-section\" id=\"%E4%BB%A3%E7%A0%81%E5%88%86%E6%9E%90%E7%90%86%E8%A7%A3\"><\/span>\u4ee3\u7801\u5206\u6790\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<h3><span class=\"ez-toc-section\" id=\"%E6%95%B0%E6%8D%AE%E5%AF%B9%E9%BD%90_collate_fn\"><\/span>\u6570\u636e\u5bf9\u9f50 collate_fn()<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u6e90\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-python\">def collate_fn(batch, tokenizer):\n\n    input_sentences, input_sentence_lens, output_sentences, output_sentence_lens = zip(\n        *batch\n    )\n\n    # \u8f6c\u7d22\u5f15\u3010\u6309\u672c\u6279\u91cf\u6700\u5927\u957f\u5ea6\u6765\u586b\u5145\u3011\n    input_sentence_len = max(input_sentence_lens)\n    input_idxes = []\n    for input_sentence in input_sentences:\n        input_idxes.append(tokenizer.encode_input(input_sentence, input_sentence_len))\n\n    # \u8f6c\u7d22\u5f15\u3010\u6309\u672c\u6279\u91cf\u6700\u5927\u957f\u5ea6\u6765\u586b\u5145\u3011\n    output_sentence_len = max(output_sentence_lens)\n    output_idxes = []\n    for output_sentence in output_sentences:\n        output_idxes.append(\n            tokenizer.encode_output(output_sentence, output_sentence_len)\n        )\n    # \u8f6c\u5f20\u91cf [batch_size, seq_len]  src\n    input_idxes = torch.LongTensor(input_idxes)\n    # src_mask [batch_size, 1, seq_len]\n    input_mask = (input_idxes != tokenizer.input_word2idx.get(&quot;&lt;PAD&gt;&quot;)).unsqueeze(-2)\n    # tgt [batch_size, seq_len]\n    output_idxes = torch.LongTensor(output_idxes)\n    # tgt [batch_size, seq_len - 1] \u53bb\u6389\u6700\u540e\u4e00\u4e2a\n    output_idxes_in = output_idxes[:, :-1]\n    # tgt_y [batch_size, seq_len - 1] \u53bb\u6389\u5f00\u5934 \u7684 SOS\n    output_idxes_out = output_idxes[:, 1:]\n    # tgt_mask [batch_size, seq_len-1, seq_len-1]\n    output_mask = tokenizer.make_std_mask(output_idxes_in, tokenizer.output_word2idx.get(&quot;&lt;PAD&gt;&quot;))\n    # \u8bb0\u5f55\u751f\u6210\u7684\u6709\u6548\u5b57\u7b26\n    ntokens = (output_idxes_out != tokenizer.output_word2idx.get(&quot;&lt;PAD&gt;&quot;)).data.sum()\n    # src, src_mask, tgt, tgt_mask, tgt_y, ntokens\n    return input_idxes, input_mask, output_idxes_in, output_mask, output_idxes_out, ntokens<\/code><\/pre>\n<p>\u4ee3\u7801\u7406\u89e3\uff1a<br \/>\n<strong>\u7b2c\u4e00\u6b65<\/strong>\uff1a\u63d0\u53d6\u8f93\u5165\u6570\u636e\u3001\u8f93\u5165\u6570\u636e\u957f\u5ea6\u3001\u8f93\u51fa\u6570\u636e\u3001\u8f93\u51fa\u6570\u636e\u957f\u5ea6<\/p>\n<pre><code class=\"language-python\">    input_sentences, input_sentence_lens, output_sentences, output_sentence_lens = zip(\n        *batch\n    )<\/code><\/pre>\n<ul>\n<li><code>batch<\/code>\uff1a\u4e00\u4e2a\u5305\u542b\u591a\u4e2a\u6837\u672c\u7684\u5217\u8868;<\/li>\n<li><code>zip(*batch)<\/code>\uff1a\u5c06 <code>batch<\/code> \u4e2d\u7684\u6bcf\u4e2a\u6837\u672c\u89e3\u5305\uff0c\u5206\u522b\u63d0\u53d6\u51fa\u8f93\u5165\u53e5\u5b50\u3001\u8f93\u5165\u53e5\u5b50\u957f\u5ea6\u3001\u8f93\u51fa\u53e5\u5b50\u548c\u8f93\u51fa\u53e5\u5b50\u957f\u5ea6\u3002<\/li>\n<\/ul>\n<p>\u793a\u4f8b\u7406\u89e3\uff1a<\/p>\n<pre><code class=\"language-shell\">input_sentences\uff1a                   output_sentences\uff1a\n[&#039;I&#039;, &#039;m&#039;, &#039;sick&#039;, &#039;.&#039;]           [&#039;\u6211&#039;, &#039;\u75c5&#039; , &#039;\u4e86&#039;, &#039;\u3002&#039;]\n[&#039;I&#039;, &#039;m&#039;, &#039;tall&#039;, &#039;.&#039;]           [&#039;\u6211&#039;, &#039;\u4e2a\u5b50&#039; , &#039;\u9ad8&#039;, &#039;\u3002&#039;]\n[&#039;Leave&#039;, &#039;me&#039;, &#039;.&#039;]              [&#039;\u8ba9&#039;, &#039;\u6211&#039;, &#039;\u4e00\u4e2a\u4eba&#039;, &#039;\u5446&#039;,&#039;\u4f1a&#039;, &#039;\u513f&#039;,&#039;\u3002&#039;]\n\ninput_sentence_lens\uff1a               output_sentence_lens:\n[4, 4, 3]                         [4\uff0c 4\uff0c 7]\n<\/code><\/pre>\n<p><strong>\u7b2c\u4e8c\u6b65<\/strong>\uff1a\u5bf9\u6570\u636e\u8fdb\u884c\u586b\u5145<\/p>\n<pre><code class=\"language-python\"># \u8f93\u5165\u7684\u6700\u5927\u957f\u5ea6\u4e3a4\uff0c\u6240\u4ee5input_idxes\u586b\u5145\u4e3a\n[&#039;I&#039;, &#039;m&#039;, &#039;sick&#039;, &#039;.&#039;] \n[&#039;I&#039;, &#039;m&#039;, &#039;tall&#039;, &#039;.&#039;] \n[&#039;Leave&#039;, &#039;me&#039;, &#039;.&#039;, &#039;&lt;PAD&gt;&#039;]   \n\n# \u8f93\u51fa\u7684\u6700\u5927\u957f\u5ea6\u4e3a7\uff0c\u6240\u4ee5output_idxes\u586b\u5145\u4e3a\n[&#039;&lt;SOS&gt;&#039;, &#039;\u6211&#039;, &#039;\u75c5&#039; , &#039;\u4e86&#039;, &#039;\u3002&#039;, &#039;&lt;EOS&gt;&#039;, &#039;&lt;PAD&gt;&#039;, &#039;&lt;PAD&gt;&#039;, &#039;&lt;PAD&gt;&#039;]\n[&#039;&lt;SOS&gt;&#039;, &#039;\u6211&#039;, &#039;\u4e2a\u5b50&#039; , &#039;\u9ad8&#039;, &#039;\u3002&#039;, &#039;&lt;EOS&gt;&#039;, &#039;&lt;PAD&gt;&#039;, &#039;&lt;PAD&gt;&#039;, &#039;&lt;PAD&gt;&#039;]\n[&#039;&lt;SOS&gt;&#039;, &#039;\u8ba9&#039;, &#039;\u6211&#039;, &#039;\u4e00\u4e2a\u4eba&#039;, &#039;\u5446&#039;, &#039;\u4f1a&#039;, &#039;\u513f&#039;, &#039;\u3002&#039;, &#039;&lt;EOS&gt;&#039;]<\/code><\/pre>\n<p><strong>\u7b2c\u4e09\u6b65<\/strong>\uff1a\u751f\u6210input\u7684mask<\/p>\n<pre><code class=\"language-python\">    input_mask = (input_idxes != tokenizer.input_word2idx.get(&quot;&lt;PAD&gt;&quot;)).unsqueeze(-2)<\/code><\/pre>\n<ul>\n<li>\u8f93\u5165\u4f4d\u7f6e\u4e0d\u4e3a<code>&lt;PAD&gt;<\/code>\u7684\u4f4d\u7f6e\uff0c\u503c\u4e3a1\uff0c\u5426\u5219\u4e3a0\uff0c\u4ece\u800c\u5f62\u6210mask\u3002<\/li>\n<\/ul>\n<p><strong>\u7b2c\u56db\u6b65<\/strong>\uff1a\u751f\u6210\u9519\u4f4d\u7684output<\/p>\n<pre><code class=\"language-python\">    # tgt [batch_size, seq_len - 1] \u53bb\u6389\u6700\u540e\u4e00\u4e2a\n    output_idxes_in = output_idxes[:, :-1]\n    # \u4f8b\u5982\uff1a[&#039;&lt;SOS&gt;&#039;, &#039;\u8ba9&#039;, &#039;\u6211&#039;, &#039;\u4e00\u4e2a\u4eba&#039;, &#039;\u5446&#039;, &#039;\u4f1a&#039;, &#039;\u513f&#039;, &#039;\u3002&#039;]\n\n    # tgt_y [batch_size, seq_len - 1] \u53bb\u6389\u5f00\u5934 \u7684 SOS\n    output_idxes_out = output_idxes[:, 1:]\n    # \u4f8b\u5982\uff1a[&#039;\u8ba9&#039;, &#039;\u6211&#039;, &#039;\u4e00\u4e2a\u4eba&#039;, &#039;\u5446&#039;, &#039;\u4f1a&#039;, &#039;\u513f&#039;, &#039;\u3002&#039;, &#039;&lt;EOS&gt;&#039;]<\/code><\/pre>\n<p><strong>\u7b2c\u4e94\u6b65<\/strong>\uff1a\u751f\u6210output\u7684mask<\/p>\n<pre><code class=\"language-python\">    output_mask = tokenizer.make_std_mask(output_idxes_in, tokenizer.output_word2idx.get(&quot;&lt;PAD&gt;&quot;))<\/code><\/pre>\n<pre><code class=\"language-python\">    def make_std_mask(cls, tgt, pad):\n        &quot;Create a mask to hide padding and future words.&quot;\n        tgt_mask = (tgt != pad).unsqueeze(-2)\n        tgt_mask = tgt_mask &amp; Tokenizer.subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data)\n        return tgt_mask<\/code><\/pre>\n<p>\u4ee3\u7801\u7406\u89e3\uff1a<\/p>\n<ul>\n<li>\u56e0\u4e3adecoder\u7684\u63a9\u7801\u591a\u5934\u6ce8\u610f\u529b(mask MultiHeadAttention)\uff0c\u65e2\u8981\u5c4f\u853d\u65e0\u6548\u7684PAD\uff0c\u540c\u65f6\u8fd8\u8981\u5c4f\u853d\u672a\u6765\u8bcd\u3002<\/li>\n<li>\u6240\u4ee5tgt_mask\u662f\u7531<code>tgt_mask &amp; Tokenizer.subsequent_mask<\/code>\u4e24\u90e8\u5206<strong>\u6309\u4f4d\u4e0e<\/strong>\u8fd0\u7b97\uff0c\u5373\u4e24\u8005\u90fd\u4e3a1\u624d\u662f\u6709\u6548\u7684\uff0c\u5982\u679c\u6709\u4e00\u4e2a\u4e3a0\uff0c\u5219\u5bf9\u5e94\u6570\u636e\u88ab\u906e\u6321\u3002<\/li>\n<li><code>Tokenizer.subsequent_mask<\/code> \u662f\u751f\u6210\u4e00\u4e2a\u4e09\u89d2\u77e9\u9635\uff0c\u5982\u4e0b\u56fe\uff1a<br \/>\n<a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/subsequent_mask\u4e09\u89d2\u77e9\u9635.png\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/subsequent_mask\u4e09\u89d2\u77e9\u9635.png\" alt=\"\" \/><\/a><\/li>\n<\/ul>\n<h3><span class=\"ez-toc-section\" id=\"%E5%BC%80%E5%A7%8B%E8%AE%AD%E7%BB%83_run_epoch\"><\/span>\u5f00\u59cb\u8bad\u7ec3 run_epoch()<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<pre><code class=\"language-python\">def run_epoch(\n        data_iter,\n        model,\n        loss_compute,\n        optimizer,\n        scheduler,\n        mode=&quot;train&quot;,\n        accum_iter=1,\n        train_state=TrainState(),\n        device=&quot;cpu&quot;\n):\n    &quot;&quot;&quot;\n    Train a single epoch\n    &quot;&quot;&quot;\n    start = time.time()\n    total_tokens = 0\n    total_loss = 0\n    tokens = 0\n    n_accum = 0\n    for i, (src, src_mask, tgt, tgt_mask, tgt_y, ntokens) in enumerate(data_iter):\n        #\n        src = src.to(device=device)\n        tgt = tgt.to(device=device)\n        tgt_y = tgt_y.to(device=device)\n        # src = src.to(device=device)\n        out = model.forward(src, tgt, src_mask, tgt_mask)\n        loss, loss_node = loss_compute(out, tgt_y, ntokens)\n        # loss_node = loss_node \/ accum_iter\n        if mode == &quot;train&quot; or mode == &quot;train+log&quot;:\n            loss_node.backward()\n            train_state.step += 1\n            train_state.samples += src.shape[0]\n            train_state.tokens += ntokens\n            if i % accum_iter == 0:\n                optimizer.step()\n                optimizer.zero_grad(set_to_none=True)\n                n_accum += 1\n                train_state.accum_step += 1\n            scheduler.step()\n\n        total_loss += loss\n        total_tokens += ntokens\n        tokens += ntokens\n        if i % 40 == 1 and (mode == &quot;train&quot; or mode == &quot;train+log&quot;):\n            lr = optimizer.param_groups[0][&quot;lr&quot;]\n            elapsed = time.time() - start\n            print(\n                (\n                        &quot;Epoch Step: %6d | Accumulation Step: %3d | Loss: %6.2f &quot;\n                        + &quot;| Tokens \/ Sec: %7.1f | Learning Rate: %6.1e&quot;\n                )\n                % (i, n_accum, loss \/ ntokens, tokens \/ elapsed, lr)\n            )\n            start = time.time()\n            tokens = 0\n        del loss\n        del loss_node\n    return total_loss \/ total_tokens, train_state<\/code><\/pre>\n<p>\u4ee3\u7801\u7406\u89e3\uff1a<\/p>\n<ul>\n<li>\u4ece <code>data_iter<\/code> \u4e2d\u5faa\u73af\u83b7\u53d6\u6e90\u6570\u636e\u3001\u76ee\u6807\u6570\u636e\u53ca\u5176\u63a9\u7801\u3002\n<ul>\n<li><code>src<\/code>\uff1a\u8f93\u5165\u4fa7\u6570\u636e\uff0c\u4f8b\u5982\uff1a &quot;Hello, how are you?&quot; \u5bf9\u5e94\u7684\u662f [101, 7592, 2024, 2017, 2011, 102]\u3002<\/li>\n<li><code>src_mask<\/code>\uff1a\u8f93\u5165\u4fa7\u7684\u63a9\u7801\uff0c\u5bf9\u5e94 <code>padding mask<\/code>\u3002<\/li>\n<li><code>tgt<\/code>\uff1a\u8f93\u51fa\u4fa7\u8f93\u5165\u6570\u636e\uff0c\u5373\u901a\u8fc7\u4e0a\u6587\u8f93\u5165\u6570\u636e\u9884\u6d4b\u7684\u4e0b\u6587\u6570\u636e\u3002<\/li>\n<li><code>tgt_mask<\/code>\uff1a\u8f93\u51fa\u4fa7\u7684\u63a9\u7801\uff0c\u5305\u542b <code>padding mask<\/code> \u548c <code>subsequence mask<\/code>\u4e24\u79cd\u4f5c\u7528\u3002<\/li>\n<li><code>tgt_y<\/code>\uff1a\u76ee\u6807\u6807\u7b7e\uff0c\u7528\u4e8e\u8ba1\u7b97\u635f\u5931\u7684\u771f\u5b9e\u8f93\u51fa\u5e8f\u5217\u3002<\/li>\n<li><code>ntokens<\/code>\uff1a\u5f53\u524d\u6279\u6b21\u4e2d\u7684\u6709\u6548token\u6570\u91cf\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"EncoderDecoder%E7%9A%84forward%E5%87%BD%E6%95%B0\"><\/span>EncoderDecoder\u7684forward()\u51fd\u6570<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u8be5\u90e8\u5206\u5bf9\u5e94\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u65f6\u5e8f\u56fe\u7684EncoderDecoder.forward()\u51fd\u6570\uff0c\u7531\u4e8e\u8be5\u51fd\u6570\u8c03\u7528\u6df1\u5ea6\u8f83\u591a\uff0c\u6240\u4ee5\u91c7\u7528\u51fd\u6570\u8c03\u7528\u6808\u5f62\u5f0f\u68b3\u7406\u5176\u8fc7\u7a0b\u3002<\/p>\n<pre><code class=\"language-python\">EncoderDecoder.forward()\n\u2502\n\u251c\u2500\u2500 self.encode()\n\u2502   \u251c\u2500\u2500 embeddings.forward()                            # \u5bf9\u5e94input Embedding\n\u2502   \u251c\u2500\u2500 Positional.forward()                            # \u5bf9\u5e94position Encoding\n\u2502   \u2514\u2500\u2500 \u8fdb\u884c Encoder.layers \u5faa\u73af\u904d\u5386                      # N \u5c42\u5faa\u73af\u904d\u5386\uff0c\u4ee3\u7801\u4e2dN=6\n\u2502       \u2514\u2500\u2500 EncoderLayer.forward()\n\u2502           \u251c\u2500\u2500 self.sublayer[0]                        # \u7b2c\u4e00\u5c42\u591a\u5934\u6ce8\u610f\u529b\u5904\u7406\u90e8\u5206\n\u2502           \u2502   \u251c\u2500\u2500 SublayerConnection.forward()\n\u2502           \u2502   \u2502   \u251c\u2500\u2500 LayerNorm.forward()             # \u5bf9\u5e94Add&amp;Norm\n\u2502           \u2502   \u2514\u2500\u2500 MultiHeadedAttention.forward()  # \u5bf9\u5e94Multi-Head Attention\n\u2502           \u2514\u2500\u2500 self.sublayer[1]                        # \u7b2c\u4e8c\u5c42Feed Forward\u5904\u7406\u90e8\u5206\n\u2502               \u251c\u2500\u2500 SublayerConnection.forward()    \n\u2502               \u2502   \u251c\u2500\u2500 LayerNorm.forward()             # \u5bf9\u5e94Add&amp;Norm\n\u2502               \u2514\u2500\u2500 PositionwiseFeedForward.forward() # \u5bf9\u5e94Feed Forward \n\u2502\n\u251c\u2500\u2500 self.decoder()\n\u2502   \u251c\u2500\u2500 embeddings.forward()                            # \u5bf9\u5e94output Embedding\n\u2502   \u251c\u2500\u2500 Positional.forward()                            # \u5bf9\u5e94position Encoding\n\u2502   \u2514\u2500\u2500 \u8fdb\u884c Decoder.layers \u5faa\u73af\u904d\u5386                      # N \u5c42\u5faa\u73af\u904d\u5386\uff0c\u4ee3\u7801\u4e2dN=6\n\u2502       \u2514\u2500\u2500 DecoderLayer.forward()\n\u2502           \u251c\u2500\u2500 self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n\u2502           \u2502   \u251c\u2500\u2500 SublayerConnection.forward()\n\u2502           \u2502   \u2502   \u251c\u2500\u2500 LayerNorm.forward()             \n\u2502           \u2502   \u2514\u2500\u2500 MultiHeadedAttention.forward()  # \u5bf9\u5e94\u591a\u5934\u63a9\u7801\u5904\u7406\u90e8\u5206\n\u2502           \u251c\u2500\u2500 self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n\u2502           \u2502   \u251c\u2500\u2500 SublayerConnection.forward()\n\u2502           \u2502   \u2502   \u251c\u2500\u2500 LayerNorm.forward()\n\u2502           \u2502   \u2514\u2500\u2500 MultiHeadedAttention.forward()  # \u878d\u5408\u6ce8\u610f\u529b\u5904\u7406\u90e8\u5206\n\u2502           \u2514\u2500\u2500 self.sublayer[2](x, self.feed_forward)\n\u2502               \u251c\u2500\u2500 SublayerConnection.forward()\n\u2502               \u2502   \u251c\u2500\u2500 LayerNorm.forward()\n\u2502               \u2514\u2500\u2500 PositionwiseFeedForward.forward() # \u5bf9\u5e94Feed Forward\n<\/code><\/pre>\n<ul>\n<li>\u5bf9\u7167<a href=\"https:\/\/17aitech.com\/?p=9911#toc-11\">\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day19\uff08\u4e2d\uff09\uff1aTransformer\u67b6\u6784\u53ca\u6ce8\u610f\u529b\u673a\u5236\u4e86\u89e3<\/a>\u4e2d\u7684\u5b8f\u89c2\u6d41\u7a0b\uff0c\u4e0a\u8ff0\u4ee3\u7801\u8c03\u7528\u6808\u4e0e\u67b6\u6784\u56fe\u4e00\u4e00\u5bf9\u5e94\u3002<\/li>\n<li>\u7b2c\u4e00\u6b65\uff1a\u8f93\u5165\u4fa7input\u6570\u636e\u9996\u5148\u7ecf\u8fc7<code>embeddings.forward()<\/code> \u548c <code>Positional.forward()<\/code>\u5904\u7406\u3002<\/li>\n<li>\u7b2c\u4e8c\u6b65\uff1a\u5728Encoder\u4e2d\u901a\u8fc7N\u5c42EncoderLayer\u8fdb\u884c\u904d\u5386\uff0c\u5176\u4e2d<code>self.sublayer[0]<\/code>\u5bf9\u5e94\u8f93\u5165\u7684\u591a\u5934\u6ce8\u610f\u529b\u65b9\u6846\u90e8\u5206\uff0c<code>self.sublayer[1]<\/code>\u5bf9\u5e94\u524d\u9988\u7f51\u7edc\u3002<\/li>\n<li>\u7b2c\u4e09\u6b65\uff1a\u8f93\u51fa\u4fa7output\u7684\u4e0a\u6587\u6570\u636e\u901a\u7528\u7ecf\u8fc7<code>embeddings.forward()<\/code> \u548c <code>Positional.forward()<\/code>\u5904\u7406\u3002<\/li>\n<li>\u7b2c\u56db\u6b65\uff1a\u5728Decoder\u4e2d\u901a\u8fc7N\u5c42DecoderLayer\u8fdb\u884c\u904d\u5386\uff0c\u5176\u4e2d\n<ul>\n<li><code>self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))<\/code> \u5bf9\u5e94\u8f93\u51fa\u7684\u4e0a\u6587\u591a\u5934\u6ce8\u610f\u529b\u65b9\u6846\u90e8\u5206<\/li>\n<li><code>self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))<\/code> \u5bf9\u5e94\u878d\u5408\u6ce8\u610f\u529b\u65b9\u6846\u90e8\u5206<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u6309\u987a\u5e8f\u4f9d\u6b21\u67e5\u770b\u5bf9\u5e94\u51fd\u6570\u7684\u5b9e\u73b0\u8fc7\u7a0b\u3002<\/p>\n<h4><span class=\"ez-toc-section\" id=\"embeddingsforward\"><\/span>embeddings.forward()<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">class Embeddings(nn.Module):\n    # \u521d\u59cb\u5316\u90e8\u5206\u5728\u542f\u52a8\u6d41\u7a0b\u5df2\u4ecb\u7ecd\uff0c\u6b64\u5904\u7701\u7565\n\n    def forward(self, x):\n        return self.lut(x) * math.sqrt(self.d_model)<\/code><\/pre>\n<ul>\n<li>\u4f5c\u7528\uff1aEmbeddings\u6a21\u5757\u7528\u4e8e\u5c06\u8f93\u5165\u7684\u6574\u6570\u5e8f\u5217\u8f6c\u6362\u4e3a\u5bf9\u5e94\u7684\u8bcd\u5411\u91cf\u3002<\/li>\n<li>\u6784\u6210\uff1a\u5728Transformer\u4e2d\uff0cEmbeddings\u6a21\u5757\u7531\u4e24\u4e2a\u7ebf\u6027\u5c42\u7ec4\u6210\uff1a\n<ul>\n<li>\u7b2c\u4e00\u4e2a\u7ebf\u6027\u5c42\u5c06\u6574\u6570\u5e8f\u5217\u8f6c\u6362\u4e3a\u8bcd\u5411\u91cf<\/li>\n<li>\u7b2c\u4e8c\u4e2a\u7ebf\u6027\u5c42\u5c06\u8bcd\u5411\u91cf\u8f6c\u6362\u4e3ad_model\u7ef4\u5ea6\u7684\u5411\u91cf\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"Positional_Encoding\"><\/span>Positional Encoding<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u7531\u4e8e\u65f6\u5e8f\u6570\u636e\u8981\u8fdb\u884c\u5e76\u884c\u8ba1\u7b97\u7684\u7684\u95ee\u9898\uff08\u8be6\u7ec6\u80cc\u666f\u4e0d\u518d\u8d58\u8ff0\uff0c\u5177\u4f53\u5185\u5bb9\u53ef\u89c1<a href=\"https:\/\/17aitech.com\/?p=9911#toc-6\">\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day19\uff08\u4e0b\uff09\uff1aTransformer\u67b6\u6784\u53ca\u6ce8\u610f\u529b\u673a\u5236\u4e86\u89e3<\/a>\uff09\uff0c\u6240\u4ee5\u5728Transformer\u67b6\u6784\u4e2d\uff0c\u9700\u8981\u4e3a\u5e8f\u5217\u4e2d\u7684\u6bcf\u4e2a\u8bcd\u6dfb\u52a0\u4e00\u4e2a\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<p>\u6e90\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-python\">class PositionalEncoding(nn.Module):\n    &quot;Implement the PE function.&quot;\n\n    def __init__(self, d_model, dropout, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        # Compute the positional encodings once in log space.\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len).unsqueeze(1)\n        div_term = torch.exp(\n            torch.arange(0, d_model, 2) * -(math.log(10000.0) \/ d_model)\n        )\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0)\n        self.register_buffer(&quot;pe&quot;, pe)\n\n    def forward(self, x):\n        x = x + self.pe[:, : x.size(1)].requires_grad_(False)\n        return self.dropout(x)<\/code><\/pre>\n<p>\u4ee5\u4e0a\u4ee3\u7801\u4e2d\u6700\u4e3a\u91cd\u8981\u7684\u90e8\u5206\u4e3a\uff1a\u8ba1\u7b97\u7f16\u7801\u90e8\u5206<\/p>\n<pre><code class=\"language-python\">pe = torch.zeros(max_len, d_model)\nposition = torch.arange(0, max_len).unsqueeze(1)\ndiv_term = torch.exp(\n    torch.arange(0, d_model, 2) * -(math.log(10000.0) \/ d_model)\n)\npe[:, 0::2] = torch.sin(position * div_term)\npe[:, 1::2] = torch.cos(position * div_term)\npe = pe.unsqueeze(0)\nself.register_buffer(&quot;pe&quot;, pe)<\/code><\/pre>\n<p><strong>\u7b2c\u4e00\u6b65<\/strong>\uff1a\u521b\u5efa\u4f4d\u7f6e\u7f16\u7801\u77e9\u9635<\/p>\n<ul>\n<li><code>pe = torch.zeros(max_len, d_model)<\/code>\uff1a\u521b\u5efa\u4e00\u4e2a\u5f62\u72b6\u4e3a (max_len, d_model) \u7684\u5168\u96f6\u5f20\u91cf\uff0c\u7528\u4e8e\u5b58\u50a8\u4f4d\u7f6e\u7f16\u7801\u3002<\/li>\n<li><code>position = torch.arange(0, max_len).unsqueeze(1)<\/code>\uff1a\u751f\u6210\u4e00\u4e2a\u4ece 0 \u5230 <code>max_len-1<\/code> \u7684\u5f20\u91cf\uff0c\u5e76\u5728\u7b2c\u4e8c\u4e2a\u7ef4\u5ea6\u4e0a\u589e\u52a0\u4e00\u4e2a\u7ef4\u5ea6\uff0c\u4f7f\u5176\u5f62\u72b6\u4e3a <code>(max_len, 1)<\/code>\u3002<\/li>\n<\/ul>\n<p><strong>\u7b2c\u4e8c\u6b65<\/strong>\uff1a\u8ba1\u7b97\u5206\u6bcd\u9879<br \/>\n\u8ba1\u7b97\u516c\u5f0f\uff1a<\/p>\n<pre><code class=\"language-katex\">\\text{div\\_term} = \\exp\\left(\\text{arange}(0, d\\_model, 2) \\times -\\frac{\\log(10000)}{d\\_model}\\right)<\/code><\/pre>\n<ul>\n<li>d_model\uff1a\u6a21\u578b\u7684\u7ef4\u5ea6\uff0c\u8868\u793a\u6bcf\u4e2a\u8bcd\u7684\u5d4c\u5165\u5411\u91cf\u7684\u5927\u5c0f\u3002<\/li>\n<\/ul>\n<p><strong>\u7b2c\u4e09\u6b65<\/strong>\uff1a\u4f7f\u7528\u6b63\u5f26\u548c\u4f59\u5f26\u51fd\u6570<\/p>\n<ul>\n<li><code>pe[:, 0::2] = torch.sin(position * div_term)<\/code>\uff1a\u5bf9\u5076\u6570\u7d22\u5f15\u7684\u7ef4\u5ea6\u4f7f\u7528\u6b63\u5f26\u51fd\u6570\u8ba1\u7b97\u4f4d\u7f6e\u7f16\u7801\u3002<\/li>\n<li><code>pe[:, 1::2] = torch.cos(position * div_term)<\/code>\uff1a\u5bf9\u5947\u6570\u7d22\u5f15\u7684\u7ef4\u5ea6\u4f7f\u7528\u4f59\u5f26\u51fd\u6570\u8ba1\u7b97\u4f4d\u7f6e\u7f16\u7801\u3002<\/li>\n<\/ul>\n<p><strong>\u7b2c\u56db\u6b65<\/strong>\uff1a\u5c06\u4f4d\u7f6e\u7f16\u7801\u6dfb\u52a0\u5230\u8bcd\u5d4c\u5165\u4e2d<\/p>\n<pre><code>def forward(self, x):\n    x = x + self.pe[:, : x.size(1)].requires_grad_(False)\n    return self.dropout(x)\n<\/code><\/pre>\n<ul>\n<li><code>x + self.pe[:, : x.size(1)].requires_grad_(False)<\/code>\uff1a\u5c06\u4f4d\u7f6e\u7f16\u7801\u52a0\u5230\u8f93\u5165 x \u4e0a\uff0c\u63d0\u4f9b\u4f4d\u7f6e\u4fe1\u606f\u3002<\/li>\n<li><code>requires_grad_(False)<\/code> \u786e\u4fdd\u4f4d\u7f6e\u7f16\u7801\u5728\u53cd\u5411\u4f20\u64ad\u4e2d\u4e0d\u4f1a\u88ab\u66f4\u65b0\u3002<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"Encoder\"><\/span>Encoder<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">class Encoder(nn.Module):\n    &quot;Core encoder is a stack of N layers&quot;\n\n    def __init__(self, layer, N):\n        super(Encoder, self).__init__()\n        self.layers = clones(layer, N)\n        self.norm = LayerNorm(layer.size)\n\n    def forward(self, x, mask):\n        &quot;Pass the input (and mask) through each layer in turn.&quot;\n        for layer in self.layers:\n            x = layer(x, mask)\n        return self.norm(x)\n\nclass EncoderLayer(nn.Module):\n    &quot;Encoder is made up of self-attn and feed forward (defined below)&quot;\n\n    def __init__(self, size, self_attn, feed_forward, dropout):\n        super(EncoderLayer, self).__init__()\n        self.self_attn = self_attn\n        self.feed_forward = feed_forward\n        self.sublayer = clones(SublayerConnection(size, dropout), 2)\n        self.size = size\n\n    def forward(self, x, mask):\n        &quot;Follow Figure 1 (left) for connections.&quot;\n        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n        return self.sublayer[1](x, self.feed_forward)\n<\/code><\/pre>\n<p>\u4ee3\u7801\u89e3\u6790\uff1a<\/p>\n<ul>\n<li><code>forward<\/code>\u51fd\u6570\u4e2d\uff1a\n<ul>\n<li>self.sublayer[0]\u4ee3\u8868\u4e24\u4e2a <code>SublayerConnection<\/code> \u5b9e\u4f8b\u7684\u5217\u8868\u7b2c\u4e00\u4e2a\u5b50\u5c42\uff0c\u5373\u81ea\u6ce8\u610f\u529b\u673a\u5236\u7684\u8fde\u63a5\u3002<\/li>\n<li><code>lambda x: self.self_attn(x, x, x, mask)<\/code> \u662f\u4e00\u4e2a\u533f\u540d\u51fd\u6570\uff0c\u5b83\u63a5\u6536\u8f93\u5165 <code>x<\/code> \u5e76\u6267\u884c <code>self.self_attn(x, x, x, mask)<\/code> \u81ea\u6ce8\u610f\u529b\u8ba1\u7b97.<\/li>\n<li><code>self.self_attn(x, x, x, mask)<\/code> \u8868\u793a\u4f7f\u7528\u8f93\u5165 <code>x<\/code> \u4f5c\u4e3a\u67e5\u8be2\uff08Q\uff09\u3001\u952e\uff08K\uff09\u548c\u503c\uff08V\uff09\uff0c\u540c\u65f6\u4f20\u5165 <code>mask<\/code>\u3002<\/li>\n<li>\u7136\u540e\uff0cSublayerConnection \u5c06\u5904\u7406\u8fd9\u4e2a\u8f93\u51fa\uff0c\u901a\u5e38\u5305\u62ec\u6b8b\u5dee\u8fde\u63a5\u548c\u5c42\u5f52\u4e00\u5316\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"attention%E5%87%BD%E6%95%B0\"><\/span>attention\u51fd\u6570<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u5728\u4e86\u89e3\u591a\u5934\u6ce8\u610f\u529bMultiHeadedAttention.forward()\u51fd\u6570\u4e4b\u524d\uff0c\u6211\u4eec\u9996\u5148\u68b3\u7406\u6ce8\u610f\u529b\u673a\u5236attention\u7684\u8ba1\u7b97\u903b\u8f91\u3002<br \/>\n\u6e90\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-python\">def attention(query, key, value, mask=None, dropout=None):\n    &quot;Compute &#039;Scaled Dot Product Attention&#039;&quot;\n    d_k = query.size(-1)\n    scores = torch.matmul(query, key.transpose(-2, -1)) \/ math.sqrt(d_k)\n    if mask is not None:\n        scores = scores.masked_fill(mask == 0, -1e9)\n    p_attn = scores.softmax(dim=-1)\n    if dropout is not None:\n        p_attn = dropout(p_attn)\n    return torch.matmul(p_attn, value), p_attn<\/code><\/pre>\n<ul>\n<li>\u4f5c\u7528\uff1a<code>attention<\/code> \u51fd\u6570\u7528\u4e8e\u8ba1\u7b97\u201c\u7f29\u653e\u70b9\u79ef\u6ce8\u610f\u529b\u201d\uff08Scaled Dot Product Attention\uff09<\/li>\n<li>\u8ba1\u7b97\u8fc7\u7a0b\uff1a(\u535a\u5ba2\u4e2d\u7684\u793a\u610f\u56fe\u5982\u4e0b)<br \/>\n<a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/attension\u7684\u8ba1\u7b97\u8fc7\u7a0b\u793a\u610f\u56fe.png\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/attension\u7684\u8ba1\u7b97\u8fc7\u7a0b\u793a\u610f\u56fe.png\" alt=\"\" \/><\/a><\/li>\n<\/ul>\n<p><strong>\u7b2c\u4e00\u6b65<\/strong>\uff1a\u8ba1\u7b97\u7ef4\u5ea6<\/p>\n<pre><code class=\"language-python\">d_k = query.size(-1)<\/code><\/pre>\n<ul>\n<li>\u4f5c\u7528\uff1a\u83b7\u53d6\u67e5\u8be2\u5411\u91cf\u7684\u6700\u540e\u4e00\u4e2a\u7ef4\u5ea6\u5927\u5c0f d_k\uff0c\u7528\u4e8e\u540e\u7eed\u7684\u7f29\u653e\u3002<\/li>\n<li>\u7406\u89e3\u793a\u4f8b\uff1a<\/li>\n<\/ul>\n<pre><code class=\"language-python\"># \u5047\u8bbequery\u5185\u5bb9\u4e3a\uff1a\nquery = torch.tensor([\n    [1.0, 0.0], \n    [0.0, 1.0]])  \n# shape: (2, 2)\n\nd_k = query.size(-1)  \n# d_k = 2<\/code><\/pre>\n<p><strong>\u7b2c\u4e8c\u6b65<\/strong>\uff1a\u8ba1\u7b97\u6ce8\u610f\u529b\u5f97\u5206<\/p>\n<pre><code class=\"language-python\">scores = torch.matmul(query, key.transpose(-2, -1)) \/ math.sqrt(d_k)<\/code><\/pre>\n<ul>\n<li>\u4f5c\u7528\uff1a\u8ba1\u7b97\u67e5\u8be2\u4e0e\u952e\u7684\u70b9\u79ef\uff0c\u5e76\u8fdb\u884c\u7f29\u653e\u3002<code>key.transpose(-2, -1)<\/code> \u5c06\u952e\u7684\u6700\u540e\u4e24\u4e2a\u7ef4\u5ea6\u4ea4\u6362\uff0c\u4ee5\u4fbf\u8fdb\u884c\u77e9\u9635\u4e58\u6cd5\u3002<\/li>\n<li>\u7406\u89e3\u793a\u4f8b\uff1a<\/li>\n<\/ul>\n<pre><code class=\"language-python\"># \u5047\u8bbequery\u5185\u5bb9\u4e3a\uff1a\nquery = torch.tensor([\n    [1.0, 0.0], \n    [0.0, 1.0]]) \n# \u5047\u8bbekey\u5185\u5bb9\u4e3a\uff1a\nkey = torch.tensor([\n    [1.0, 0.0], \n    [0.0, 1.0], \n    [0.5, 0.5]])  \n# shape: (3, 2)\n\n# \u8f6c\u7f6e\uff1akey.transpose(-2, -1)\u8868\u793a\u8f6c\u7f6e\u5f20\u91cf\uff0c-1\u4ee3\u8868\u6700\u540e\u4e00\u4e2a\u7ef4\u5ea6\uff0c-2\u4ee3\u8868\u5012\u6570\u7b2c\u4e8c\u4e2a\u7ef4\u5ea6\n# \u8f6c\u7f6e\u540e\u7ed3\u679c\uff1a\n# tensor([[1.0, 0.0, 0.5],\n#         [0.0, 1.0, 0.5]])\n\n# \u70b9\u79ef\u8ba1\u7b97\uff1a\nscores = torch.matmul(query, key.transpose(-2, -1)) \/ math.sqrt(d_k)\n# \u70b9\u79ef\u8ba1\u7b97\u7ed3\u679c\uff1a\n# scores = [\n#   [1.0, 0.0, 0.5], \n#   [0.0, 1.0, 0.5]]  \n# shape: (2, 3)<\/code><\/pre>\n<p><strong>\u7b2c\u4e09\u6b65<\/strong>\uff1a\u5e94\u7528\u906e\u853d<\/p>\n<pre><code class=\"language-python\">if mask is not None:\n    scores = scores.masked_fill(mask == 0, -1e9)<\/code><\/pre>\n<ul>\n<li>\u4f5c\u7528\uff1a\u5982\u679c\u63d0\u4f9b\u4e86\u906e\u853d\u77e9\u9635\uff0c\u5c06\u5f97\u5206\u4e2d\u76f8\u5e94\u4f4d\u7f6e\u7684\u503c\u8bbe\u7f6e\u4e3a -1e9\uff0c\u8fd9\u6837\u5728 softmax \u8ba1\u7b97\u65f6\u4f1a\u88ab\u5ffd\u7565\u3002<\/li>\n<li>\u7406\u89e3\u793a\u4f8b\uff1a<\/li>\n<\/ul>\n<pre><code class=\"language-python\"># scores = [\n#   [1.0, 0.0, 0.5], \n#   [0.0, 1.0, 0.5]]  \n\n# \u906e\u853d\u77e9\u9635\nmask = torch.tensor([\n    [1, 1, 0], \n    [1, 1, 1]])\n\nscores = scores.masked_fill(mask == 0, -1e9)\n# \u906e\u853d\u8ba1\u7b97\u540e\u7ed3\u679c\uff1a\n# scores = [\n#   [1.0, 0.0, -1e9], \n#   [0.0, 1.0, 0.5]] \n<\/code><\/pre>\n<p><strong>\u7b2c\u56db\u6b65<\/strong>\uff1a\u8ba1\u7b97\u6ce8\u610f\u529b\u6743\u91cd<\/p>\n<ul>\n<li>\u4f5c\u7528\uff1a\u5bf9\u5f97\u5206\u5e94\u7528 softmax \u51fd\u6570\uff0c\u5f97\u5230\u6ce8\u610f\u529b\u6743\u91cd p_attn<\/li>\n<li>\u7406\u89e3\u793a\u4f8b\uff1a<\/li>\n<\/ul>\n<pre><code class=\"language-python\"># \u4e0a\u4e00\u6b65\u7ed3\u679c\u4e3a\uff1a\n# scores = [\n#   [1.0, 0.0, -1e9], \n#   [0.0, 1.0, 0.5]] \n\np_attn = scores.softmax(dim=-1)\n# \u8ba1\u7b97\u7ed3\u679c\uff1a\n# p_attn = [\n#   [0.7311, 0.2689, 0.0000], \n#   [0.2689, 0.7311, 0.0000]]  \n<\/code><\/pre>\n<p><strong>\u7b2c\u4e94\u6b65<\/strong>\uff1a\u5e94\u7528dropout<\/p>\n<ul>\n<li>\u4f5c\u7528\uff1a\u5982\u679c\u63d0\u4f9b\u4e86 dropout \u5c42\uff0c\u5219\u4f7f\u7528 dropout \u5c42\u8fdb\u884c\u968f\u673a\u4e22\u5f03\u4e00\u90e8\u5206\u795e\u7ecf\u5143\u3002<\/li>\n<li>\u7406\u89e3\u793a\u4f8b\uff1a<\/li>\n<\/ul>\n<pre><code class=\"language-python\"># \u5047\u8bbe dropout \u6982\u7387\u4e3a 0.5\n# p_attn \u53ef\u80fd\u53d8\u4e3a [\n#   [0.0, 0.0, 0.0], \n#   [0.2689, 0.7311, 0.0]]  \n# \u5f62\u72b6\u4ecd\u4e3a (2, 3)<\/code><\/pre>\n<blockquote>\n<p>\u5173\u4e8edropout\u5c42\u7684\u4f5c\u7528\uff0c\u53ef\u4ee5\u67e5\u770b<a href=\"https:\/\/17aitech.com\/?p=2250\">\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011Day10\uff1a\u5377\u79ef\u7f51\u7edc\u7684\u57fa\u672c\u6784\u6210<\/a>\u56de\u987e<\/p>\n<\/blockquote>\n<p><strong>\u7b2c\u516d\u6b65<\/strong>\uff1a\u8ba1\u7b97\u8f93\u51fa<\/p>\n<ul>\n<li>\u4f5c\u7528\uff1a\u5c06\u6ce8\u610f\u529b\u6743\u91cd\u4e0e\u503c\u5411\u91cf\u76f8\u4e58\uff0c\u5f97\u5230\u6700\u7ec8\u7684\u8f93\u51fa\uff0c\u540c\u65f6\u8fd4\u56de\u6ce8\u610f\u529b\u6743\u91cd p_attn\u3002<\/li>\n<li>\u7406\u89e3\u793a\u4f8b\uff1a<\/li>\n<\/ul>\n<pre><code class=\"language-python\"># \u5047\u8bbevalue\u5185\u5bb9\u4e3a\nvalue = torch.tensor([\n    [10.0, 20.0], \n    [30.0, 40.0], \n    [50.0, 60.0]])  \n# shape: (3, 2)\n\n# \u5047\u8bbe\u4e0a\u4e00\u6b65p_attn\u5185\u5bb9\u4e3a\n#   [0.0, 0.0, 0.0], \n#   [0.2689, 0.7311, 0.0]]  \n\noutput = torch.matmul(p_attn, value)\n# \u70b9\u79ef\u8ba1\u7b97\u7ed3\u679c\uff1a\n# output = [\n#   [0.0 * 10 + 0.0 * 30 + 0.0 * 50, \n#    0.0 * 20 + 0.0 * 40 + 0.0 * 60],\n#   [0.2689 * 10 + 0.7311 * 30 + 0.0 * 50, \n#    0.2689 * 20 + 0.7311 * 40 + 0.0 * 60]]\n# output = [[0.0, 0.0], [21.05, 31.05]]<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"MultiHeadedAttentionforward\"><\/span>MultiHeadedAttention.forward()<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">class MultiHeadedAttention(nn.Module):\n    # \u521d\u59cb\u5316\u90e8\u5206\u5728\u542f\u52a8\u6d41\u7a0b\u5df2\u4ecb\u7ecd\uff0c\u6b64\u5904\u7701\u7565\n\n    def forward(self, query, key, value, mask=None):\n        &quot;Implements Figure 2&quot;\n        if mask is not None:\n            # Same mask applied to all h heads.\n            mask = mask.unsqueeze(1)\n        nbatches = query.size(0)\n\n        # 1) Do all the linear projections in batch from d_model =&gt; h x d_k\n        query, key, value = [\n            lin(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n            for lin, x in zip(self.linears, (query, key, value))\n        ]\n\n        # 2) Apply attention on all the projected vectors in batch.\n        x, self.attn = attention(\n            query, key, value, mask=mask, dropout=self.dropout\n        )\n\n        # 3) &quot;Concat&quot; using a view and apply a final linear.\n        x = (\n            x.transpose(1, 2)\n            .contiguous()\n            .view(nbatches, -1, self.h * self.d_k)\n        )\n        del query\n        del key\n        del value\n        return self.linears[-1](x)<\/code><\/pre>\n<p>\u4ee3\u7801\u7406\u89e3\uff1a<br \/>\n<strong>\u7b2c\u4e00\u6b65<\/strong>\uff1a\u63a5\u53d7\u53c2\u6570<\/p>\n<pre><code class=\"language-python\">def forward(self, query, key, value, mask=None):\n    &quot;Implements Figure 2&quot;\n    if mask is not None:\n        # Same mask applied to all h heads.\n        mask = mask.unsqueeze(1)\n    nbatches = query.size(0)\n<\/code><\/pre>\n<ul>\n<li><code>query\u3001key\u3001value<\/code>\uff1a\u5bf9\u5e94\u8f93\u5165\u7684\u67e5\u8be2(Q)\u3001\u952e(K)\u548c\u503c(V)\u5f20\u91cf<\/li>\n<li><code>mask<\/code>\uff1a\u53ef\u9009\u7684\u906e\u853d\u5f20\u91cf\uff0c\u7528\u4e8e\u5728\u8ba1\u7b97\u6ce8\u610f\u529b\u65f6\u5ffd\u7565\u67d0\u4e9b\u4f4d\u7f6e\u3002<\/li>\n<\/ul>\n<p><strong>\u7b2c\u4e8c\u6b65<\/strong>\uff1a\u7ebf\u6027\u53d8\u6362\u548c\u91cd\u5851<\/p>\n<pre><code class=\"language-python\">query, key, value = [\n    lin(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n    for lin, x in zip(self.linears, (query, key, value))\n]<\/code><\/pre>\n<ul>\n<li><code>query<\/code>, <code>key<\/code>, <code>value<\/code>\u63a5\u53d7\u5217\u8868\uff0c\u5206\u522b\u5bf9\u5e94\u8f93\u5165\u7684<code>\u67e5\u8be2(Q)<\/code>\u3001<code>\u952e(K)<\/code> \u548c <code>\u503c(V)<\/code> \u5f20\u91cf\u3002<\/li>\n<li>[&#8230;]\u4e2d\u5185\u5bb9\u4e3a\u63a8\u5bfc\u5f0f\n<ul>\n<li>\u9996\u5148\uff0c\u67e5\u770b<code>for lin, x in zip(self.linears, (query, key, value))<\/code>\uff0c\u5176\u4f5c\u7528\u662f\uff1a\u4f9d\u6b21\u904d\u5386 <code>self.linears<\/code> \u548c <code>(query, key, value)<\/code> \u5217\u8868\uff0c\u5c06\u6bcf\u4e2a\u5143\u7d20\u5206\u522b\u4f20\u5165lin\u51fd\u6570\u4e2d\u3002<\/li>\n<li>\u7136\u540e\uff0c\u901a\u8fc7<code>lin(x)<\/code>\u5bf9\u8f93\u5165\u8fdb\u884c\u7ebf\u6027\u53d8\u6362\uff0c\u5e76\u4f7f\u7528<code>view()<\/code>\u51fd\u6570\u91cd\u5851\u5f20\u91cf\u7684\u5f62\u72b6\uff0c\u91cd\u5851\u7ed3\u679c\u4e3a <code>(nbatches, -1, h, d_k)<\/code><\/li>\n<li>\u7136\u540e\uff0c\u901a\u8fc7<code>transpose(1, 2)<\/code>\u5c06\u7ef4\u5ea6\u8c03\u6574\u4e3a <code>(nbatches, h, seq_length, d_k)<\/code><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p><strong>\u7b2c\u4e09\u6b65<\/strong>\uff1a\u5e94\u7528\u6ce8\u610f\u529b\u673a\u5236<\/p>\n<pre><code class=\"language-python\">x, self.attn = attention(\n    query, key, value, mask=mask, dropout=self.dropout\n)<\/code><\/pre>\n<ul>\n<li>\u5e94\u7528\u4e4b\u524d\u5df2\u7ecf\u5206\u6790\u8fc7\u7684attension\u51fd\u6570\u8fdb\u884c\u6ce8\u610f\u529b\u8ba1\u7b97\u3002<\/li>\n<li>\u8fd4\u56de\u7684 <code>x<\/code> \u662f\u6ce8\u610f\u529b\u7684\u8f93\u51fa\uff0c\u800c <code>self.attn<\/code> \u662f\u6ce8\u610f\u529b\u6743\u91cd\u3002<\/li>\n<\/ul>\n<p><strong>\u7b2c\u56db\u6b65<\/strong>\uff1a\u8fde\u63a5\u548c\u6700\u7ec8\u7ebf\u6027\u53d8\u6362<\/p>\n<pre><code class=\"language-python\">x = (\n    x.transpose(1, 2)\n    .contiguous()\n    .view(nbatches, -1, self.h * self.d_k)\n)\ndel query\ndel key\ndel value\nreturn self.linears[-1](x)\n<\/code><\/pre>\n<ul>\n<li>\u4f7f\u7528 <code>transpose(1, 2)<\/code> \u5c06\u8f93\u51fa\u7684\u7ef4\u5ea6\u8c03\u6574\u4e3a <code>(nbatches, seq_length, h)<\/code>\u3002<\/li>\n<li>\u4f7f\u7528 <code>contiguous()<\/code> \u786e\u4fdd\u6570\u636e\u5728\u5185\u5b58\u4e2d\u662f\u8fde\u7eed\u7684\uff08\u5bf9\u4e8e\u540e\u7eed\u7684 view \u64cd\u4f5c\u662f\u5fc5\u8981\u7684\uff09\u3002<\/li>\n<li>\u91cd\u5851\u4e3a <code>(nbatches, -1, h * d_k)<\/code>\uff0c\u5c06\u6240\u6709\u5934\u7684\u8f93\u51fa\u8fde\u63a5\u5728\u4e00\u8d77\u3002<\/li>\n<li>\u6700\u540e\uff0c\u901a\u8fc7\u6700\u540e\u4e00\u4e2a\u7ebf\u6027\u5c42<code>self.linears<\/code>\u5c06\u8f93\u51fa\u53d8\u6362\u4e3a d_model \u7ef4\u5ea6\u3002<\/li>\n<li>del \u8bed\u53e5\u7528\u4e8e\u5220\u9664\u4e2d\u95f4\u53d8\u91cf\uff0c\u4ee5\u51cf\u5c11\u5185\u5b58\u5360\u7528\u3002<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"Decoder\"><\/span>Decoder<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">class DecoderLayer(nn.Module):\n    &quot;Decoder is made of self-attn, src-attn, and feed forward (defined below)&quot;\n\n    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):\n        super(DecoderLayer, self).__init__()\n        self.size = size\n        self.self_attn = self_attn\n        self.src_attn = src_attn\n        self.feed_forward = feed_forward\n        self.sublayer = clones(SublayerConnection(size, dropout), 3)\n\n    def forward(self, x, memory, src_mask, tgt_mask):\n        &quot;Follow Figure 1 (right) for connections.&quot;\n        m = memory\n        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n        return self.sublayer[2](x, self.feed_forward)\n\nclass Decoder(nn.Module):\n    &quot;Generic N layer decoder with masking.&quot;\n\n    def __init__(self, layer, N):\n        super(Decoder, self).__init__()\n        self.layers = clones(layer, N)\n        self.norm = LayerNorm(layer.size)\n\n    def forward(self, x, memory, src_mask, tgt_mask):\n        for layer in self.layers:\n            x = layer(x, memory, src_mask, tgt_mask)\n        return self.norm(x)<\/code><\/pre>\n<p>\u4ee3\u7801\u7406\u89e3\uff1a<\/p>\n<ul>\n<li><code>Decoder<\/code> \u7c7b\u7684\u524d\u5411\u4f20\u64ad <code>forward<\/code> \u51fd\u6570\u4e2d\uff0c\u4f1a\u901a\u8fc7<code>x = layer(x, memory, src_mask, tgt_mask)<\/code>\u4f20\u9012\u7ed9\u6bcf\u4e2a<code>DecoderLayer<\/code>\u5bf9\u8c61<\/li>\n<li><code>x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask)) <\/code> \u5bf9\u5e94\u63a9\u7801\u591a\u5934\u6ce8\u610f\u529b\u5904\u7406\u90e8\u5206\u3002<\/li>\n<li><code>x = self.sublayer[1](x, lambda x: self.src_attn(x, memory, memory, src_mask))<\/code> \u5bf9\u5e94\u878d\u5408\u6ce8\u610f\u529b\u5904\u7406\u90e8\u5206\u3002<\/li>\n<\/ul>\n<p>\u81f3\u6b64\uff0c\u6574\u4f53\u8bad\u7ec3\u6d41\u7a0b\u5df2\u7ecf\u68b3\u7406\u5b8c\u6210\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E6%95%B0%E6%8D%AE%E5%BD%A2%E7%8A%B6%E6%A2%B3%E7%90%86\"><\/span>\u6570\u636e\u5f62\u72b6\u68b3\u7406<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u7531\u4e8eshape\u5f62\u72b6(\u4fd7\u79f0\u5bf9\u53e3\u578b)\u5728Transformer\u6a21\u578b\u5b9e\u73b0\u4e2d\u6781\u4e3a\u91cd\u8981\uff0c\u540c\u65f6\u4e5f\u4fbf\u4e8e\u6211\u4eec\u7406\u89e3\u6574\u4e2aTransformer\u7684\u5b9e\u73b0\u8fc7\u7a0b\uff0c\u6240\u4ee5\u6211\u4eec\u5bf9\u6574\u4f53\u6d41\u7a0b\u4e2d\u7684\u5173\u952e\u5f62\u72b6\u505a\u4e00\u6b21\u68b3\u7406\u76d8\u70b9\u3002<\/p>\n<h3><span class=\"ez-toc-section\" id=\"1_%E5%88%9D%E5%A7%8B%E8%BE%93%E5%85%A5_src%E3%80%81src_mask%E3%80%81tgt%E3%80%81tgt_mask%E3%80%81tgt_y\"><\/span>1. \u521d\u59cb\u8f93\u5165 <code>src<\/code>\u3001<code>src_mask<\/code>\u3001<code>tgt<\/code>\u3001<code>tgt_mask<\/code>\u3001<code>tgt_y<\/code><span class=\"ez-toc-section-end\"><\/span><\/h3>\n<pre><code class=\"language-python\">for i, (src, src_mask, tgt, tgt_mask, tgt_y, ntokens) in enumerate(data_iter):\n    # src\u7684\u5f62\u72b6\uff1a        [1024, 24]\n    # src_mask\u7684\u5f62\u72b6\uff1a   [1024, 1, 24]\n    # tgt\u7684\u5f62\u72b6\uff1a        [1024, 22]\n    # tgt_mask\u7684\u5f62\u72b6\uff1a   [1024, 22, 22]\n    # tgt_y\u7684\u5f62\u72b6\uff1a      [1024, 22]\n\n    # \u4ee5\u4e0b\u4ee3\u7801\u7701\u7565...<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<br \/>\n<code>src<\/code>\u7684\u5f62\u72b6\uff1a<code>(batch_size,seq_len)<\/code> &#8211; <code>[1024, 24]<\/code><\/p>\n<blockquote>\n<ul>\n<li><strong><code>1024<\/code><\/strong>\uff1a\u4e00\u6b21\u5904\u7406\u7684\u6837\u672c\u6570\u91cf\uff0c\u4f8b\u5982\uff1a1024\u53e5\u8bdd\u3002<\/li>\n<li><strong><code>24<\/code><\/strong>\uff1a\u8f93\u5165\u5e8f\u5217\u7684\u957f\u5ea6\uff0c\u4f8b\u5982\uff1a\u6700\u957f\u7684\u957f\u5ea6\u4e3a24\u4e2a\u8bcd\u3002<\/li>\n<\/ul>\n<\/blockquote>\n<p><code>src_mask<\/code>\u7684\u5f62\u72b6\uff1a<code>(batch_size, 1, seq_len)<\/code> &#8211; <code>[1024, 1, 24]<\/code><\/p>\n<blockquote>\n<ul>\n<li><code>src_mask<\/code> \u662f\u8f93\u5165\u5e8f\u5217\u7684\u63a9\u7801\uff0c\u7528\u4e8e\u6307\u793a\u54ea\u4e9b\u4f4d\u7f6e\u662f\u6709\u6548\u7684\uff0c\u54ea\u4e9b\u662f\u586b\u5145\uff08padding\uff09\u4f4d\u7f6e\u3002<\/li>\n<li>\u5176\u5f62\u72b6\u4e0e<code>src<\/code>\u7c7b\u4f3c\uff0c\u53ea\u4e0d\u8fc7\u4e2d\u95f4\u591a\u4e861\u4e2a\u7ef4\u5ea6\uff0c\u901a\u5e38\u8868\u793a\u5934\u6570\u3002<\/li>\n<\/ul>\n<\/blockquote>\n<p><code>tgt<\/code>\u7684\u5f62\u72b6\uff1a<code>(batch_size,target_seq_len)<\/code> &#8211; <code>[1024, 22]<\/code><\/p>\n<blockquote>\n<ul>\n<li><code>tgt<\/code> \u8868\u793a\u76ee\u6807\u5e8f\u5217\uff08\u8f93\u51fa\u5e8f\u5217\uff09\uff0c\u5373\u6a21\u578b\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u8981\u751f\u6210\u7684\u5e8f\u5217\u3002<\/li>\n<li><strong><code>22<\/code><\/strong>\uff1a\u76ee\u6807\u5e8f\u5217\u7684\u957f\u5ea6\u3002<\/li>\n<\/ul>\n<\/blockquote>\n<p><code>tgt_mask\u7684<\/code>\u5f62\u72b6\uff1a<code>(batch_size, target_seq_len, target_seq_len)<\/code> &#8211; <code>[1024, 22, 22]<\/code><\/p>\n<blockquote>\n<ul>\n<li><code>tgt_mask<\/code> \u662f\u76ee\u6807\u5e8f\u5217\u7684\u63a9\u7801\uff0c\u7528\u4e8e\u63a7\u5236\u81ea\u56de\u5f52\u751f\u6210\u8fc7\u7a0b\u4e2d\u7684\u6ce8\u610f\u529b\u673a\u5236\u3002<\/li>\n<li><code>22, 22<\/code>\u5f62\u6210\u4e00\u4e2a\u4e8c\u7ef4\u63a9\u7801\u77e9\u9635\uff0c\u7528\u4e8e\u6307\u793a\u5728\u8ba1\u7b97\u6ce8\u610f\u529b\u65f6\u54ea\u4e9b\u4f4d\u7f6e\u53ef\u4ee5\u88ab\u5173\u6ce8\uff08\u6709\u6548\u4f4d\u7f6e\uff09\u548c\u54ea\u4e9b\u4f4d\u7f6e\u9700\u8981\u88ab\u906e\u853d\uff08\u65e0\u6548\u4f4d\u7f6e\uff09\u3002<\/li>\n<\/ul>\n<\/blockquote>\n<h3><span class=\"ez-toc-section\" id=\"2_%E7%BB%8F%E8%BF%87%E5%B5%8C%E5%85%A5%E5%B1%82_Embeddings\"><\/span>2. \u7ecf\u8fc7\u5d4c\u5165\u5c42 (<code>Embeddings<\/code>)<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<pre><code class=\"language-python\">    def forward(self, x):\n        return self.lut(x) * math.sqrt(self.d_model)\n        # x\u7684\u5f62\u72b6\uff1a[1024, 24]\n        # lut(x)\u7684\u5f62\u72b6\uff1a[1024, 24, 512]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<br \/>\n<code>lut(x)<\/code> \u7684\u5f62\u72b6\uff1a<code>(batch_size, seq_len, d_model)<\/code> &#8211; <code>[1024, 24, 512]<\/code><\/p>\n<blockquote>\n<ul>\n<li><strong><code>d_model<\/code><\/strong>\uff1a\u5d4c\u5165\u5411\u91cf\u7684\u7ef4\u5ea6\uff0c\u4f8b\u5982\uff1a1024\u53e5\u8bdd\uff0c\u6bcf\u53e5\u8bdd24\u4e2a\u8bcd\uff0c\u6bcf\u4e2a\u8bcd\u6709512\u4e2a\u7279\u5f81\u3002<\/li>\n<\/ul>\n<\/blockquote>\n<h3><span class=\"ez-toc-section\" id=\"3_%E4%BD%8D%E7%BD%AE%E7%BC%96%E7%A0%81_PositionalEncoding\"><\/span>3. \u4f4d\u7f6e\u7f16\u7801 (<code>PositionalEncoding<\/code>)<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<pre><code class=\"language-python\">    def forward(self, x):\n        x = x + self.pe[:, : x.size(1), :].requires_grad_(False)\n        return self.dropout(x)\n        # x\u7684\u5f62\u72b6\uff1a[1024, 24, 512]\n        # pe\u7684\u5f62\u72b6\uff1a[1, 5000, 512]\n        # return\uff1a[1024, 24, 512]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li>pe\u7684\u5f62\u72b6\uff1a<code>(1, 5000, 512)<\/code>\uff0c\u4ee3\u8868\u4f4d\u7f6e\u7f16\u7801\u7684\u5f62\u72b6\uff0c\u5176\u4e2d<code>5000<\/code>\u662f\u4f4d\u7f6e\u7f16\u7801\u7684\u7ef4\u5ea6\uff0c<code>512<\/code>\u662f\u5d4c\u5165\u5411\u91cf\u7684\u7ef4\u5ea6\u3002<\/li>\n<li><code>self.pe[:, : x.size(1), :]<\/code>\uff0c\u4ee3\u8868\u53d6\u51fa\u4f4d\u7f6e\u7f16\u7801\u77e9\u9635\u7684\u524d<code>x.size(1)<\/code>\u4e2a\u4f4d\u7f6e\uff0c\u537324\u4e2a\u4f4d\u7f6e\u3002<\/li>\n<li>\u7ecf\u8fc7\u4e0a\u8ff0\u9884\u7b97\uff0c\u5f62\u72b6\u4ecd\u7136\u4e3a<code>[1024, 24, 512]<\/code>\u3002<\/li>\n<\/ul>\n<h3><span class=\"ez-toc-section\" id=\"4_%E7%BC%96%E7%A0%81%E5%99%A8%E5%B1%82%EF%BC%88Encoder%EF%BC%89\"><\/span>4. \u7f16\u7801\u5668\u5c42\uff08<code>Encoder<\/code>\uff09<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<h4><span class=\"ez-toc-section\" id=\"41_Add_Norm\"><\/span>4.1 Add&amp;Norm<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">    def forward(self, x):\n        mean = x.mean(-1, keepdim=True)\n        std = x.std(-1, keepdim=True)\n        return self.a_2 * (x - mean) \/ (std + self.eps) + self.b_2\n    # x\u7684\u5f62\u72b6\uff1a[1024, 24, 512]\n    # mean\u7684\u5f62\u72b6\uff1a[1024, 24, 1]\n    # std\u7684\u5f62\u72b6\uff1a[1024, 24, 1]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>mean<\/code> \u548c <code>std<\/code>\u5728\u6c42\u5747\u503c\u548c\u6807\u51c6\u5dee\u65f6\uff0c\u662f\u5bf9dim=512\u7684\u7ef4\u5ea6\u6c42\u5747\u503c\u548c\u6807\u51c6\u5dee\u3002<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"41_%E8%87%AA%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%EF%BC%88MultiHeadedAttention%EF%BC%89\"><\/span>4.1 \u81ea\u6ce8\u610f\u529b\u673a\u5236\uff08<code>MultiHeadedAttention<\/code>\uff09<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">    def forward(self, query, key, value, mask=None):\n        # (\u76f8\u5173\u4ee3\u7801\u5df2\u7701\u7565)\n\n        # query,key,value\u7684\u5f62\u72b6\uff1a[1024, 24, 512]\n        query, key, value = [\n            lin(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n            for lin, x in zip(self.linears, (query, key, value))\n        ]\n        # query, key, value\u7684\u5f62\u72b6\uff1a[1024, 8, 24, 64]\n\n        # (\u76f8\u5173\u4ee3\u7801\u5df2\u7701\u7565)<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>query<\/code>\u3001<code>key<\/code>\u3001<code>value<\/code>\u7684\u5f62\u72b6\uff1a<code>[1024, 8, 24, 64]<\/code><\/li>\n<li>\u7ecf\u8fc7\u5206\u5934\u548c\u7ebf\u6027\u8f6c\u6362\uff0c<code>query<\/code>\u3001<code>key<\/code>\u3001<code>value<\/code>\u7684\u5f62\u72b6\u4ece<code>[1024, 24, 512]<\/code>\u53d8\u4e3a<code>[1024, 8, 24, 64]<\/code>\u3002<\/li>\n<li><code>[1024, 8, 24, 64]<\/code>\u8868\u793a1024\u53e5\u8bdd\uff0c8\u4e2a\u5934\uff0c\u6bcf\u4e2a\u5934\u90fd\u662f24\u4e2a\u8bcd\uff0c\u6bcf\u4e2a\u8bcd64\u4e2a\u7279\u5f81\u3002<\/li>\n<\/ul>\n<pre><code class=\"language-python\">def attention(query, key, value, mask=None, dropout=None):\n    d_k = query.size(-1)\n\n    # key\u7684\u5f62\u72b6\uff1a[1024, 8, 24, 64]\n    # mask\u7684\u5f62\u72b6\uff1a[1024, 1, 1, 24]\n\n    scores = torch.matmul(query, key.transpose(-2, -1)) \/ math.sqrt(d_k)\n    # key.transpose(-2, -1) \u7684\u5f62\u72b6\uff1a[1024, 8, 64, 24]\n    # scores\u7684\u5f62\u72b6\uff1a[1024, 8, 24, 24]\n\n    if mask is not None:\n        mask = mask.to(device=device)\n        scores = scores.masked_fill(mask == 0, -1e9)\n    p_attn = scores.softmax(dim=-1)\n    # p_attn\u7684\u5f62\u72b6\uff1a[1024, 8, 24, 24]\n\n    if dropout is not None:\n        p_attn = dropout(p_attn)\n    # value\u7684\u5f62\u72b6\uff1a[1024, 8, 24, 64]\n    return torch.matmul(p_attn, value), p_attn\n    # \u8fd4\u56de\uff1a[1024, 8, 24, 64]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>key.transpose(-2, -1)<\/code>\uff1a\u5c06 <code>key<\/code> \u7684\u6700\u540e\u4e24\u4e2a\u7ef4\u5ea6\u4ea4\u6362\uff0c\u5f62\u72b6\u53d8\u4e3a <code>[1024, 8, 64, 24]<\/code><\/li>\n<li><code>scores<\/code> \u7684\u5f62\u72b6\u662f <code>[1024, 8, 24, 24]<\/code>\uff0c\u8868\u793a\u6bcf\u4e2a query \u4e0e\u6240\u6709 key \u7684\u76f8\u4f3c\u5ea6\u5f97\u5206\u3002<\/li>\n<li><code>p_attn<\/code> \u7684\u5f62\u72b6\u662f <code>[1024, 8, 24, 24]<\/code>\uff0c\u8868\u793a\u6bcf\u4e2a query \u5bf9\u6240\u6709 key \u7684\u6ce8\u610f\u529b\u6743\u91cd\u3002<\/li>\n<li><code>value<\/code> \u7684\u5f62\u72b6\u4e3a <code>[1024, 8, 24, 64]<\/code>\uff0c\u53731024\u53e5\u8bdd\uff0c8\u4e2a\u5934\uff0c\u6bcf\u4e2a\u5934\u90fd\u662f24\u4e2a\u8bcd\uff0c\u6bcf\u4e2a\u8bcd64\u4e2a\u7279\u5f81\u3002<\/li>\n<li>\u8fd4\u56de\u7684\u52a0\u6743\u503c\u548c\u6ce8\u610f\u529b\u6743\u91cd\u7684\u5f62\u72b6\u5206\u522b\u4e3a <code>[1024, 8, 24, 64]<\/code> \u548c <code>[1024, 8, 24, 24]<\/code>\u3002<\/li>\n<\/ul>\n<pre><code class=\"language-python\">    # x\u7684\u5f62\u72b6\uff1a[1024, 8, 24, 64]\n    # x.transpose(1, 2)\u7684\u5f62\u72b6\uff1a [1024, 24, 8, 64]\n    x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)\n    # x\u7684\u5f62\u72b6\uff1a[1024, 24, 512]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>x.transpose(1, 2)<\/code> \u5c06<code>[1024, 8, 24, 64]<\/code> \u8f6c\u7f6e\u4e3a <code>[1024, 24, 8, 64]<\/code><\/li>\n<li><code>x.contiguous().view(nbatches, -1, self.h * self.d_k)<\/code>\u5c06\u591a\u5934\u91cd\u5851\u4e3a <code>[1024, 24, 512]<\/code><\/li>\n<\/ul>\n<h3><span class=\"ez-toc-section\" id=\"5_%E8%A7%A3%E7%A0%81%E5%99%A8%E5%B1%82%EF%BC%88Decoder%EF%BC%89\"><\/span>5. \u89e3\u7801\u5668\u5c42\uff08<code>Decoder<\/code>\uff09<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<pre><code class=\"language-python\">class DecoderLayer(nn.Module):\n    # (\u76f8\u5173\u4ee3\u7801\u5df2\u7701\u7565)\n\n    def forward(self, x, memory, src_mask, tgt_mask):\n        m = memory\n        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n        return self.sublayer[2](x, self.feed_forward)\n    # x\u7684\u5f62\u72b6\uff1a[1024, 22, 512]\n    # memory\u7684\u5f62\u72b6\uff1a[1024, 24, 512]\n    # src_mask\u7684\u5f62\u72b6\uff1a[1024, 1, 24]\n    # tgt\u7684\u5f62\u72b6\uff1a[1024, 22]\n    # tgt_mask\u7684\u5f62\u72b6\uff1a[1024, 22, 22]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>x<\/code> \u5bf9\u5e94\u7684\u662f\u8f93\u51fa\u4fa7\u7684\u4e0a\u6587\uff0c\u5373<code>self.tgt_embed(tgt)<\/code>\uff0c\u4e5f\u5c31\u662f\u5728\u521d\u59cb\u5316\u65f6<code>tgt<\/code>\u7ecf\u8fc7<code>embed<\/code>\uff0c\u7531\u5f62\u72b6<code>[1024, 22]<\/code> \u53d8\u4e3a<code>[1024, 22, 512]<\/code><\/li>\n<li><code>x<\/code> \u7684\u5f62\u72b6\u662f <code>[1024, 22, 512]<\/code>\u3002<\/li>\n<li><code>memory<\/code> \u5bf9\u5e94\u7684\u662f\u7f16\u7801\u5668\u7684\u8f93\u51fa\uff0c\u5373<code>self.encode(src, src_mask)<\/code>\uff0c\u6240\u4ee5\u5176\u5f62\u72b6\u4e3a<code>[1024, 24, 512]<\/code>\u3002<\/li>\n<li><code>src_mask<\/code> \u5bf9\u5e94\u7684\u662f\u7f16\u7801\u5668\u7684\u63a9\u7801\uff0c\u57281.\u521d\u59cb\u5316\u4e2d\u4f20\u5165\u7684\u5f62\u72b6\u4e3a<code>[1024, 1, 24]<\/code>\u3002<\/li>\n<li><code>tgt_mask<\/code> \u5bf9\u5e94\u7684\u662f\u89e3\u7801\u5668\u7684\u63a9\u7801\uff0c\u662f\u4f5c\u7528\u4e8e<code>x<\/code>\u7684\uff0c\u6240\u4ee5\u5176\u5f62\u72b6\u4e3a<code>[1024, 22, 22]<\/code>\u3002<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"51_%E7%9B%AE%E6%A0%87%E5%B5%8C%E5%85%A5Embeddings\"><\/span>5.1 \u76ee\u6807\u5d4c\u5165(<code>Embeddings<\/code>)<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">    def forward(self, x):\n        return self.lut(x) * math.sqrt(self.d_model)\n        # x\u7684\u5f62\u72b6\uff1a[1024, 22]\n        # lut(x)\u7684\u5f62\u72b6\uff1a[1024, 22, 512]<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"52_%E4%BD%8D%E7%BD%AE%E7%BC%96%E7%A0%81PositionalEncoding\"><\/span>5.2 \u4f4d\u7f6e\u7f16\u7801(<code>PositionalEncoding<\/code>)<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">    def forward(self, x):\n        x = x + self.pe[:, : x.size(1), :].requires_grad_(False)\n        return self.dropout(x)\n        # x\u7684\u5f62\u72b6\uff1a[1024, 22, 512]\n        # pe\u7684\u5f62\u72b6\uff1a[1, 5000, 512]\n        # return\uff1a[1024, 22, 512]<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"53_%E8%87%AA%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%EF%BC%88Masked_MultiHeadedAttention%EF%BC%89\"><\/span>5.3 \u81ea\u6ce8\u610f\u529b\u673a\u5236\uff08<code>Masked MultiHeadedAttention<\/code>\uff09<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">    x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>tgt_mask<\/code> \u5bf9\u5e94\u7684\u662f\u8f93\u51fa\u4fa7\u4e0a\u6587\u7684mask\uff0c\u5f62\u72b6\u662f <code>[1024, 22, 22]<\/code>\uff0c\u5bf9\u5e94<code>mask<\/code>\u7c7b\u578b\u4e3a<code>subsquence mask<\/code> \u548c <code>padding mask<\/code>\u3002<\/li>\n<li>\u5907\u6ce8\uff1a\u56e0\u4e3a\u8f93\u51fa\u4fa7\u662f\u4e00\u4e2a\u4e00\u4e2a\u8bcd\u81ea\u56de\u5f52\u751f\u6210\u7684\uff0c\u6240\u4ee5\u8fd8\u6ca1\u6709\u751f\u6210\u7684\u8bcd(\u5373\u672a\u6765\u8bcd)\u8981\u901a\u8fc7subsquence mask\u8fdb\u884c\u63a9\u7801\u3002<\/li>\n<\/ul>\n<blockquote>\n<p>mask\u7684\u7c7b\u578b\u4ee5\u53ca\u4f5c\u7528\uff0c\u53ef\u4ee5\u67e5\u770b<a href=\"https:\/\/17aitech.com\/?p=9911#toc-19\">\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day19\uff08\u4e2d\uff09\uff1aTransformer\u67b6\u6784\u53ca\u6ce8\u610f\u529b\u673a\u5236\u4e86\u89e3<\/a>\u8fdb\u884c\u56de\u987e\u3002<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"54_%E8%9E%8D%E5%90%88%E6%B3%A8%E6%84%8F%E5%8A%9B%E6%9C%BA%E5%88%B6%EF%BC%88MultiHeadedAttention%EF%BC%89\"><\/span>5.4 \u878d\u5408\u6ce8\u610f\u529b\u673a\u5236\uff08<code>MultiHeadedAttention<\/code>\uff09<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-python\">    x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n    # x\u7684\u5f62\u72b6\uff1a[1024, 22, 512]\n    # m\u7684\u5f62\u72b6\uff1a[1024, 24, 512]\n    # src_mask\u7684\u5f62\u72b6\uff1a[1024, 1, 24]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>x<\/code> \u5bf9\u5e94\u7ecf\u8fc7\u81ea\u6ce8\u610f\u529b\u5904\u7406\u7684\u8f93\u51fa\u4fa7\u4e0a\u6587\u5185\u5bb9\uff0c\u5176\u542b\u4e49\u8868\u793a\u662fQ(\u67e5\u8be2\u90e8\u5206)\uff0c\u5f62\u72b6\u4e3a <code>[1024, 22, 512]<\/code><\/li>\n<li><code>m<\/code> \u5bf9\u5e94\u8f93\u5165\u4fa7\u4e0a\u6587\u5185\u5bb9\uff0c\u5176\u542b\u4e49\u8868\u793a\u662fK(\u952e\u90e8\u5206)\u548cV(\u503c\u90e8\u5206)\uff0c\u5f62\u72b6\u4e3a <code>[1024, 24, 512]<\/code><br \/>\n<blockquote>\n<p>\u5907\u6ce8\uff1a\u5728\u6ce8\u610f\u529b\u8ba1\u7b97\u4e2d\uff0c<code>Q<\/code>\u3001<code>K<\/code> \u548c <code>V<\/code> \u7684\u957f\u5ea6\u4e0d\u540c\u65f6\uff0c<strong>\u53ef\u4ee5<\/strong>\u8fdb\u884c\u6ce8\u610f\u529b\u8ba1\u7b97\u3002<br \/>\n\u4f8b\u5982\uff1a<\/p>\n<ol>\n<li><code>Q<\/code> \u7684\u957f\u5ea6\u4e3a 22 \u4e2a\u8bcd\uff0c<code>K<\/code> \u548c <code>V<\/code> \u7684\u957f\u5ea6\u4e3a 24 \u4e2a\u8bcd\u3002<\/li>\n<li><code>Q<\/code> \u4e0e <code>K<\/code> \u8fdb\u884c\u70b9\u79ef\u8ba1\u7b97\u3001\u63a9\u7801\u53casoftmax\u540e\uff0c\u5f97\u5230\u5f62\u72b6\u4e3a [22, 24] \u7684\u6ce8\u610f\u529b\u6743\u91cd\u77e9\u9635\u3002<\/li>\n<li>\u6700\u540e\uff0c\u4f7f\u7528\u4e0a\u8ff0\u6743\u91cd\u5bf9 <code>V<\/code> \u8fdb\u884c<strong>\u52a0\u6743\u6c42\u548c<\/strong>\uff0c\u5f97\u5230\u4e0e <code>Q<\/code> \u7684\u957f\u5ea6\u76f8\u540c\u7684\u7ed3\u679c\uff0c\u5373 22\u3002<\/li>\n<\/ol>\n<\/blockquote>\n<\/li>\n<\/ul>\n<h3><span class=\"ez-toc-section\" id=\"6_%E8%AE%A1%E7%AE%97%E6%8D%9F%E5%A4%B1\"><\/span>6. \u8ba1\u7b97\u635f\u5931<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<pre><code class=\"language-python\">    loss, loss_node = loss_compute(out, tgt_y, ntokens)\n    # out\u7684\u5f62\u72b6\uff1a[1024, 22, 512]\n    # tgt_y\u7684\u5f62\u72b6\uff1a[1024, 22]<\/code><\/pre>\n<pre><code class=\"language-python\">class SimpleLossCompute(object):\n    # \u6b64\u5904\u4ee3\u7801\u5df2\u7701\u7565\n\n    def __call__(self, x, y, norm):\n        x = self.generator(x)\n        sloss = (\n                self.criterion(\n                    x.contiguous().view(-1, x.size(-1)), y.contiguous().view(-1))\n                \/ norm\n        )\n        return sloss.data * norm, sloss\n    # x\u7684\u5f62\u72b6\uff1a[1024, 22, 512]\n    # self.generator\u540e\u7684\u5f62\u72b6\uff1a[1024, 22, 12547]\n    # x.size(-1)\u8868\u793a\u53d6x\u5f20\u91cf\u7684\u6700\u540e\u4e00\u7ef4\u7684\u5927\u5c0f\uff0c\u5bf9\u5e9412547\n    # \u539f\u59cb\u5f20\u91cfx\u7684\u603b\u5143\u7d20\u4e2a\u6570\u662f1024\u00d722\u00d712547\uff0c\u65b0\u5f20\u91cf\u7684\u5f62\u72b6\u662f[a, 12547]\n    # a=(1024\u00d722\u00d712547)\/12547 = 1024\u00d722 = 22528\n    # x.contiguous().view(-1, x.size(-1))\u7684\u5f62\u72b6\uff1a[22528, 12547]\n    # y.contiguous().view(-1)\u7684\u5f62\u72b6\uff1a[22528]<\/code><\/pre>\n<p>\u8bf4\u660e\uff1a<\/p>\n<ul>\n<li><code>self.generator<\/code> \u7684\u5f62\u72b6\u662f <code>[batch_size, seq_len, vocab_size]<\/code> &#8211; <code>[1024, 22, 12547]<\/code>\u3002\n<ul>\n<li><code>1024<\/code>\uff1a\u6279\u91cf\u5927\u5c0f\uff0c\u53731024\u53e5\u8bdd\u3002<\/li>\n<li><code>22<\/code>\uff1a\u5e8f\u5217\u957f\u5ea6\uff0c\u537322\u4e2a\u8bcd\u3002<\/li>\n<li><code>12547<\/code>\uff1a\u8bcd\u8868\u5927\u5c0f\uff0c\u537312547\u4e2a\u8bcd\u3002<\/li>\n<\/ul>\n<\/li>\n<li><code>x.contiguous().view(-1, x.size(-1))<\/code> \u5c06\u8f93\u51fa\u5f20\u91cf x \u91cd\u65b0\u5f62\u72b6\u4e3a <code>[batch_size * seq_len, vocab_size]<\/code> \u4ee5\u4fbf\u8ba1\u7b97\u635f\u5931\u3002<\/li>\n<li><code>y.contiguous().view(-1)<\/code> \u5c06\u771f\u5b9e\u6807\u7b7e <code>y<\/code> \u91cd\u65b0\u5f62\u72b6\u4e3a\u4e00\u7ef4\u5f20\u91cf <code>[batch_size * seq_len]<\/code><\/li>\n<li>\u6700\u540e\uff0c\u4f7f\u7528\u635f\u5931\u51fd\u6570 <code>criterion<\/code> \u8ba1\u7b97\u635f\u5931\uff0c\u5e76\u9664\u4ee5 <code>norm<\/code> \u8fdb\u884c\u5f52\u4e00\u5316\u3002<\/li>\n<\/ul>\n<h2><span class=\"ez-toc-section\" id=\"%E5%86%85%E5%AE%B9%E5%B0%8F%E7%BB%93\"><\/span>\u5185\u5bb9\u5c0f\u7ed3<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<ul>\n<li>\u8bad\u7ec3\u8fc7\u7a0b\u7684\u4ee3\u7801\u4e3b\u8981\u7531\u56db\u90e8\u5206\u5185\u5bb9\u7ec4\u6210\uff1a\n<ul>\n<li>\u7b2c\u4e00\u90e8\u5206\uff1a<strong>\u52a0\u8f7d\u6570\u636e\u96c6<\/strong>\u3002\u901a\u8fc7 <code>get_dataloader()<\/code> \u52a0\u8f7d\u6570\u636e\u96c6\uff0c\u8fd9\u4e00\u8fc7\u7a0b\u4e0eSeq2Seq\u7c7b\u4f3c\u3002<\/li>\n<li>\u7b2c\u4e8c\u90e8\u5206\uff1a<strong>\u6570\u636e\u5bf9\u9f50<\/strong>\u3002\u8c03\u7528collate_fn()\u51fd\u6570\uff0c\u5bf9\u6570\u636e\u96c6\u6574\u7406\u5e76\u5bf9\u9f50\u3002<\/li>\n<li>\u7b2c\u4e09\u90e8\u5206\uff1a<strong>\u5f00\u59cb\u8bad\u7ec3<\/strong>\u3002\u8fd9\u4e00\u90e8\u5206\u901a\u8fc7 <code>run_epoch()<\/code> \u51fd\u6570\uff0c\u5faa\u73af\u904d\u5386dataloader\u4e2d\u7684\u6570\u636e\u96c6\u5e76\u8fdb\u884c\u8bad\u7ec3\u3002<\/li>\n<li>\u7b2c\u56db\u90e8\u5206\uff1a<strong>\u524d\u5411\u4f20\u64ad<\/strong>\u3002\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\uff0c\u8c03\u7528EncoderDecoder\u7684forward()\u51fd\u6570\u8fdb\u884c\u524d\u5411\u4f20\u64ad\u3002<\/li>\n<\/ul>\n<\/li>\n<li>\u6570\u636e\u5bf9\u9f50\u7684\u8fc7\u7a0b\u4e2d\u6838\u5fc3\u7684\u903b\u8f91\u6709\u4e09\u4e2a\uff1a\n<ul>\n<li>\u901a\u8fc7\u83b7\u53d6\u6279\u91cf\u7684\u8f93\u5165\u6570\u636e\u7684\u6700\u5927\u957f\u5ea6\uff0c\u5bf9\u6bcf\u4e00\u6761\u6570\u636e\u8865\u9f50PAD\u3002<\/li>\n<li>\u901a\u8fc7\u5bf9\u6bd4\u662f\u5426\u4e3aPAD\u8ba1\u7b97\u5f97\u5230input_mask\uff0c\u7528\u4e8e\u5bf9PAD\u8fdb\u884c\u63a9\u7801\u3002<\/li>\n<li>output\u56e0\u4e3a\u5728\u81ea\u56de\u5f52\u7684\u65f6\u5019\u9700\u8981\u9519\u4f4d\uff0c\u6240\u4ee5out_idxes_in\u4f1a\u53bb\u6389\u6700\u540e\u4e00\u4f4d\uff0cout_idxex_out\u4f1a\u53bb\u6389\u5f00\u5934\u7684SOS<\/li>\n<li>decoder\u7684\u591a\u5934\u6ce8\u610f\u529b\u9700\u8981padding mask\u548csubsequent mask\uff0c\u6240\u4ee5\u5176\u662f\u901a\u8fc7<code>tgt_mask &amp; Tokenizer.subsequent_mask<\/code>\u6309\u4f4d\u4e0e\u8fd0\u7b97\u5b9e\u73b0\u3002<\/li>\n<\/ul>\n<\/li>\n<li>\u591a\u5934\u6ce8\u610f\u529b\u8ba1\u7b97\u8fc7\u7a0b\u5927\u81f4\u7531\u4ee5\u4e0b\u51e0\u6b65\u6784\u6210\uff1a\n<ul>\n<li>\u7b2c\u4e00\u6b65\uff1a\u63a5\u53d7\u53c2\u6570\uff1a\u63a5\u53d7\u8f93\u5165\u7684\u67e5\u8be2(Q)\u3001\u952e(K)\u548c\u503c(V)\u5f20\u91cf<\/li>\n<li>\u7b2c\u4e8c\u6b65\uff1a\u7ebf\u6027\u53d8\u6362\uff1a\u901a\u8fc7\u4e09\u4e2a\u7ebf\u6027\u53d8\u6362\uff0c\u5206\u522b\u5c06\u67e5\u8be2\u3001\u952e\u548c\u503c\u6620\u5c04\u5230\u4e0d\u540c\u7684\u7ef4\u5ea6\u3002<\/li>\n<li>\u7b2c\u4e09\u6b65\uff1a\u5e94\u7528\u6ce8\u610f\u529battension\u51fd\u6570\uff0c\u8be5\u51fd\u6570\u5305\u62ec\u4ee5\u4e0b\u6b65\u9aa4\uff1a<\/li>\n<li><strong>\u8ba1\u7b97\u7ef4\u5ea6<\/strong>\uff1a\u83b7\u53d6\u67e5\u8be2\u5411\u91cf\u7684\u6700\u540e\u4e00\u4e2a\u7ef4\u5ea6\u5927\u5c0f d_k\uff0c\u7528\u4e8e\u540e\u7eed\u7684\u7f29\u653e\u3002<\/li>\n<li><strong>\u8ba1\u7b97\u6ce8\u610f\u529b\u5f97\u5206<\/strong>\uff1a\u8ba1\u7b97\u67e5\u8be2\u5411\u91cf\u4e0e\u952e\u5411\u91cf\u7684\u70b9\u79ef\uff0c\u5e76\u8fdb\u884c\u7f29\u653e\u3002<\/li>\n<li><strong>\u5e94\u7528\u906e\u853d<\/strong>\uff1a\u5982\u679c\u63d0\u4f9b\u4e86\u906e\u853d\u77e9\u9635\uff0c\u5c06\u5f97\u5206\u4e2d\u76f8\u5e94\u4f4d\u7f6e\u7684\u503c\u8bbe\u7f6e\u4e3a -1e9\uff0c\u8fd9\u6837\u5728 softmax \u8ba1\u7b97\u65f6\u4f1a\u88ab\u5ffd\u7565\u3002<\/li>\n<li><strong>\u8ba1\u7b97\u6ce8\u610f\u529b\u6743\u91cd<\/strong>\uff1a\u5bf9\u5f97\u5206\u5e94\u7528 softmax \u51fd\u6570\uff0c\u5f97\u5230\u6ce8\u610f\u529b\u6743\u91cd p_attn\u3002<\/li>\n<li><strong>\u8ba1\u7b97\u8f93\u51fa<\/strong>\uff1a\u5c06\u6ce8\u610f\u529b\u6743\u91cd\u4e0e\u503c\u5411\u91cf\u76f8\u4e58\uff0c\u5f97\u5230\u6700\u7ec8\u7684\u8f93\u51fa\u3002<\/li>\n<li>\u7b2c\u56db\u6b65\uff1a\u8fde\u63a5\u548c\u6700\u7ec8\u7ebf\u6027\u53d8\u6362\uff1a\u5c06\u6ce8\u610f\u529b\u7684\u8f93\u51fa\u8fdb\u884c\u8fde\u63a5\uff0c\u5e76\u4f7f\u7528\u4e00\u4e2a\u7ebf\u6027\u5c42\u8fdb\u884c\u6700\u7ec8\u7684\u7ebf\u6027\u53d8\u6362\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<h2><span class=\"ez-toc-section\" id=\"%E5%8F%82%E8%80%83%E8%B5%84%E6%96%99\"><\/span>\u53c2\u8003\u8d44\u6599<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p><a href=\"https:\/\/nlp.seas.harvard.edu\/2018\/04\/03\/attention.html\">The Annotated Transformer<\/a><\/p>\n<p align=\"center\">\u6b22\u8fce\u5173\u6ce8\u516c\u4f17\u53f7\u4ee5\u83b7\u5f97\u6700\u65b0\u7684\u6587\u7ae0\u548c\u65b0\u95fb<\/p>\n<p><a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/09\/\u626b\u7801_\u641c\u7d22\u8054\u5408\u4f20\u64ad\u6837\u5f0f-\u767d\u8272\u7248.bmp\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/09\/\u626b\u7801_\u641c\u7d22\u8054\u5408\u4f20\u64ad\u6837\u5f0f-\u767d\u8272\u7248.bmp\" alt=\"\" \/><\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00 \u5728\u4e0a\u4e00\u7ae0\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day19\uff08\u4e0b [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":10011,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"aside","meta":{"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"default","adv-header-id-meta":"","stick-header-meta":"default","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"set","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":""},"categories":[28],"tags":[68],"class_list":["post-11860","post","type-post","status-publish","format-aside","has-post-thumbnail","hentry","category-blog","tag-68","post_format-post-format-aside"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v27.9 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b - \u4e00\u8d77AI\u6280\u672f<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/17aitech.com\/?p=11860\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860\"},\"author\":{\"name\":\"Dongming\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/#\\\/schema\\\/person\\\/3d23bb6f7f115fcefc9ae7803a691739\"},\"headline\":\"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b\",\"datePublished\":\"2024-08-11T12:19:58+00:00\",\"dateModified\":\"2024-12-04T09:58:32+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860\"},\"wordCount\":185,\"commentCount\":0,\"image\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2024\\\/08\\\/\u5c01\u9762_transformer.png\",\"keywords\":[\"\u5927\u6a21\u578b\"],\"articleSection\":[\"\u535a\u5ba2\u6587\u7ae0\"],\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"CommentAction\",\"name\":\"Comment\",\"target\":[\"https:\\\/\\\/17aitech.com\\\/?p=11860#respond\"]}]},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860\",\"url\":\"https:\\\/\\\/17aitech.com\\\/?p=11860\",\"name\":\"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b - \u4e00\u8d77AI\u6280\u672f\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2024\\\/08\\\/\u5c01\u9762_transformer.png\",\"datePublished\":\"2024-08-11T12:19:58+00:00\",\"dateModified\":\"2024-12-04T09:58:32+00:00\",\"author\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/#\\\/schema\\\/person\\\/3d23bb6f7f115fcefc9ae7803a691739\"},\"breadcrumb\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/17aitech.com\\\/?p=11860\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#primaryimage\",\"url\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2024\\\/08\\\/\u5c01\u9762_transformer.png\",\"contentUrl\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2024\\\/08\\\/\u5c01\u9762_transformer.png\",\"width\":1080,\"height\":608},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=11860#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\\\/\\\/17aitech.com\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/#website\",\"url\":\"https:\\\/\\\/17aitech.com\\\/\",\"name\":\"\u4e00\u8d77AI\u6280\u672f\",\"description\":\"\u8ba9AI\u77e5\u8bc6\u89e6\u624b\u53ef\u53ca\",\"alternateName\":\"\u4e00\u8d77AI\u6280\u672f\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/17aitech.com\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/#\\\/schema\\\/person\\\/3d23bb6f7f115fcefc9ae7803a691739\",\"name\":\"Dongming\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/member\\\/avatars\\\/238a0b923820dcc5.1732798681.jpg\",\"url\":\"\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/member\\\/avatars\\\/238a0b923820dcc5.1732798681.jpg\",\"contentUrl\":\"\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/member\\\/avatars\\\/238a0b923820dcc5.1732798681.jpg\",\"caption\":\"Dongming\"},\"description\":\"\u89c1\u5929\u5730\uff0c\u89c1\u4f17\u751f\uff0c\u89c1\u81ea\u5df1\u3002\",\"sameAs\":[\"http:\\\/\\\/17aitech.com\"],\"url\":\"https:\\\/\\\/17aitech.com\\\/?page_id=33738&user=1\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b - \u4e00\u8d77AI\u6280\u672f","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/17aitech.com\/?p=11860","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/17aitech.com\/?p=11860#article","isPartOf":{"@id":"https:\/\/17aitech.com\/?p=11860"},"author":{"name":"Dongming","@id":"https:\/\/17aitech.com\/#\/schema\/person\/3d23bb6f7f115fcefc9ae7803a691739"},"headline":"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b","datePublished":"2024-08-11T12:19:58+00:00","dateModified":"2024-12-04T09:58:32+00:00","mainEntityOfPage":{"@id":"https:\/\/17aitech.com\/?p=11860"},"wordCount":185,"commentCount":0,"image":{"@id":"https:\/\/17aitech.com\/?p=11860#primaryimage"},"thumbnailUrl":"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/\u5c01\u9762_transformer.png","keywords":["\u5927\u6a21\u578b"],"articleSection":["\u535a\u5ba2\u6587\u7ae0"],"inLanguage":"zh-Hans","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/17aitech.com\/?p=11860#respond"]}]},{"@type":"WebPage","@id":"https:\/\/17aitech.com\/?p=11860","url":"https:\/\/17aitech.com\/?p=11860","name":"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b - \u4e00\u8d77AI\u6280\u672f","isPartOf":{"@id":"https:\/\/17aitech.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/17aitech.com\/?p=11860#primaryimage"},"image":{"@id":"https:\/\/17aitech.com\/?p=11860#primaryimage"},"thumbnailUrl":"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/\u5c01\u9762_transformer.png","datePublished":"2024-08-11T12:19:58+00:00","dateModified":"2024-12-04T09:58:32+00:00","author":{"@id":"https:\/\/17aitech.com\/#\/schema\/person\/3d23bb6f7f115fcefc9ae7803a691739"},"breadcrumb":{"@id":"https:\/\/17aitech.com\/?p=11860#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/17aitech.com\/?p=11860"]}]},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/17aitech.com\/?p=11860#primaryimage","url":"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/\u5c01\u9762_transformer.png","contentUrl":"https:\/\/17aitech.com\/wp-content\/uploads\/2024\/08\/\u5c01\u9762_transformer.png","width":1080,"height":608},{"@type":"BreadcrumbList","@id":"https:\/\/17aitech.com\/?p=11860#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/17aitech.com\/"},{"@type":"ListItem","position":2,"name":"\u3010\u8bfe\u7a0b\u603b\u7ed3\u3011day20\uff1aTransformer\u6e90\u7801\u6df1\u5165\u7406\u89e3\u4e4b\u8bad\u7ec3\u8fc7\u7a0b"}]},{"@type":"WebSite","@id":"https:\/\/17aitech.com\/#website","url":"https:\/\/17aitech.com\/","name":"\u4e00\u8d77AI\u6280\u672f","description":"\u8ba9AI\u77e5\u8bc6\u89e6\u624b\u53ef\u53ca","alternateName":"\u4e00\u8d77AI\u6280\u672f","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/17aitech.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/17aitech.com\/#\/schema\/person\/3d23bb6f7f115fcefc9ae7803a691739","name":"Dongming","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"\/\/17aitech.com\/wp-content\/uploads\/member\/avatars\/238a0b923820dcc5.1732798681.jpg","url":"\/\/17aitech.com\/wp-content\/uploads\/member\/avatars\/238a0b923820dcc5.1732798681.jpg","contentUrl":"\/\/17aitech.com\/wp-content\/uploads\/member\/avatars\/238a0b923820dcc5.1732798681.jpg","caption":"Dongming"},"description":"\u89c1\u5929\u5730\uff0c\u89c1\u4f17\u751f\uff0c\u89c1\u81ea\u5df1\u3002","sameAs":["http:\/\/17aitech.com"],"url":"https:\/\/17aitech.com\/?page_id=33738&user=1"}]}},"_links":{"self":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts\/11860","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=11860"}],"version-history":[{"count":15,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts\/11860\/revisions"}],"predecessor-version":[{"id":34331,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts\/11860\/revisions\/34331"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/media\/10011"}],"wp:attachment":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=11860"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=11860"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=11860"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}