{"id":37835,"date":"2025-02-05T14:05:29","date_gmt":"2025-02-05T06:05:29","guid":{"rendered":"https:\/\/17aitech.com\/?p=37835"},"modified":"2025-02-05T14:29:59","modified_gmt":"2025-02-05T06:29:59","slug":"%e3%80%90%e8%ae%ba%e6%96%87%e7%ae%80%e8%af%bb%e3%80%91deepseek-llm-scaling-open-source-language-models-with-longtermism","status":"publish","type":"post","link":"https:\/\/17aitech.com\/?p=37835","title":{"rendered":"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_85 ez-toc-wrap-left-text counter-hierarchy ez-toc-counter ez-toc-light-blue ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u6587\u7ae0\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E5%89%8D%E8%A8%80\" >\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87\" >\u8bba\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E6%A0%B8%E5%BF%83%E5%86%85%E5%AE%B9%E9%98%85%E8%AF%BB\" >\u6838\u5fc3\u5185\u5bb9\u9605\u8bfb<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/17aitech.com\/?p=37835\/#0_%E6%91%98%E8%A6%81\" >0. \u6458\u8981<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/17aitech.com\/?p=37835\/#1_%E5%BC%95%E8%A8%80\" >1. \u5f15\u8a00<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-2\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-2\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-2\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/17aitech.com\/?p=37835\/#2_%E9%A2%84%E8%AE%AD%E7%BB%83\" >2. \u9884\u8bad\u7ec3<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"https:\/\/17aitech.com\/?p=37835\/#21_%E6%95%B0%E6%8D%AE%E9%9B%86\" >2.1 \u6570\u636e\u96c6<\/a><ul class='ez-toc-list-level-5' ><li class='ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-3\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-15\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-3\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-16\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-3\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-17\" href=\"https:\/\/17aitech.com\/?p=37835\/#22_%E6%A8%A1%E5%9E%8B%E7%BB%93%E6%9E%84\" >2.2 \u6a21\u578b\u7ed3\u6784<\/a><ul class='ez-toc-list-level-5' ><li class='ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-18\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-4\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-19\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-4\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-20\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-4\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-21\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E5%BB%B6%E4%BC%B8%E4%BA%86%E8%A7%A3\" >\u5ef6\u4f38\u4e86\u89e3<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-22\" href=\"https:\/\/17aitech.com\/?p=37835\/#23_%E8%B6%85%E5%8F%82%E6%95%B0\" >2.3 \u8d85\u53c2\u6570<\/a><ul class='ez-toc-list-level-5' ><li class='ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-23\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-5\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-24\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-5\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-25\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-5\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-26\" href=\"https:\/\/17aitech.com\/?p=37835\/#24_%E5%9F%BA%E7%A1%80%E8%AE%BE%E6%96%BD\" >2.4 \u57fa\u7840\u8bbe\u65bd<\/a><ul class='ez-toc-list-level-5' ><li class='ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-27\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-6\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-28\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-6\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-29\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-6\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-30\" href=\"https:\/\/17aitech.com\/?p=37835\/#3_%E7%BC%A9%E6%94%BE%E5%AE%9A%E5%BE%8BScaling_Laws\" >3. \u7f29\u653e\u5b9a\u5f8b(Scaling Laws)<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-31\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-7\" >\u8bba\u6587\u539f\u6587<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-32\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-7\" >\u8bba\u6587\u7ffb\u8bd1<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-33\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-7\" >\u8bba\u6587\u7406\u89e3<\/a><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-34\" href=\"https:\/\/17aitech.com\/?p=37835\/#%E5%86%85%E5%AE%B9%E5%B0%8F%E7%BB%93%EF%BC%9A\" >\u5185\u5bb9\u5c0f\u7ed3\uff1a<\/a><\/li><\/ul><\/nav><\/div>\n<h2><span class=\"ez-toc-section\" id=\"%E5%89%8D%E8%A8%80\"><\/span>\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u5728\u751f\u6210\u5f0fAI\u6d6a\u6f6e\u5e2d\u5377\u5168\u7403\u7684\u5f53\u4e0b\uff0c\u56fd\u4ea7\u5927\u6a21\u578bDeepSeek\u51ed\u501f\u5176\u5f00\u6e90\u751f\u6001\u548c\u6280\u672f\u7a81\u7834\u5f02\u519b\u7a81\u8d77\u3002\u8fd1\u671f\u5176\u63a8\u51fa\u76847B\/67B\u53c2\u6570\u6a21\u578b\u5728\u6743\u5a01\u8bc4\u6d4b\u4e2d\u8868\u73b0\u4eae\u773c\uff0c\u66f4\u4ee5&quot;1\u51431\u767e\u4e07tokens&quot;\u7684\u5b9a\u4ef7\u7b56\u7565\u5f15\u53d1\u884c\u4e1a\u9707\u52a8\u3002<\/p>\n<p>\u672c\u6587\u805a\u7126DeepSeek\u9996\u7bc7\u5960\u57fa\u6027\u8bba\u6587\u300aDeepSeek LLM: Scaling Open-Source Language Models with Longtermism\u300b\u5c55\u5f00\u9605\u8bfb\u7406\u89e3\uff0c\u901a\u8fc7\u5bf9\u5176\u6a21\u578b\u67b6\u6784\u3001\u8bad\u7ec3\u65b9\u6cd5\u8bba\u7406\u89e3\uff0c\u63ed\u793a\u4e2d\u56fdAI\u56e2\u961f\u5728\u5927\u578b\u8bed\u8a00\u6a21\u578b\u9886\u57df\u7684\u521b\u65b0\u7a81\u7834\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87\"><\/span>\u8bba\u6587<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u8bba\u6587\u6807\u9898\uff1a\u300aDeepSeek LLM: Scaling Open-Source Language Models with Longtermism\u300b<\/p>\n<p>\u8bba\u6587\u5730\u5740\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/2401.02954\">https:\/\/arxiv.org\/pdf\/2401.02954<\/a><\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E6%A0%B8%E5%BF%83%E5%86%85%E5%AE%B9%E9%98%85%E8%AF%BB\"><\/span>\u6838\u5fc3\u5185\u5bb9\u9605\u8bfb<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<h3><span class=\"ez-toc-section\" id=\"0_%E6%91%98%E8%A6%81\"><\/span>0. \u6458\u8981<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote>\n<p>The rapid development of open-source large language models (LLMs) has been truly remarkable. However, <strong>the scaling laws described in previous literature presents varying conclusions<\/strong>, which casts a dark cloud over scaling LLMs. We delve into the study of scaling laws and <strong>present our distinctive findings that facilitate the scaling of large scale models<\/strong> in two prevalent used opensource configurations, 7B and 67B. Guided by the scaling laws, we introduce DeepSeek LLM, a project dedicated to advancing open-source language models with a long-term perspective. To support the pre-training phase, we have developed a dataset that currently consists of 2 trillion tokens and is continuously expanding. We further conduct supervised fine-tuning (SFT) and direct preference optimization (DPO) on DeepSeek LLM Base models, resulting in the creation of DeepSeek Chat models. Our evaluation results demonstrate that DeepSeek LLM 67B surpasses LLaMA-2 70B across a range of benchmarks, especially in the domains of code, mathematics, and reasoning. Furthermore, open-ended evaluations reveal that our DeepSeek LLM 67B Chat exhibits superior performance compared to GPT-3.5.<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote>\n<p>\u5f00\u6e90\u5927\u8bed\u8a00\u6a21\u578b\uff08LLMs\uff09\u7684\u5feb\u901f\u53d1\u5c55\u7740\u5b9e\u4ee4\u4eba\u77a9\u76ee\u3002\u7136\u800c\uff0c<strong>\u4ee5\u5f80\u6587\u732e\u4e2d\u63cf\u8ff0\u7684\u7f29\u653e\u5b9a\u5f8b\u5b58\u5728\u4e0d\u540c\u7ed3\u8bba<\/strong>\uff0c\u8fd9\u7ed9\u5927\u8bed\u8a00\u6a21\u578b\u7684\u6269\u5c55\u8499\u4e0a\u4e86\u4e00\u5c42\u9634\u5f71\u3002\u6211\u4eec\u6df1\u5165\u7814\u7a76\u7f29\u653e\u5b9a\u5f8b\uff0c\u5e76<strong>\u5448\u73b0\u72ec\u7279\u53d1\u73b0\uff0c\u8fd9\u4e9b\u53d1\u73b0\u6709\u52a9\u4e8e\u5728\u4e24\u79cd\u5e38\u7528\u7684\u5f00\u6e90\u914d\u7f6e\uff0870 \u4ebf\u548c 670 \u4ebf\u53c2\u6570\uff09\u4e0b\u8fdb\u884c\u5927\u89c4\u6a21\u6a21\u578b\u7684\u6269\u5c55<\/strong>\u3002\u5728\u7f29\u653e\u5b9a\u5f8b\u7684\u6307\u5bfc\u4e0b\uff0c\u6211\u4eec\u63a8\u51fa DeepSeek LLM \u9879\u76ee\uff0c\u81f4\u529b\u4e8e\u4ece\u957f\u8fdc\u89d2\u5ea6\u63a8\u8fdb\u5f00\u6e90\u8bed\u8a00\u6a21\u578b\u7684\u53d1\u5c55\u3002\u4e3a\u652f\u6301\u9884\u8bad\u7ec3\u9636\u6bb5\uff0c\u6211\u4eec\u5f00\u53d1\u4e86\u4e00\u4e2a\u76ee\u524d\u5305\u542b 2 \u4e07\u4ebf\u8bcd\u5143\u7684\u6570\u636e\u96c6\uff0c\u4e14\u8be5\u6570\u636e\u96c6\u8fd8\u5728\u6301\u7eed\u6269\u5c55\u3002\u6211\u4eec\u8fdb\u4e00\u6b65\u5bf9 DeepSeek LLM \u57fa\u7840\u6a21\u578b\u8fdb\u884c\u76d1\u7763\u5fae\u8c03\uff08SFT\uff09\u548c\u76f4\u63a5\u504f\u597d\u4f18\u5316\uff08DPO\uff09\uff0c\u4ece\u800c\u521b\u5efa\u4e86 DeepSeek Chat \u6a21\u578b\u3002\u8bc4\u4f30\u7ed3\u679c\u8868\u660e\uff0cDeepSeek LLM 670 \u4ebf\u53c2\u6570\u6a21\u578b\u5728\u4e00\u7cfb\u5217\u57fa\u51c6\u6d4b\u8bd5\u4e2d\u4f18\u4e8e LLaMA &#8211; 2 700 \u4ebf\u53c2\u6570\u6a21\u578b\uff0c\u5c24\u5176\u662f\u5728\u4ee3\u7801\u3001\u6570\u5b66\u548c\u63a8\u7406\u9886\u57df\u3002\u6b64\u5916\uff0c\u5f00\u653e\u5f0f\u8bc4\u4f30\u663e\u793a\uff0c\u6211\u4eec\u7684 DeepSeek LLM 670 \u4ebf\u53c2\u6570 Chat \u6a21\u578b\u7684\u6027\u80fd\u4f18\u4e8e GPT &#8211; 3.5\u3002<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u5728\u6458\u8981\u90e8\u5206\uff0c\u8bba\u6587\u4e3b\u8981\u9610\u8ff0\u4e862\u4e2a\u8981\u70b9\uff1a<\/p>\n<ol>\n<li><strong>\u7814\u7a76<code>Scaling Law<\/code>\u7684\u5fc5\u8981\u6027<\/strong>\uff1a\u7531\u4e8e\u5927\u8bed\u8a00\u6a21\u578b\uff08LLMs\uff09\u7684\u5feb\u901f\u53d1\u5c55\uff0c\u4eba\u4eec\u5bf9\u4e8e <code>\u7f29\u653e\u5b9a\u5f8b\uff08Scaling Law\uff09<\/code> \u7f3a\u5c11\u6df1\u5165\u7814\u7a76\uff1b\u800c\u5728\u4ee5\u5f80\u7684\u6587\u732e\u4e2d\uff0c\u5bf9\u4e8e <code>Scaling Law<\/code> \u7684\u63cf\u8ff0\u5b58\u5728\u4e0d\u540c\u7ed3\u8bba\uff0c\u6240\u4ee5\u9700\u8981\u5bf9\u5176\u8fdb\u884c\u6df1\u5165\u7814\u7a76\u3002<\/li>\n<li><strong>\u7814\u7a76<code>Scaling Law<\/code>\u7684\u7ed3\u8bba<\/strong>\uff1a\u901a\u8fc7\u6df1\u5165\u7814\u7a76\u7f29\u653e\u5b9a\u5f8b\uff0c\u6700\u7ec8\u5728\u4e24\u79cd\u5e38\u7528\u7684\u5f00\u6e90\u914d\u7f6e\uff08<code>70\u4ebf<\/code>\u548c<code>670\u4ebf<\/code>\u53c2\u6570\uff09\u4e0b\u8fdb\u884c\u5927\u89c4\u6a21\u6a21\u578b\u7684\u6269\u5c55\uff0c\u4f7f\u5f97<code>DeepSeek LLM<\/code> 67B \u5728\u591a\u4e2a\u57fa\u51c6\u6d4b\u8bd5\u4e2d\u8868\u73b0\u4f18\u5f02\uff0c\u5c24\u5176\u662f\u5728\u4ee3\u7801\u3001\u6570\u5b66\u548c\u63a8\u7406\u9886\u57df\u3002<\/li>\n<\/ol>\n<h3><span class=\"ez-toc-section\" id=\"1_%E5%BC%95%E8%A8%80\"><\/span>1. \u5f15\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-2\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote>\n<p>Over the past few years, Large Language Models (LLMs) based on decoder-only Transformers (Vaswani et al., 2017) have increasingly become the cornerstone and pathway to achieving Artificial General Intelligence (AGI). By predicting the next word in continuous text, LLMs undergo self-supervised pre-training on massive datasets, enabling them to achieve various purposes and possess many abilities, such as novel creation, text summarization, code completion, and more. Subsequent developments like supervised fine-tuning and reward modeling have enabled Large Language Models (LLMs) to better follow user intentions and instructions. This has endowed them with more versatile conversational capabilities and rapidly expanded their influence.<\/p>\n<p>This wave is sparked with closed products, such as ChatGPT (OpenAI, 2022), Claude (Anthropic, 2023), and Bard (Google, 2023), which are developed with extensive computational resources and substantial annotation costs. These products have significantly raised the community\u2019s expectations for the capabilities of open-source LLMs, consequently inspiring a series of work (Bai et al., 2023; Du et al., 2022; Jiang et al., 2023; Touvron et al., 2023a,b; Yang et al., 2023). Among these, the LLaMA series models (Touvron et al., 2023a,b) stand out. It consolidates a range of works to create an efficient and stable architecture, building well-performing models ranging from 7B to 70B parameters. Consequently, the LLaMA series has become the de facto benchmark for architecture and performance among open-source models.<\/p>\n<p>Following LLaMA, the open-source community has primarily focused on training fixed-size (7B, 13B, 34B, and 70B), high-quality models, <strong>often neglecting research exploration into LLM scaling laws<\/strong> (Hoffmann et al., 2022; Kaplan et al., 2020). Nonetheless, <strong>research on scaling laws is of utmost importance<\/strong>, considering that the current open-source models are merely at the initial stage of Artificial General Intelligence (AGI) development. In addition, early works (Hoffmann et al., 2022; Kaplan et al., 2020) reached <strong>varying conclusions on the scaling of model and data with increased compute budgets and inadequately addressed hyperparameter discussions<\/strong>. In this paper, we extensively investigate the scaling behavior of language models and apply our findings in two widely used large-scale model configurations, namely 7B and 67B. <strong>Our study aims to lay the groundwork for future scaling of open-source LLMs, paving the way for further advancements in this domain<\/strong>. Specifically, we first examined <strong>the scaling laws of batch size and learning rate<\/strong>, and found their trends with model size. Building on this, we conducted a comprehensive study of the scaling laws of the data and model scale, successfully <strong>revealing the optimal model\/data scaling-up allocation strategy<\/strong> and predicting the expected performance of our large-scale models. Additionally, during development, we discovered that the scaling laws derived from different datasets show significant differences. <strong>This suggests that choice of dataset remarkably affects the scaling behavior<\/strong>, indicating that caution should be exercised when generalizing scaling laws across datasets.<\/p>\n<p>Under the guidance of our scaling laws, we build from scratch open-source large language models, and release as much information as possible for community reference. We collect 2 trillion tokens for pre-training, primarily in Chinese and English. At the model level, we generally followed the architecture of LLaMA, but <strong>replaced the cosine learning rate scheduler with a multi-step learning rate scheduler<\/strong>, maintaining performance while facilitating continual training. We collected over 1 million instances for supervised fine-tuning (SFT) (Ouyang et al., 2022) from diverse sources. This paper shares our experiences with different SFT strategies and findings in data ablation techniques. Additionally, we have utilized direct preference optimization (DPO) (Rafailov et al., 2023) to improve the conversational performance of the model.<\/p>\n<p>We conduct extensive evaluations using our base and chat models. The evaluation results demonstrate that DeepSeek LLM surpasses LLaMA-2 70B across various benchmarks, particularly in the fields of code, mathematics, and reasoning. Following SFT and DPO, the DeepSeek 67B chat model outperforms GPT-3.5 in both Chinese and English open-ended evaluations. This highlights the superior performance of DeepSeek 67B in generating high-quality responses and engaging in meaningful conversations in both languages. Furthermore, the safety evaluation indicates that DeepSeek 67B Chat can provide harmless responses in practice.<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-2\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote>\n<p>\u5728\u8fc7\u53bb\u51e0\u5e74\u91cc\uff0c\u57fa\u4e8e\u4ec5\u89e3\u7801\u5668\u67b6\u6784\u7684 <code>Transformer<\/code>\uff08Vaswani \u7b49\u4eba\uff0c2017\uff09\u7684<code>\u5927\u8bed\u8a00\u6a21\u578b\uff08LLMs\uff09<\/code>\u8d8a\u6765\u8d8a\u6210\u4e3a\u5b9e\u73b0<code>\u901a\u7528\u4eba\u5de5\u667a\u80fd\uff08AGI\uff09<\/code>\u7684\u57fa\u77f3\u548c\u9014\u5f84\u3002\u901a\u8fc7\u9884\u6d4b\u8fde\u7eed\u6587\u672c\u4e2d\u7684\u4e0b\u4e00\u4e2a\u5355\u8bcd\uff0c\u5927\u8bed\u8a00\u6a21\u578b\u5728\u5927\u89c4\u6a21\u6570\u636e\u96c6\u4e0a\u8fdb\u884c\u81ea\u76d1\u7763\u9884\u8bad\u7ec3\uff0c\u8fd9\u4f7f\u5b83\u4eec\u80fd\u591f\u8fbe\u6210\u591a\u79cd\u76ee\u7684\uff0c\u5e76\u5177\u5907\u8bf8\u591a\u80fd\u529b\uff0c\u5982\u5c0f\u8bf4\u521b\u4f5c\u3001\u6587\u672c\u6458\u8981\u3001\u4ee3\u7801\u8865\u5168\u7b49\u3002\u968f\u540e\u51fa\u73b0\u7684\u76d1\u7763\u5fae\u8c03\u3001\u5956\u52b1\u5efa\u6a21\u7b49\u6280\u672f\uff0c\u8ba9\u5927\u8bed\u8a00\u6a21\u578b\u80fd\u66f4\u597d\u5730\u7406\u89e3\u7528\u6237\u610f\u56fe\u3001\u9075\u5faa\u6307\u4ee4\uff0c\u8d4b\u4e88\u5176\u66f4\u4e30\u5bcc\u7684\u5bf9\u8bdd\u80fd\u529b\uff0c\u5f71\u54cd\u529b\u4e5f\u8fc5\u901f\u6269\u5927\u3002<\/p>\n<p>\u8fd9\u4e00\u6ce2\u53d1\u5c55\u7531\u95ed\u6e90\u4ea7\u54c1\u5f15\u53d1\uff0c\u6bd4\u5982 <code>ChatGPT<\/code>\uff08OpenAI\uff0c2022 \u5e74\uff09\u3001<code>Claude<\/code>\uff08Anthropic\uff0c2023 \u5e74\uff09\u548c <code>Bard<\/code>\uff08\u8c37\u6b4c\uff0c2023 \u5e74\uff09\uff0c\u8fd9\u4e9b\u4ea7\u54c1\u7684\u5f00\u53d1\u9700\u8981\u5927\u91cf\u8ba1\u7b97\u8d44\u6e90\u548c\u9ad8\u6602\u7684\u6807\u6ce8\u6210\u672c\u3002\u8fd9\u4e9b\u4ea7\u54c1\u6781\u5927\u5730\u63d0\u9ad8\u4e86\u793e\u533a\u5bf9\u5f00\u6e90\u5927\u8bed\u8a00\u6a21\u578b\u80fd\u529b\u7684\u671f\u671b\uff0c\u4ece\u800c\u6fc0\u53d1\u4e86\u4e00\u7cfb\u5217\u7814\u7a76\u5de5\u4f5c\uff08Bai \u7b49\u4eba\uff0c2023 \u5e74\uff1bDu \u7b49\u4eba\uff0c2022 \u5e74\uff1bJiang \u7b49\u4eba\uff0c2023 \u5e74\uff1bTouvron \u7b49\u4eba\uff0c2023a\uff0cb\uff1bYang \u7b49\u4eba\uff0c2023 \u5e74\uff09\u3002\u5728\u8fd9\u4e9b\u5de5\u4f5c\u4e2d\uff0c<code>LLaMA <\/code>\u7cfb\u5217\u6a21\u578b\uff08Touvron \u7b49\u4eba\uff0c2023a\uff0cb\uff09\u8131\u9896\u800c\u51fa\u3002\u5b83\u6574\u5408\u4e86\u4e00\u7cfb\u5217\u6210\u679c\uff0c\u521b\u5efa\u4e86\u9ad8\u6548\u7a33\u5b9a\u7684\u67b6\u6784\uff0c\u6784\u5efa\u4e86\u53c2\u6570\u89c4\u6a21\u4ece <code>70 \u4ebf<\/code>\u5230 <code>700 \u4ebf<\/code>\u4e0d\u7b49\u7684\u9ad8\u6027\u80fd\u6a21\u578b\u3002\u56e0\u6b64\uff0c<code>LLaMA <\/code>\u7cfb\u5217\u5df2\u6210\u4e3a\u5f00\u6e90\u6a21\u578b\u4e2d\u67b6\u6784\u548c\u6027\u80fd\u65b9\u9762\u4e8b\u5b9e\u4e0a\u7684\u57fa\u51c6\u3002<\/p>\n<p>\u5728 <code>LLaMA<\/code> \u4e4b\u540e\uff0c\u5f00\u6e90\u793e\u533a\u4e3b\u8981\u4e13\u6ce8\u4e8e\u8bad\u7ec3\u56fa\u5b9a\u89c4\u6a21\uff0870 \u4ebf\u3001130 \u4ebf\u3001340 \u4ebf\u548c 700 \u4ebf\u53c2\u6570\uff09\u7684\u9ad8\u8d28\u91cf\u6a21\u578b\uff0c\u5374\u5e38\u5e38\u5ffd\u89c6\u5bf9\u5927\u8bed\u8a00\u6a21\u578b\u7f29<code>\u653e\u5b9a\u5f8b<\/code>\u7684\u7814\u7a76\u63a2\u7d22\uff08Hoffmann \u7b49\u4eba\uff0c2022\uff1bKaplan \u7b49\u4eba\uff0c2020\uff09\u3002\u5c3d\u7ba1\u5982\u6b64\uff0c\u9274\u4e8e\u5f53\u524d\u7684\u5f00\u6e90\u6a21\u578b\u4ec5\u4ec5\u5904\u4e8e<code>\u901a\u7528\u4eba\u5de5\u667a\u80fd\uff08AGI\uff09<\/code>\u53d1\u5c55\u7684\u521d\u59cb\u9636\u6bb5\uff0c\u5bf9<strong>\u7f29\u653e\u5b9a\u5f8b\u7684\u7814\u7a76\u81f3\u5173\u91cd\u8981<\/strong>\u3002\u6b64\u5916\uff0c\u65e9\u671f\u7684\u7814\u7a76\uff08Hoffmann \u7b49\u4eba\uff0c2022\uff1bKaplan \u7b49\u4eba\uff0c2020\uff09<strong>\u5728\u6a21\u578b\u548c\u6570\u636e\u968f\u7740\u8ba1\u7b97\u8d44\u6e90\u589e\u52a0\u7684\u7f29\u653e\u95ee\u9898\u4e0a\u5f97\u51fa\u4e86\u4e0d\u540c\u7ed3\u8bba\uff0c\u800c\u4e14\u5bf9\u8d85\u53c2\u6570\u7684\u8ba8\u8bba\u4e5f\u4e0d\u591f\u5145\u5206<\/strong>\u3002\u5728\u672c\u6587\u4e2d\uff0c\u6211\u4eec\u5e7f\u6cdb\u7814\u7a76\u4e86\u8bed\u8a00\u6a21\u578b\u7684\u7f29\u653e\u884c\u4e3a\uff0c\u5e76\u5c06\u7814\u7a76\u7ed3\u679c\u5e94\u7528\u4e8e\u4e24\u79cd\u5e7f\u6cdb\u4f7f\u7528\u7684\u5927\u89c4\u6a21\u6a21\u578b\u914d\u7f6e\uff0c\u5373 <code>70 \u4ebf<\/code>\u548c <code>670 \u4ebf<\/code>\u53c2\u6570\u7684\u6a21\u578b\u3002<strong>\u6211\u4eec\u7684\u7814\u7a76\u65e8\u5728\u4e3a\u672a\u6765\u5f00\u6e90\u5927\u8bed\u8a00\u6a21\u578b\u7684\u6269\u5c55\u5960\u5b9a\u57fa\u7840\uff0c\u4e3a\u8be5\u9886\u57df\u7684\u8fdb\u4e00\u6b65\u53d1\u5c55\u94fa\u5e73\u9053\u8def<\/strong>\u3002\u5177\u4f53\u800c\u8a00\uff0c\u6211\u4eec\u9996\u5148\u7814\u7a76\u4e86<strong>\u6279\u91cf\u5927\u5c0f\u548c\u5b66\u4e60\u7387\u7684\u7f29\u653e\u5b9a\u5f8b<\/strong>\uff0c\u53d1\u73b0\u4e86\u5b83\u4eec\u968f\u6a21\u578b\u89c4\u6a21\u53d8\u5316\u7684\u8d8b\u52bf\u3002\u5728\u6b64\u57fa\u7840\u4e0a\uff0c\u6211\u4eec\u5bf9\u6570\u636e\u548c\u6a21\u578b\u89c4\u6a21\u7684\u7f29\u653e\u5b9a\u5f8b\u8fdb\u884c\u4e86\u5168\u9762\u7814\u7a76\uff0c\u6210\u529f<strong>\u63ed\u793a\u4e86\u6700\u4f18\u7684\u6a21\u578b\/\u6570\u636e\u6269\u5c55\u5206\u914d\u7b56\u7565<\/strong>\uff0c\u5e76\u9884\u6d4b\u4e86\u5927\u89c4\u6a21\u6a21\u578b\u7684\u9884\u671f\u6027\u80fd\u3002\u6b64\u5916\uff0c\u5728\u5f00\u53d1\u8fc7\u7a0b\u4e2d\u6211\u4eec\u53d1\u73b0\uff0c\u4e0d\u540c\u6570\u636e\u96c6\u5f97\u51fa\u7684\u7f29\u653e\u5b9a\u5f8b\u5b58\u5728\u663e\u8457\u5dee\u5f02\u3002<strong>\u8fd9\u8868\u660e\u6570\u636e\u96c6\u7684\u9009\u62e9\u5bf9\u7f29\u653e\u884c\u4e3a\u6709\u663e\u8457\u5f71\u54cd<\/strong>\uff0c\u610f\u5473\u7740\u5728\u8de8\u6570\u636e\u96c6\u63a8\u5e7f\u7f29\u653e\u5b9a\u5f8b\u65f6\u5e94\u8c28\u614e\u884c\u4e8b\u3002<\/p>\n<p>\u5728\u6211\u4eec\u7684\u7f29\u653e\u5b9a\u5f8b\u6307\u5bfc\u4e0b\uff0c\u6211\u4eec\u4ece\u5934\u5f00\u59cb\u6784\u5efa\u5f00\u6e90\u5927\u8bed\u8a00\u6a21\u578b\uff0c\u5e76\u5c3d\u53ef\u80fd\u591a\u5730\u53d1\u5e03\u4fe1\u606f\u4f9b\u793e\u533a\u53c2\u8003\u3002\u6211\u4eec\u6536\u96c6\u4e86 <code>2 \u4e07\u4ebf<\/code>\u4e2a\u8bcd\u5143\u7528\u4e8e\u9884\u8bad\u7ec3\uff0c\u4e3b\u8981\u662f\u4e2d\u6587\u548c\u82f1\u6587\u6570\u636e\u3002\u5728\u6a21\u578b\u5c42\u9762\uff0c\u6211\u4eec\u603b\u4f53\u4e0a\u9075\u5faa <code>LLaMA<\/code> \u7684\u67b6\u6784\uff0c\u4f46\u5c06<strong>\u4f59\u5f26\u9000\u706b\u5b66\u4e60\u7387\u7684\u8c03\u5ea6\u5668\u66ff\u6362\u4e3a\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668<\/strong>\uff0c\u8fd9\u6837\u5728\u4fdd\u6301\u6a21\u578b\u6027\u80fd\u7684\u540c\u65f6\uff0c\u66f4\u4fbf\u4e8e\u6301\u7eed\u8bad\u7ec3\u3002\u6211\u4eec\u4ece\u591a\u79cd\u6765\u6e90\u6536\u96c6\u4e86\u8d85\u8fc7 <code>100\u4e07<\/code>\u4e2a\u5b9e\u4f8b\uff0c\u7528\u4e8e\u76d1\u7763\u5fae\u8c03\uff08SFT\uff09\uff08\u6b27\u9633\u7b49\u4eba\uff0c2022\uff09\u3002\u672c\u6587\u5c06\u5206\u4eab\u6211\u4eec\u5728\u4e0d\u540c\u76d1\u7763\u5fae\u8c03\u7b56\u7565\u65b9\u9762\u7684\u7ecf\u9a8c\uff0c\u4ee5\u53ca\u5728\u6570\u636e\u6d88\u878d\u6280\u672f\u4e0a\u7684\u53d1\u73b0\u3002\u6b64\u5916\uff0c\u6211\u4eec\u5229\u7528\u76f4\u63a5\u504f\u597d\u4f18\u5316\uff08DPO\uff09\uff08\u62c9\u6cd5\u4f0a\u6d1b\u592b\u7b49\u4eba\uff0c2023\uff09\u6765\u63d0\u5347\u6a21\u578b\u7684\u5bf9\u8bdd\u6027\u80fd\u3002<\/p>\n<p>\u6211\u4eec\u4f7f\u7528\u57fa\u7840\u6a21\u578b\u548c\u804a\u5929\u6a21\u578b\u8fdb\u884c\u4e86\u5e7f\u6cdb\u8bc4\u4f30\u3002\u8bc4\u4f30\u7ed3\u679c\u8868\u660e\uff0c<code>DeepSeek LLM<\/code> \u5728\u5404\u79cd\u57fa\u51c6\u6d4b\u8bd5\u4e2d\u5747\u4f18\u4e8e <code>LLaMA-2 70B<\/code>\uff0c\u5c24\u5176\u662f\u5728\u4ee3\u7801\u3001\u6570\u5b66\u548c\u63a8\u7406\u9886\u57df\u3002\u7ecf\u8fc7\u76d1\u7763\u5fae\u8c03\uff08SFT\uff09\u548c\u76f4\u63a5\u504f\u597d\u4f18\u5316\uff08DPO\uff09\u540e\uff0c<code>DeepSeek 67B<\/code> \u804a\u5929\u6a21\u578b\u5728\u4e2d\u6587\u548c\u82f1\u6587\u5f00\u653e\u5f0f\u8bc4\u4f30\u4e2d\u5747\u4f18\u4e8e <code>GPT-3.5<\/code>\u3002\u8fd9\u51f8\u663e\u4e86 <code>DeepSeek 67B<\/code> \u5728\u751f\u6210\u9ad8\u8d28\u91cf\u56de\u590d\u4ee5\u53ca\u7528\u4e24\u79cd\u8bed\u8a00\u8fdb\u884c\u6709\u610f\u4e49\u5bf9\u8bdd\u65b9\u9762\u7684\u5353\u8d8a\u6027\u80fd\u3002\u6b64\u5916\uff0c\u5b89\u5168\u8bc4\u4f30\u8868\u660e\uff0c<code>DeepSeek 67B Chat<\/code> \u5728\u5b9e\u9645\u5e94\u7528\u4e2d\u80fd\u591f\u7ed9\u51fa\u65e0\u5bb3\u7684\u56de\u590d\u3002<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-2\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u901a\u8fc7\u5bf9\u5f15\u8a00\u7684\u9605\u8bfb\uff0c\u6211\u4eec\u4e86\u89e3\u5230\uff1a<\/p>\n<ul>\n<li><strong>\u7f29\u653e\u5b9a\u5f8b(Scaling Law)\u7684\u7814\u7a76\u81f3\u5173\u91cd\u8981<\/strong>\uff0c\u7279\u522b\u662f\u5728\u5f53\u524d\u5f00\u6e90\u6a21\u578b\u5904\u4e8e\u901a\u7528\u4eba\u5de5\u667a\u80fd\uff08AGI\uff09\u53d1\u5c55\u7684\u521d\u59cb\u9636\u6bb5\uff1b<\/li>\n<li>\u5728\u4ee5\u5f80\u7684\u7814\u7a76\u4e2d\uff0c\u5bf9\u4e8e<code>Scaling Law<\/code>\u7684\u63cf\u8ff0\u5b58\u5728\u4e0d\u540c\u7ed3\u8bba\uff0c\u800c\u4e14\u5bf9\u4e8e\u8d85\u53c2\u6570\u7684\u8ba8\u8bba\u4e5f\u4e0d\u591f\u5145\u5206\uff1b<\/li>\n<li>\u57fa\u4e8e\u4ee5\u4e0a\u7684\u95ee\u9898\uff0c\u8fd9\u7bc7\u8bba\u6587\u4e3b\u8981\u7814\u7a76\u4e86Scaling Law\uff0c<strong>\u53d1\u73b0\u6570\u636e\u96c6\u7684\u9009\u62e9\u5bf9\u4e8eScaling Law\u7684\u5f71\u54cd\u663e\u8457<\/strong>\uff0c\u5e76\u63d0\u51fa\u4e86Scaling Law\u7684\u4f18\u5316\u7b56\u7565\uff1b<\/li>\n<li>\u5728\u5177\u4f53\u7814\u7a76\u8fc7\u7a0b\u4e2d\uff0c\u603b\u4f53\u4e0a\u9075\u5faa\u4e86<code>LLaMA<\/code>\u7684\u67b6\u6784\uff0c\u4f46<strong>\u5c06\u4f59\u5f26\u9000\u706b\u5b66\u4e60\u7387\u7684\u8c03\u5ea6\u5668\u66ff\u6362\u4e3a\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668<\/strong>\uff0c\u540c\u65f6\u901a\u8fc7\u76d1\u7763\u5fae\u8c03\uff08SFT\uff09\u548c\u76f4\u63a5\u504f\u597d\u4f18\u5316\uff08DPO\uff09\u6765\u63d0\u5347\u6a21\u578b\u7684\u5bf9\u8bdd\u6027\u80fd\u3002<\/li>\n<li>\u5728\u8bc4\u4f30\u8fc7\u7a0b\u4e2d\uff0c<code>DeepSeek LLM 67B <\/code>\u5728\u591a\u4e2a\u57fa\u51c6\u6d4b\u8bd5\u4e2d\u8868\u73b0\u4f18\u5f02\uff0c\u5c24\u5176\u662f\u5728\u4ee3\u7801\u3001\u6570\u5b66\u548c\u63a8\u7406\u9886\u57df\u3002<\/li>\n<\/ul>\n<h3><span class=\"ez-toc-section\" id=\"2_%E9%A2%84%E8%AE%AD%E7%BB%83\"><\/span>2. \u9884\u8bad\u7ec3<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<h4><span class=\"ez-toc-section\" id=\"21_%E6%95%B0%E6%8D%AE%E9%9B%86\"><\/span>2.1 \u6570\u636e\u96c6<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-3\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>Our main objective is to comprehensively enhance the richness and diversity of the dataset. We have gained valuable insights from reputable sources such as (Computer, 2023; Gao et al., 2020; Penedo et al., 2023; Touvron et al., 2023a). To achieve these goals, we have organized our approach into three essential stages: <strong>deduplication, filtering, and remixing<\/strong>. The deduplication and remixing stages ensure a diverse representation of the data by sampling unique instances. The filtering stage enhances the density of information, thereby enabling more efficient and effective model training.<\/p>\n<p>We adopted <strong>an aggressive deduplication strategy<\/strong>, <strong>expanding the deduplication scope<\/strong>. Our analysis revealed that deduplicating the entire Common Crawl corpus results in higher removal of duplicate instances compared to deduplicating within a single dump. Table 1 illustrates that <strong>deduplicating across 91 dumps eliminates four times more documents than a single dump method.<\/strong><\/p>\n<\/blockquote>\n<p><a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/table1_deduplication.png\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/table1_deduplication.png\" alt=\"\" \/><\/a><\/p>\n<blockquote>\n<p>For our tokenizer, we implemented the Byte-level Byte-Pair Encoding (BBPE) algorithm based on the tokenizers library (Huggingface Team, 2019). Pre-tokenization was employed to prevent the merging of tokens from different character categories such as new lines, punctuation, and Chinese-Japanese-Korean (CJK) symbols, similar to GPT-2 (Radford et al., 2019). We also chose to split numbers into individual digits following the approach used in (Touvron et al., 2023a,b). Based on our prior experience, we set the number of conventional tokens in the vocabulary at 100000. The tokenizer was trained on a multilingual corpus of approximately 24 GB, and we augmented the final vocabulary with 15 special tokens, bringing the total size to 100015. To ensure computational efficiency during training and to reserve space for any additional special tokens that might be needed in the future, we configured the model\u2019s vocabulary size to 102400 for training.<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-3\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>\u6211\u4eec\u7684\u4e3b\u8981\u76ee\u6807\u662f\u5168\u9762\u63d0\u5347\u6570\u636e\u96c6\u7684\u4e30\u5bcc\u6027\u548c\u591a\u6837\u6027\u3002\u6211\u4eec\u4ece\u4e00\u4e9b\u53ef\u9760\u7684\u8d44\u6599\u6765\u6e90\uff08\u5982 Computer, 2023; Gao \u7b49\u4eba\uff0c2020; Penedo \u7b49\u4eba\uff0c2023; Touvron \u7b49\u4eba\uff0c2023a\uff09\u4e2d\u83b7\u5f97\u4e86\u5b9d\u8d35\u7684\u89c1\u89e3\u3002\u4e3a\u5b9e\u73b0\u8fd9\u4e9b\u76ee\u6807\uff0c\u6211\u4eec\u5c06\u65b9\u6cd5\u5206\u4e3a<strong>\u4e09\u4e2a\u5173\u952e\u9636\u6bb5\uff1a\u53bb\u91cd\u3001\u7b5b\u9009\u548c\u91cd\u65b0\u6df7\u5408<\/strong>\u3002\u53bb\u91cd\u548c\u91cd\u65b0\u6df7\u5408\u9636\u6bb5\u901a\u8fc7\u5bf9\u72ec\u7279\u5b9e\u4f8b\u8fdb\u884c\u91c7\u6837\uff0c\u786e\u4fdd\u6570\u636e\u5177\u6709\u591a\u6837\u5316\u7684\u4ee3\u8868\u6027\u3002\u7b5b\u9009\u9636\u6bb5\u5219\u63d0\u9ad8\u4e86\u4fe1\u606f\u5bc6\u5ea6\uff0c\u4ece\u800c\u4f7f\u6a21\u578b\u8bad\u7ec3\u66f4\u52a0\u9ad8\u6548\u3002<\/p>\n<p><strong>\u6211\u4eec\u91c7\u7528\u4e86\u6fc0\u8fdb\u7684\u53bb\u91cd\u7b56\u7565\uff0c\u6269\u5927\u4e86\u53bb\u91cd\u8303\u56f4<\/strong>\u3002\u6211\u4eec\u7684\u5206\u6790\u663e\u793a\uff0c\u4e0e\u5728\u5355\u4e2a\u6570\u636e\u8f6c\u50a8\u4e2d\u8fdb\u884c\u53bb\u91cd\u76f8\u6bd4\uff0c\u5bf9\u6574\u4e2a <code>Common Crawl<\/code> \u8bed\u6599\u5e93\u8fdb\u884c\u53bb\u91cd\u80fd\u5220\u9664\u66f4\u591a\u7684\u91cd\u590d\u5b9e\u4f8b\u3002\u8868 1 \u8868\u660e\uff0c\u8de8 91 \u4e2a\u6570\u636e\u8f6c\u50a8\u8fdb\u884c\u53bb\u91cd\u6240\u5220\u9664\u7684\u6587\u6863\u6570\u91cf\u662f\u5355\u6570\u636e\u8f6c\u50a8\u53bb\u91cd\u65b9\u6cd5\u7684\u56db\u500d\u3002<\/p>\n<p>\u5bf9\u4e8e\u6211\u4eec\u7684\u5206\u8bcd\u5668\uff0c\u6211\u4eec\u57fa\u4e8e <code>Huggingface<\/code> \u56e2\u961f\uff082019 \u5e74\uff09\u5f00\u53d1\u7684 <code>tokenizers<\/code> \u5e93\uff0c\u5b9e\u73b0\u4e86<code>\u5b57\u8282\u7ea7\u5b57\u8282\u5bf9\u7f16\u7801\uff08BBPE\uff09<\/code>\u7b97\u6cd5\u3002\u6211\u4eec\u91c7\u7528\u4e86\u9884\u5206\u8bcd\u6280\u672f\u6765\u9632\u6b62\u4e0d\u540c\u5b57\u7b26\u7c7b\u522b\u7684\u6807\u8bb0\u5408\u5e76\uff0c\u4f8b\u5982\u6362\u884c\u7b26\u3001\u6807\u70b9\u7b26\u53f7\u4ee5\u53ca\u4e2d\u65e5\u97e9\uff08CJK\uff09\u7b26\u53f7\uff0c\u8fd9\u4e0e <code>GPT-2<\/code>\uff08Radford \u7b49\u4eba\uff0c2019 \u5e74\uff09\u7684\u505a\u6cd5\u7c7b\u4f3c\u3002\u6211\u4eec\u8fd8\u53c2\u7167\uff08Touvron \u7b49\u4eba\uff0c2023a\uff0cb\uff09\u7684\u65b9\u6cd5\uff0c\u9009\u62e9\u5c06\u6570\u5b57\u62c6\u5206\u4e3a\u5355\u4e2a\u6570\u5b57\u3002\u6839\u636e\u6211\u4eec\u4e4b\u524d\u7684\u7ecf\u9a8c\uff0c\u6211\u4eec\u5c06\u8bcd\u6c47\u8868\u4e2d\u7684\u5e38\u89c4\u6807\u8bb0\u6570\u91cf\u8bbe\u7f6e\u4e3a <code>100000<\/code>\u3002\u5206\u8bcd\u5668\u5728\u5927\u7ea6 <code>24GB<\/code> \u7684\u591a\u8bed\u8a00\u8bed\u6599\u5e93\u4e0a\u8fdb\u884c\u8bad\u7ec3\uff0c\u5e76\u4e14\u6211\u4eec\u5728\u6700\u7ec8\u7684\u8bcd\u6c47\u8868\u4e2d\u589e\u52a0\u4e86 <code>15<\/code> \u4e2a\u7279\u6b8a\u6807\u8bb0\uff0c\u4f7f\u603b\u5927\u5c0f\u8fbe\u5230 <code>100015<\/code>\u3002\u4e3a\u786e\u4fdd\u8bad\u7ec3\u671f\u95f4\u7684\u8ba1\u7b97\u6548\u7387\uff0c\u5e76\u4e3a\u672a\u6765\u53ef\u80fd\u9700\u8981\u7684\u4efb\u4f55\u989d\u5916\u7279\u6b8a\u6807\u8bb0\u9884\u7559\u7a7a\u95f4\uff0c\u6211\u4eec\u5c06\u8bad\u7ec3\u65f6\u6a21\u578b\u7684\u8bcd\u6c47\u8868\u5927\u5c0f\u914d\u7f6e\u4e3a <code>102400<\/code>\u3002<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-3\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<ul>\n<li>\u8bba\u6587\u4e2d\u4ecb\u7ecd\u5230\uff0c\u5728\u6570\u636e\u96c6\u5904\u7406\u65b9\u9762\u4e3b\u8981\u91c7\u7528\u4e09\u79cd\u65b9\u6cd5\uff1a\u53bb\u91cd\u3001\u7b5b\u9009\u548c\u91cd\u65b0\u6df7\u5408\u3002<\/li>\n<li>\u5728\u53bb\u91cd\u65b9\u9762\uff0c\u91c7\u7528\u4e86\u6fc0\u8fdb\u7684\u53bb\u91cd\u7b56\u7565\uff0c\u6269\u5927\u4e86\u53bb\u91cd\u8303\u56f4\u3002\u5b9e\u9a8c\u6570\u636e\u8868\u660e\uff0c\u8de891\u4e2a\u6570\u636e\u8f6c\u50a8\u8fdb\u884c\u53bb\u91cd\u6240\u5220\u9664\u7684\u6587\u6863\u6570\u91cf\u662f\u5355\u6570\u636e\u8f6c\u50a8\u53bb\u91cd\u65b9\u6cd5\u7684\u56db\u500d\u3002<\/li>\n<li>\u5728\u5206\u8bcd\u5668\u65b9\u9762\uff0c\u91c7\u7528\u4e86\u5b57\u8282\u7ea7\u5b57\u8282\u5bf9\u7f16\u7801\uff08BBPE\uff09\u7b97\u6cd5\uff0c\u5e76\u91c7\u7528\u4e86\u9884\u5206\u8bcd\u6280\u672f\u6765\u9632\u6b62\u4e0d\u540c\u5b57\u7b26\u7c7b\u522b\u7684\u6807\u8bb0\u5408\u5e76\uff0c\u4f8b\u5982\u6362\u884c\u7b26\u3001\u6807\u70b9\u7b26\u53f7\u4ee5\u53ca\u4e2d\u65e5\u97e9\uff08CJK\uff09\u7b26\u53f7\uff0c\u8fd9\u4e0e GPT-2\uff08Radford \u7b49\u4eba\uff0c2019 \u5e74\uff09\u7684\u505a\u6cd5\u7c7b\u4f3c\u3002<\/li>\n<\/ul>\n<blockquote>\n<p>\u4e2a\u4eba\u7406\u89e3\uff0c\u8bad\u7ec3\u5927\u6a21\u578b\u5c31\u597d\u6bd4\u8bfb\u4e66\u4e00\u6837\uff0c\u91cd\u70b9\u5728\u4e8e\u8bfb\u597d\u4e66\uff0c\u800c\u4e0d\u662f\u4e71\u516b\u4e03\u7cdf\u4ec0\u4e48\u4e66\u90fd\u8bfb\u3002<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"22_%E6%A8%A1%E5%9E%8B%E7%BB%93%E6%9E%84\"><\/span>2.2 \u6a21\u578b\u7ed3\u6784<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p><a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/table2_model_structure.png\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/table2_model_structure.png\" alt=\"\" \/><\/a><\/p>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-4\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>The micro design of DeepSeek LLM largely follows the design of LLaMA (Touvron et al., 2023a,b), adopting a Pre-Norm structure with RMSNorm (Zhang and Sennrich, 2019) function and using SwiGLU (Shazeer, 2020) as the activation function for the Feed-Forward Network (FFN), with an intermediate layer dimension of $\\frac{8}{3} d_{model }$ . It also incorporates Rotary Embedding (Su et al., 2024) for positional encoding. To optimize inference cost, the 67B model uses GroupedQuery Attention (GQA) (Ainslie et al., 2023) instead of the traditional Multi-Head Attention (MHA).<\/p>\n<p>However, in terms of macro design, DeepSeek LLM differs slightly. Specifically, DeepSeek LLM 7B is a 30-layer network, while DeepSeek LLM 67B has 95 layers. These layer adjustments, while maintaining parameter consistency with other open-source models, also facilitate model pipeline partitioning to optimize training and inference.<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-4\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>DeepSeek LLM \u7684\u5fae\u89c2\u8bbe\u8ba1\u5728\u5f88\u5927\u7a0b\u5ea6\u4e0a\u9075\u5faa <code>LLaMA<\/code>\uff08Touvron \u7b49\u4eba\uff0c2023a\uff0cb\uff09\u7684\u8bbe\u8ba1\uff0c\u91c7\u7528\u5e26\u6709 <code>RMSNorm<\/code>\uff08Zhang \u548c Sennrich\uff0c2019\uff09\u51fd\u6570\u7684 <code>Pre-Norm<\/code> \u7ed3\u6784\uff0c\u5e76\u4f7f\u7528 <code>SwiGLU<\/code>\uff08Shazeer\uff0c2020\uff09\u4f5c\u4e3a<code>MARKDOWN_HASH7d211f462f3b56c0dbf9f92028c9903aMARKDOWN<em>HASH<\/code>\u7684\u6fc0\u6d3b\u51fd\u6570\uff0c\u5176\u4e2d\u95f4\u5c42\u7ef4\u5ea6\u4e3a $\\frac{8}{3} d<\/em>{model }$ \u3002\u5b83\u8fd8\u91c7\u7528<code>\u65cb\u8f6c\u5d4c\u5165<\/code>\uff08Su \u7b49\u4eba\uff0c2024\uff09\u8fdb\u884c\u4f4d\u7f6e\u7f16\u7801\u3002\u4e3a\u4e86\u4f18\u5316\u63a8\u7406\u6210\u672c\uff0c<code>670 \u4ebf<\/code>\u53c2\u6570\u7684\u6a21\u578b\u4f7f\u7528<code>\u5206\u7ec4\u67e5\u8be2\u6ce8\u610f\u529b\u673a\u5236\uff08GQA\uff09<\/code>\uff08Ainslie \u7b49\u4eba\uff0c2023\uff09\uff0c\u800c\u975e\u4f20\u7edf\u7684<code>\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\uff08MHA\uff09<\/code>\u3002<\/p>\n<p>\u7136\u800c\uff0c\u5728\u5b8f\u89c2\u8bbe\u8ba1\u65b9\u9762\uff0c<code>DeepSeek LLM<\/code> \u7565\u6709\u4e0d\u540c\u3002\u5177\u4f53\u6765\u8bf4\uff0c<code>DeepSeek LLM<\/code> 7B \u662f\u4e00\u4e2a <code>30<\/code> \u5c42\u7684\u7f51\u7edc\uff0c\u800c <code>DeepSeek LLM<\/code> 67B \u6709 <code>95<\/code> \u5c42\u3002\u8fd9\u4e9b\u5c42\u6570\u7684\u8c03\u6574\uff0c\u5728\u4fdd\u6301\u4e0e\u5176\u4ed6\u5f00\u6e90\u6a21\u578b\u53c2\u6570\u4e00\u81f4\u6027\u7684\u540c\u65f6\uff0c\u4e5f\u6709\u5229\u4e8e\u6a21\u578b\u6d41\u6c34\u7ebf\u5206\u533a\uff0c\u4ece\u800c\u4f18\u5316\u8bad\u7ec3\u548c\u63a8\u7406\u8fc7\u7a0b\u3002<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-4\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<p>\u901a\u8fc7\u5bf9\u6bd4Deepseek\u4e0eLLaMa\uff0c\u4e3b\u8981\u533a\u522b\u70b9\u6709\uff1a<\/p>\n<ul>\n<li><strong>\u524d\u9988\u7f51\u7edc<\/strong>\uff1aDeepSeek LLM \u91c7\u7528\u4e86\u5e26\u6709 <code>RMSNorm<\/code> \u7684 <code>Pre-Norm<\/code> \u7ed3\u6784\uff0c\u5e76\u4f7f\u7528 <code>SwiGLU<\/code> \u4f5c\u4e3a\u524d\u9988\u7f51\u7edc\uff08FFN\uff09\u7684\u6fc0\u6d3b\u51fd\u6570\uff0c\u5176\u4e2d\u95f4\u5c42\u7ef4\u5ea6\u4e3a <code class=\"katex-inline\">\\frac{8}{3} d_{model }<\/code> \u3002<\/li>\n<li><strong>\u4f4d\u7f6e\u7f16\u7801<\/strong>\uff1a\u91c7\u7528\u4e86 <code>\u65cb\u8f6c\u5d4c\u5165<\/code> \u8fdb\u884c\u4f4d\u7f6e\u7f16\u7801\u3002<\/li>\n<li><strong>\u6ce8\u610f\u529b\u673a\u5236<\/strong>\uff1a\u4e3a\u4e86\u4f18\u5316\u63a8\u7406\u6210\u672c\uff0c670 \u4ebf\u53c2\u6570\u7684\u6a21\u578b\u4f7f\u7528 <code>\u5206\u7ec4\u67e5\u8be2\u6ce8\u610f\u529b\u673a\u5236\uff08GQA\uff09<\/code>\uff0c\u800c\u975e\u4f20\u7edf\u7684 <code>\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\uff08MHA\uff09<\/code>\u3002<\/li>\n<li><strong>\u5b8f\u89c2\u8bbe\u8ba1<\/strong>\uff1a\u5982\u4e0b\u8868\u6240\u793a\uff0cDeepSeek\u5728 <code>7B<\/code> \u548c <code>670B<\/code> \u4e24\u4e2a\u53c2\u6570\u4e0b\uff0c\u7f51\u7edc\u5c42\u6570\u3001\u5b66\u4e60\u7387\u3001\u6279\u91cf\u5927\u5c0f\u7b49\u5747\u6709\u4e0d\u540c\u3002<\/li>\n<\/ul>\n<table>\n<thead>\n<tr>\n<th>\u53c2\u6570<\/th>\n<th>\u5c42\u6570<\/th>\n<th>\u6a21\u578b\u7ef4\u5ea6<\/th>\n<th>\u5934\u6570<\/th>\n<th>\u952e\u5934\u6570<\/th>\n<th>\u4e0a\u4e0b\u6587\u957f\u5ea6<\/th>\n<th>\u5e8f\u5217\u6279\u91cf\u5927\u5c0f<\/th>\n<th>\u5b66\u4e60\u7387<\/th>\n<th>\u8bcd\u5143\u6570<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>70\u4ebf\u53c2\u6570\u6a21\u578b<\/td>\n<td>30<\/td>\n<td>4096<\/td>\n<td>32<\/td>\n<td>32<\/td>\n<td>4096<\/td>\n<td>2304<\/td>\n<td>4.2\u00d710\u207b\u2074<\/td>\n<td>2.0\u4e07\u4ebf<\/td>\n<\/tr>\n<tr>\n<td>670\u4ebf\u53c2\u6570\u6a21\u578b<\/td>\n<td>95<\/td>\n<td>8192<\/td>\n<td>64<\/td>\n<td>8<\/td>\n<td>4096<\/td>\n<td>4608<\/td>\n<td>3.2\u00d710\u207b\u2074<\/td>\n<td>2.0\u4e07\u4ebf<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h5><span class=\"ez-toc-section\" id=\"%E5%BB%B6%E4%BC%B8%E4%BA%86%E8%A7%A3\"><\/span>\u5ef6\u4f38\u4e86\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<p>\u76f8\u5173\u77e5\u8bc6\u70b9\u7684\u8f85\u52a9\u8d44\u6599\u67e5\u8be2\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>\n<p><code>RMSNorm<\/code>\uff1a<code>RMSNorm<\/code> \u662f <code>LayerNorm<\/code> \u7684\u4e00\u4e2a\u7b80\u5355\u53d8\u4f53\uff0c\u6765\u81ea 2019 \u5e74\u7684\u8bba\u6587 Root Mean Square Layer Normalization\uff0c\u88ab T5 \u548c\u5f53\u524d\u6d41\u884c lamma \u6a21\u578b\u6240\u4f7f\u7528\u3002\u5176\u63d0\u51fa\u7684\u52a8\u673a\u662f <code>LayerNorm<\/code> \u8fd0\u7b97\u91cf\u6bd4\u8f83\u5927\uff0c\u6240\u63d0\u51fa\u7684 <code>RMSNorm<\/code> \u6027\u80fd\u548c <code>LayerNorm<\/code> \u76f8\u5f53\uff0c\u4f46\u662f\u53ef\u4ee5\u8282\u77017%\u523064%\u7684\u8fd0\u7b97\u3002<\/p>\n<blockquote>\n<p>\u8d44\u6599\u6765\u6e90\uff1a<a href=\"https:\/\/blog.csdn.net\/m0_57102661\/article\/details\/142741221\">CSDN\uff1a\u8be6\u89e3\u4e09\u79cd\u5e38\u7528\u6807\u51c6\u5316\uff1aBatch Norm &amp; Layer Norm &amp; RMSNorm<\/a><\/p>\n<\/blockquote>\n<\/li>\n<li>\n<p><code>SwiGLU<\/code>\uff1a<code>swiGLU<\/code> \u662f\u5bf9 <code>GLU<\/code> \u7684\u6539\u8fdb\u4e00\u79cd\u6fc0\u6d3b\u51fd\u6570\u6a21\u5757\uff0c\u901a\u8fc7\u5f15\u5165 <code>Swish<\/code> \u6fc0\u6d3b\u51fd\u6570\u6765\u63d0\u4f9b\u66f4\u5e73\u6ed1\u7684\u975e\u7ebf\u6027\u6620\u5c04\uff0c\u6709\u52a9\u4e8e\u63d0\u5347\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u7684\u8868\u73b0\uff0c\u5c24\u5176\u662f\u5728 <code>Transformer<\/code> \u67b6\u6784\u4e2d\u3002<\/p>\n<blockquote>\n<p>\u8d44\u6599\u6765\u6e90\uff1a<a href=\"https:\/\/blog.csdn.net\/wm199\/article\/details\/144476653\">\u6fc0\u6d3b\u51fd\u6570-swiGLU<\/a><\/p>\n<\/blockquote>\n<\/li>\n<li>\n<p><code>\u5206\u7ec4\u67e5\u8be2\u6ce8\u610f\u529b\u673a\u5236\uff08GQA\uff09<\/code>\uff1a<code>Grouped Multi-Query Attention<\/code> \u662f\u4e00\u79cd <code>Multi-Head Attention<\/code> \u7684\u53d8\u4f53\uff0c\u5b83\u662f\u5728\u8ba1\u7b97\u6210\u672c\u548c\u751f\u6210\u7ed3\u679c\u8d28\u91cf\u4e4b\u95f4\u7684\u4e00\u79cd\u6298\u4e2d\u65b9\u6cd5\u3002<\/p>\n<blockquote>\n<p>\u8d44\u6599\u6765\u6e90\uff1a<a href=\"https:\/\/blog.csdn.net\/coolyoung520\/article\/details\/143925684\">CSDN\uff1a\u4e09\u79cd\u6ce8\u610f\u529b\u673a\u5236\uff1a \u591a\u5934\u6ce8\u610f\u529b\u3001\u5206\u7ec4\u591a\u67e5\u8be2\u4e0e\u591a\u67e5\u8be2\u6ce8\u610f\u529b\uff08Multi-Head , Grouped Multi-Query , Multi-Query \uff09\uff1a\u56fe\u4e66\u9986\u7c7b\u6bd4\u89e3\u8bfb\u4e09\u79cd\u6ce8\u610f\u529b\u673a\u5236\u7684\u533a\u522b\u4e0e\u4f18\u52a3<\/a><\/p>\n<\/blockquote>\n<\/li>\n<li>\n<p><code>\u4f59\u5f26\u9000\u706b\u7b97\u6cd5<\/code>\uff1a<code>CosineAnnealing<\/code> \u662f\u4e00\u79cd <code>\u5b66\u4e60\u7387\u8870\u51cf<\/code> \u7684\u7b97\u6cd5\uff0c\u5b83\u901a\u8fc7 <code>\u4f59\u5f26\u51fd\u6570<\/code> \u6765\u8c03\u6574\u5b66\u4e60\u7387\uff0c\u4ece\u800c\u8fbe\u5230 <code>\u5b66\u4e60\u7387\u968f\u8fed\u4ee3\u6b21\u6570\u589e\u52a0\u800c\u8870\u51cf<\/code> \u7684\u6548\u679c\u3002<\/p>\n<blockquote>\n<p>\u8d44\u6599\u6765\u6e90\uff1a<a href=\"https:\/\/blog.csdn.net\/weixin_35848967\/article\/details\/108493217\">\u5b66\u4e60\u7387\u8870\u51cf\u4e4b\u4f59\u5f26\u9000\u706b(CosineAnnealing\uff09<\/a><\/p>\n<\/blockquote>\n<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"23_%E8%B6%85%E5%8F%82%E6%95%B0\"><\/span>2.3 \u8d85\u53c2\u6570<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-5\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>A multi-step learning rate scheduler is employed during pre-training instead of the typical<br \/>\ncosine scheduler. Specifically, the learning rate of the model reaches its maximum value after<br \/>\n2000 warmup steps, and then decreases to 31.6% of the maximum value after processing 80% of<br \/>\nthe training tokens. It further reduces to 10% of the maximum value after 90% of the tokens.<br \/>\nThe gradient clipping during the training phase is set to 1.0<\/p>\n<p>Based on our empirical findings, we observed that despite differences in the loss reduction trend during training, the final performance using <strong>a multi-step learning rate scheduler is essentially consistent with that of a cosine scheduler<\/strong>, as shown in Figure 1(a). When adjusting the training scale while keeping the model size fixed, the multi-step learning rate scheduler allows for the reuse of training from the first phase, offering a unique convenience for continual training. Therefore, we chose the multi-step learning rate scheduler as our default setting. We also demonstrate in Figure 1(b) that <strong>adjusting the proportions of different stages in the multi-step learning rate scheduler can yield slightly better performance<\/strong>. However, for the sake of balancing reuse ratios in continual training and model performance, we opted for the aforementioned distribution of 80%, 10%, and 10% for the three stages respectively.<\/p>\n<\/blockquote>\n<p><a href=\"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/table3_multistep.png\" data-fancybox=\"images\" data-fancybox=\"gallery\"><img decoding=\"async\" src=\"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/table3_multistep.png\" alt=\"\" \/><\/a><\/p>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-5\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>\u9884\u8bad\u7ec3\u671f\u95f4\u91c7\u7528\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\uff0c\u800c\u975e\u5178\u578b\u7684\u4f59\u5f26\u9000\u706b\u8c03\u5ea6\u5668\u3002\u5177\u4f53\u800c\u8a00\uff0c\u6a21\u578b\u7684\u5b66\u4e60\u7387\u5728\u7ecf\u8fc7 <code>2000<\/code> \u4e2a\u9884\u70ed\u6b65\u9aa4\u540e\u8fbe\u5230\u6700\u5927\u503c\uff0c\u968f\u540e\u5728\u5904\u7406\u5b8c <code>80%<\/code> \u7684\u8bad\u7ec3\u8bcd\u5143\u540e\u964d\u81f3\u6700\u5927\u503c\u7684 <code>31.6%<\/code> \u3002\u5728\u5904\u7406\u5b8c <code>90%<\/code> \u7684\u8bcd\u5143\u540e\uff0c\u5b66\u4e60\u7387\u8fdb\u4e00\u6b65\u964d\u81f3\u6700\u5927\u503c\u7684 <code>10%<\/code>\u3002\u8bad\u7ec3\u9636\u6bb5\u7684\u68af\u5ea6\u88c1\u526a\u8bbe\u7f6e\u4e3a <code>1.0<\/code> \u3002<\/p>\n<p>\u57fa\u4e8e\u6211\u4eec\u7684\u5b9e\u8bc1\u7814\u7a76\u7ed3\u679c\uff0c\u6211\u4eec\u6ce8\u610f\u5230\uff0c\u5c3d\u7ba1\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u4e0d\u540c\u9636\u6bb5\u635f\u5931\u51cf\u5c11\u8d8b\u52bf\u6709\u6240\u4e0d\u540c\uff0c\u4f46<strong>\u4f7f\u7528\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\u5f97\u5230\u7684\u6700\u7ec8\u6027\u80fd\u4e0e\u4f59\u5f26\u8c03\u5ea6\u5668\u57fa\u672c\u4e00\u81f4<\/strong>\uff0c\u5982\u56fe 1\uff08a\uff09\u6240\u793a\u3002\u5728\u4fdd\u6301\u6a21\u578b\u89c4\u6a21\u4e0d\u53d8\u7684\u60c5\u51b5\u4e0b\u8c03\u6574\u8bad\u7ec3\u89c4\u6a21\u65f6\uff0c\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\u5141\u8bb8\u91cd\u590d\u5229\u7528\u7b2c\u4e00\u9636\u6bb5\u7684\u8bad\u7ec3\u6210\u679c\uff0c\u4e3a\u6301\u7eed\u8bad\u7ec3\u63d0\u4f9b\u4e86\u72ec\u7279\u7684\u4fbf\u5229\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u9009\u62e9\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\u4f5c\u4e3a\u9ed8\u8ba4\u8bbe\u7f6e\u3002\u6211\u4eec\u8fd8\u5728\u56fe 1\uff08b\uff09\u4e2d\u5c55\u793a\u4e86\uff0c<strong>\u8c03\u6574\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\u4e2d\u4e0d\u540c\u9636\u6bb5\u7684\u6bd4\u4f8b<\/strong>\uff0c\u53ef\u4f7f<strong>\u6027\u80fd\u7565\u6709\u63d0\u5347<\/strong>\u3002\u7136\u800c\uff0c\u4e3a\u4e86\u5e73\u8861\u6301\u7eed\u8bad\u7ec3\u4e2d\u7684\u590d\u7528\u7387\u548c\u6a21\u578b\u6027\u80fd\uff0c\u6211\u4eec\u9009\u62e9\u4e86\u4e0a\u8ff0\u4e09\u4e2a\u9636\u6bb5\u5206\u522b\u4e3a 80%\u300110% \u548c 10% \u7684\u6bd4\u4f8b\u5206\u914d\u3002<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-5\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<ul>\n<li>\u76f8\u6bd4\u8f83LlaMa\uff0cDeepSeek\u91c7\u7528\u7684\u662f <code>\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668<\/code>\u3002<\/li>\n<li>\u5b83\u662f\u4e00\u79cd\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\uff0c\u901a\u8fc7\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u6574\uff0c\u4ece\u800c\u8fbe\u5230\u5b66\u4e60\u7387\u968f\u8fed\u4ee3\u6b21\u6570\u589e\u52a0\u800c\u8870\u51cf\u7684\u6548\u679c\u3002<\/li>\n<li>\u5176\u6027\u80fd\u4e0e\u5e38\u89c4\u7684\u4f59\u5f26\u8c03\u5ea6\u5668\u57fa\u672c\u4e00\u81f4\uff0c\u901a\u8fc7\u8c03\u6574\u591a\u6b65\u5b66\u4e60\u7387\u8c03\u5ea6\u5668\u4e2d\u4e0d\u540c\u9636\u6bb5\u7684\u6bd4\u4f8b\uff0c\u53ef\u4ee5\u7565\u5fae\u63d0\u5347\u6027\u80fd\u3002<\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"24_%E5%9F%BA%E7%A1%80%E8%AE%BE%E6%96%BD\"><\/span>2.4 \u57fa\u7840\u8bbe\u65bd<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-6\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>We use an efficient and light-weight training framework named <strong>HAI-LLM<\/strong> (High-flyer, 2023) to train and evaluate large language models. Data parallelism, tensor parallelism, sequence parallelism, and 1F1B pipeline parallelism are integrated into this framework as done in Megatron (Korthikanti et al., 2023; Narayanan et al., 2021; Shoeybi et al., 2019). We also leverage the flash attention (Dao, 2023; Dao et al., 2022) technique to improve hardware utilization. ZeRO-1 (Rajbhandari et al., 2020) is exploited to partition optimizer states over data parallel ranks. Efforts are also made to overlap computation and communication to minimize additional waiting overhead, including the backward procedure of the last micro-batch and reduce-scatter operation in ZeRO-1, and GEMM computation and all-gather\/reduce-scatter in sequence parallel. Some layers\/operators are fused to speed up training, including LayerNorm, GEMM whenever possible, and Adam updates. To improve model training stability, we train the model in bf16 precision but accumulate gradients in fp32 precision. In-place cross-entropy is performed to reduce GPU memory consumption, i.e.: we convert bf16 logits to fp32 precision on the fly in the cross-entropy CUDA kernel (instead of converting it beforehand in HBM), calculate the corresponding bf16 gradient, and overwrite logits with its gradient.<\/p>\n<p>Model weights and optimizer states are saved every 5 minutes asynchronously, which means we will lose no more than 5 minutes of training in the worst case of occasional hardware or network failures. These temporary model checkpoints are cleared up regularly to avoid consuming too much storage space. We also support resuming training from a different 3D parallel configuration to cope with dynamic changes in computing cluster load.<\/p>\n<p>As for evaluation, we employ vLLM (Kwon et al., 2023) in generative tasks, and continuous batching in non-generative tasks to avoid manual batch size tuning and reduce token padding.<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-6\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<blockquote>\n<p>\u6211\u4eec\u4f7f\u7528\u4e00\u4e2a\u540d\u4e3a <code>HAI-LLM<\/code>\uff08High &#8211; flyer\uff0c2023\uff09\u7684\u9ad8\u6548\u8f7b\u91cf\u7ea7\u8bad\u7ec3\u6846\u67b6\u6765\u8bad\u7ec3\u548c\u8bc4\u4f30\u5927\u8bed\u8a00\u6a21\u578b\u3002\u4e0e Megatron\uff08Korthikanti \u7b49\u4eba\uff0c2023\uff1bNarayanan \u7b49\u4eba\uff0c2021\uff1bShoeybi \u7b49\u4eba\uff0c2019\uff09\u4e00\u6837\uff0c<strong>\u6570\u636e\u5e76\u884c\u3001\u5f20\u91cf\u5e76\u884c\u3001\u5e8f\u5217\u5e76\u884c<\/strong>\u548c 1F1B \u6d41\u6c34\u7ebf\u5e76\u884c\u90fd\u96c6\u6210\u5230\u4e86\u8fd9\u4e2a\u6846\u67b6\u4e2d\u3002\u6211\u4eec\u8fd8\u5229\u7528\u4e86 <code>FlashAttention<\/code>\uff08Dao\uff0c2023\uff1bDao \u7b49\u4eba\uff0c2022\uff09\u6280\u672f\u6765\u63d0\u9ad8\u786c\u4ef6\u5229\u7528\u7387\u3002\u91c7\u7528 <code>ZeRO-1<\/code>\uff08Rajbhandari \u7b49\u4eba\uff0c2020\uff09\u5728\u6570\u636e\u5e76\u884c\u7b49\u7ea7\u4e0a\u5bf9\u4f18\u5316\u5668\u72b6\u6001\u8fdb\u884c\u5206\u533a\u3002\u6211\u4eec\u8fd8\u52aa\u529b\u8ba9\u8ba1\u7b97\u548c\u901a\u4fe1\u91cd\u53e0\uff0c\u4ee5\u5c3d\u91cf\u51cf\u5c11\u989d\u5916\u7684\u7b49\u5f85\u5f00\u9500\uff0c\u5305\u62ec\u6700\u540e\u4e00\u4e2a\u5fae\u6279\u6b21\u7684\u53cd\u5411\u4f20\u64ad\u8fc7\u7a0b\u4ee5\u53ca ZeRO-1 \u4e2d\u7684\u89c4\u7ea6-\u6563\u5c04\u64cd\u4f5c\uff0c\u8fd8\u6709\u5e8f\u5217\u5e76\u884c\u4e2d\u7684<code>\u901a\u7528\u77e9\u9635\u4e58\u6cd5\uff08GEMM\uff09\u8ba1\u7b97<\/code>\u4e0e\u5168\u6536\u96c6\/\u89c4\u7ea6-\u6563\u5c04\u64cd\u4f5c\u3002\u4e3a\u52a0\u5feb\u8bad\u7ec3\u901f\u5ea6\uff0c\u5bf9\u4e00\u4e9b\u5c42 \/ \u64cd\u4f5c\u7b26\u8fdb\u884c\u4e86\u878d\u5408\uff0c\u5305\u62ec\u5c3d\u53ef\u80fd\u5bf9\u5c42<code>\u5f52\u4e00\u5316\uff08LayerNorm\uff09<\/code>\u3001<code>\u901a\u7528\u77e9\u9635\u4e58\u6cd5\uff08GEMM\uff09<\/code>\u4ee5\u53ca <code>Adam \u66f4\u65b0\u64cd\u4f5c<\/code>\u3002\u4e3a\u63d0\u9ad8\u6a21\u578b\u8bad\u7ec3\u7684\u7a33\u5b9a\u6027\uff0c\u6211\u4eec\u4ee5 <code>bf16<\/code> \u7cbe\u5ea6\u8bad\u7ec3\u6a21\u578b\uff0c\u4f46\u4ee5 <code>fp32<\/code> \u7cbe\u5ea6\u7d2f\u79ef\u68af\u5ea6\u3002\u901a\u8fc7\u6267\u884c\u539f\u5730\u4ea4\u53c9\u71b5\u8ba1\u7b97\u6765\u51cf\u5c11 <code>GPU<\/code> \u5185\u5b58\u6d88\u8017\uff0c\u5373\uff1a\u6211\u4eec\u5728\u4ea4\u53c9\u71b5 <code>CUDA<\/code> \u5185\u6838\u4e2d\u5373\u65f6\u5c06 <code>bf16<\/code> \u683c\u5f0f\u7684\u5bf9\u6570\u51e0\u7387\uff08logits\uff09\u8f6c\u6362\u4e3a <code>fp32<\/code> \u7cbe\u5ea6\uff08\u800c\u4e0d\u662f\u4e8b\u5148\u5728<code>\u9ad8\u5e26\u5bbd\u5185\u5b58\uff08HBM\uff09<\/code>\u4e2d\u8fdb\u884c\u8f6c\u6362\uff09\uff0c\u8ba1\u7b97\u76f8\u5e94\u7684 <code>bf16<\/code> \u68af\u5ea6\uff0c\u5e76\u7528\u68af\u5ea6\u8986\u76d6\u5bf9\u6570\u51e0\u7387\u3002<\/p>\n<p>\u6a21\u578b\u6743\u91cd\u548c\u4f18\u5316\u5668\u72b6\u6001\u6bcf 5 \u5206\u949f\u5f02\u6b65\u4fdd\u5b58\u4e00\u6b21\uff0c\u8fd9\u610f\u5473\u7740\u5728\u5076\u5c14\u51fa\u73b0\u786c\u4ef6\u6216\u7f51\u7edc\u6545\u969c\u7684\u6700\u574f\u60c5\u51b5\u4e0b\uff0c\u6211\u4eec\u635f\u5931\u7684\u8bad\u7ec3\u8fdb\u5ea6\u4e0d\u4f1a\u8d85\u8fc7 5 \u5206\u949f\u3002\u8fd9\u4e9b\u4e34\u65f6\u7684\u6a21\u578b\u68c0\u67e5\u70b9\u4f1a\u5b9a\u671f\u6e05\u7406\uff0c\u4ee5\u907f\u514d\u5360\u7528\u8fc7\u591a\u5b58\u50a8\u7a7a\u95f4\u3002\u6211\u4eec\u8fd8\u652f\u6301\u4ece\u4e0d\u540c\u7684\u4e09\u7ef4\u5e76\u884c\u914d\u7f6e\u6062\u590d\u8bad\u7ec3\uff0c\u4ee5\u5e94\u5bf9\u8ba1\u7b97\u96c6\u7fa4\u8d1f\u8f7d\u7684\u52a8\u6001\u53d8\u5316\u3002<\/p>\n<\/blockquote>\n<h5><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-6\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h5>\n<p>\uff08\u4ee5\u4e0a\u5185\u5bb9\u8fc7\u4e8e\u4e13\u4e1a\uff0c\u5bf9\u4e8e\u534a\u8def\u51fa\u5bb6\u7684\u6211\u6765\u8bf4\uff0c\u7406\u89e3\u5176\u539f\u7406\u8d85\u7eb2\u4e86\uff09<br \/>\n\u672c\u7740\u6c42\u77e5\u7684\u5fc3\u6001\uff0c\u6211\u5728Deepseek\u7684\u6240\u5c5e\u516c\u53f8(\u5e7b\u65b9)\uff0c\u67e5\u8be2\u5230\u76f8\u5173\u7684\u8d44\u6599\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li><code>HAI-LLM<\/code> \u662f<code>\u5e7b\u65b9-\u6df1\u5ea6\u6c42\u7d22\uff08Deepseek\uff09<\/code>\u7814\u53d1\u7684\u4e00\u6b3e\u6df1\u5ea6\u5b66\u4e60\u8bad\u7ec3\u5de5\u5177\u3002<\/li>\n<li><code>HAI-LLM<\/code> \u5b9e\u73b0\u4e86\u56db\u79cd\u5e76\u884c\u8bad\u7ec3\u65b9\u5f0f\uff1a<strong>ZeRO \u652f\u6301\u7684\u6570\u636e\u5e76\u884c<\/strong>\u3001<strong>\u6d41\u6c34\u7ebf\u5e76\u884c<\/strong>\u3001<strong>\u5f20\u91cf\u5207\u7247\u6a21\u578b\u5e76\u884c<\/strong>\u548c<strong>\u5e8f\u5217\u5e76\u884c<\/strong>\u3002<\/li>\n<\/ul>\n<blockquote>\n<p>\u5907\u6ce8\uff1a\u8fd1\u671f\u82f1\u4f1f\u8fbe\u80a1\u4ef7\u66b4\u8dcc\uff0c\u53ef\u80fd\u5c31\u662f\u7531\u4e8eDeepseek\u7684\u8fd9\u4e2a\u8bad\u7ec3\u65b9\u5f0f\uff0c\u7ed5\u8fc7\u4e86\u82f1\u4f1f\u8fbe\u7684CUDA\u62a4\u57ce\u6cb3\u3002<\/p>\n<p>\u8d44\u6599\u6765\u6e90\uff1a<a href=\"https:\/\/www.high-flyer.cn\/en\/blog\/hai-llm\/\">HAI-LLM\uff1a\u9ad8\u6548\u4e14\u8f7b\u91cf\u7684\u5927\u6a21\u578b\u8bad\u7ec3\u5de5\u5177<\/a><\/p>\n<\/blockquote>\n<h3><span class=\"ez-toc-section\" id=\"3_%E7%BC%A9%E6%94%BE%E5%AE%9A%E5%BE%8BScaling_Laws\"><\/span>3. \u7f29\u653e\u5b9a\u5f8b(Scaling Laws)<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E5%8E%9F%E6%96%87-7\"><\/span>\u8bba\u6587\u539f\u6587<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote>\n<p>Research on scaling laws (Hestness et al., 2017) predates the emergence of large language models. Scaling laws (Henighan et al., 2020; Hoffmann et al., 2022; Kaplan et al., 2020) suggest that model performance can be predictably improved with increases in compute budget c , model scale N and data scale D . When model scale N is represented by model parameters and data scale D by the number of tokens, c can be approximated as <code class=\"katex-inline\">C=6 N D<\/code> . Therefore, how to optimize the allocation between model and data scales when increasing the compute budget is also a crucial research objective in scaling laws.<\/p>\n<p>&#8230;\uff08\u4e2d\u95f4\u90e8\u5206\u7701\u7565\uff09<\/p>\n<p>We then study the scaling laws of the model and data scales. To reduce experimental costs and fitting difficulties, we adopted the IsoFLOP profile approach from Chinchilla (Hoffmann et al., 2022) to fit the scaling curve. To represent the model scale more accurately, we utilized a new model scale representation, non-embedding FLOPs\/token M , replacing the earlier-used model parameters N , and substituted the approximate compute budget formula <code class=\"katex-inline\">C=6 N D<\/code> with the more precise <code class=\"katex-inline\">C=M D<\/code> . The experimental results provided insights into the optimal model\/data scaling-up allocation strategy and performance predictions, and also accurately forecasted the expected performance of DeepSeek LLM 7B and 67B models.<\/p>\n<p>Additionally, in the process of exploring scaling laws, the data we used underwent multiple iterations, continually improving in quality. We attempted to fit the scaling curve on various datasets and found that the data quality significantly influences the optimal model\/data scalingup allocation strategy. The higher the data quality, the more the increased compute budget should be allocated to model scaling. This implies that high-quality data can drive the training of larger models given the same data scale. The differences in the optimal model\/data scaling-up allocation strategy may also serve as an indirect approach to assess the quality of data. We will continue to pay close attention to the changes in data quality and its impact on scaling laws, and provide more analysis in future works.<\/p>\n<p>In summary, our contributions and findings in scaling laws can be summarized as follows:<br \/>\n\u2022 We established the scaling laws for hyperparameters, providing an empirical framework for determining the optimal hyperparameters.<br \/>\n\u2022 Instead of model parameters N , we adopt non-embedding FLOPs\/token M to represent the model scale, leading to a more accurate optimal model\/data scaling-up allocation strategy and a better prediction of generalization loss for large-scale models.<br \/>\n\u2022 The quality of pre-training data impacts the optimal model\/data scaling-up allocation strategy. The higher the data quality, the more the increased compute budget should be allocated to model scaling.<\/p>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%BF%BB%E8%AF%91-7\"><\/span>\u8bba\u6587\u7ffb\u8bd1<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote>\n<p>\u5173\u4e8e\u7f29\u653e\u5b9a\u5f8b\u7684\u7814\u7a76\uff08\u8d6b\u65af\u5185\u65af\u7b49\u4eba\uff0c2017 \u5e74\uff09\u65e9\u4e8e\u5927\u8bed\u8a00\u6a21\u578b\u7684\u51fa\u73b0\u3002\u7f29\u653e\u5b9a\u5f8b\uff08\u4ea8\u5c3c\u6839\u7b49\u4eba\uff0c2020 \u5e74\uff1b\u970d\u592b\u66fc\u7b49\u4eba\uff0c2022 \u5e74\uff1b\u5361\u666e\u5170\u7b49\u4eba\uff0c2020 \u5e74\uff09\u8868\u660e\uff0c\u968f\u7740<strong>\u8ba1\u7b97\u8d44\u6e90\u9884\u7b97 <code class=\"katex-inline\">C<\/code><\/strong>\u3001<strong>\u6a21\u578b\u89c4\u6a21 <code class=\"katex-inline\">N<\/code><\/strong> \u548c <strong>\u6570\u636e\u89c4\u6a21 <code class=\"katex-inline\">D<\/code><\/strong> \u7684\u589e\u52a0\uff0c\u6a21\u578b\u6027\u80fd\u6709\u671b\u5f97\u5230\u53ef\u9884\u6d4b\u7684\u63d0\u5347\u3002\u5f53\u6a21\u578b\u89c4\u6a21 <code class=\"katex-inline\">N<\/code> \u7528\u6a21\u578b\u53c2\u6570\u8868\u793a\uff0c\u6570\u636e\u89c4\u6a21 <code class=\"katex-inline\">D<\/code> \u7528\u4ee4\u724c\u6570\u91cf\u8868\u793a\u65f6\uff0c<code class=\"katex-inline\">C<\/code> \u53ef\u8fd1\u4f3c\u8868\u793a\u4e3a <strong><code class=\"katex-inline\">C=6 N D<\/code><\/strong>\u3002\u56e0\u6b64\uff0c\u5728\u589e\u52a0\u8ba1\u7b97\u8d44\u6e90\u9884\u7b97\u65f6\uff0c\u5982\u4f55\u4f18\u5316\u6a21\u578b\u4e0e\u6570\u636e\u89c4\u6a21\u4e4b\u95f4\u7684\u5206\u914d\uff0c\u4e5f\u662f\u7f29\u653e\u5b9a\u5f8b\u7814\u7a76\u4e2d\u7684\u4e00\u4e2a\u5173\u952e\u76ee\u6807\u3002<\/p>\n<p>&#8230;\uff08\u4e2d\u95f4\u90e8\u5206\u7701\u7565\uff09<\/p>\n<p>\u63a5\u7740\uff0c\u6211\u4eec\u7814\u7a76<strong>\u6a21\u578b\u89c4\u6a21<\/strong>\u548c<strong>\u6570\u636e\u89c4\u6a21<\/strong>\u7684\u7f29\u653e\u5b9a\u5f8b\u3002\u4e3a\u964d\u4f4e\u5b9e\u9a8c\u6210\u672c\u4e0e\u62df\u5408\u96be\u5ea6\uff0c\u6211\u4eec\u91c7\u7528\u4e86 Chinchilla \u8bba\u6587\uff08\u970d\u592b\u66fc\u7b49\u4eba\uff0c2022 \u5e74\uff09\u4e2d\u7684\u7b49\u6d6e\u70b9\u8fd0\u7b97\u6b21\u6570\uff08IsoFLOP\uff09\u66f2\u7ebf\u6cd5\u6765\u62df\u5408\u7f29\u653e\u66f2\u7ebf\u3002\u4e3a\u66f4\u51c6\u786e\u5730\u8868\u793a\u6a21\u578b\u89c4\u6a21\uff0c\u6211\u4eec\u4f7f\u7528\u4e86<strong>\u4e00\u79cd\u65b0\u7684\u6a21\u578b\u89c4\u6a21\u8868\u793a\u65b9\u5f0f<\/strong> \u2014\u2014 \u6bcf\u4e2a\u8bcd\u5143\u7684\u975e\u5d4c\u5165\u6d6e\u70b9\u8fd0\u7b97\u6b21\u6570 <code class=\"katex-inline\">M<\/code>\uff0c\u4ee5\u53d6\u4ee3\u4e4b\u524d\u4f7f\u7528\u7684\u6a21\u578b\u53c2\u6570 <code class=\"katex-inline\">N<\/code>\uff0c\u5e76\u5bf9\u8fd1\u4f3c\u8ba1\u7b97\u9884\u7b97\u516c\u5f0f <code class=\"katex-inline\">C=6 N D<\/code> \u8fdb\u884c\u4e86\u66ff\u6362\uff0c\u5c06\u5176\u66ff\u6362\u4e3a\u66f4\u7cbe\u786e\u7684<strong>\u516c\u5f0f <code class=\"katex-inline\">C=M D<\/code><\/strong>\u3002\u5b9e\u9a8c\u7ed3\u679c\u4e3a\u6700\u4f18\u7684\u6a21\u578b\/\u6570\u636e\u6269\u5bb9\u5206\u914d\u7b56\u7565\u53ca\u6027\u80fd\u9884\u6d4b\u63d0\u4f9b\u4e86\u6df1\u523b\u89c1\u89e3\uff0c\u8fd8\u51c6\u786e\u9884\u6d4b\u4e86 DeepSeek LLM 70 \u4ebf\u53c2\u6570\u548c 670 \u4ebf\u53c2\u6570\u6a21\u578b\u7684\u9884\u671f\u6027\u80fd\u3002<\/p>\n<p>\u6b64\u5916\uff0c\u5728\u63a2\u7d22\u7f29\u653e\u5b9a\u5f8b\u7684\u8fc7\u7a0b\u4e2d\uff0c\u6211\u4eec\u6240\u4f7f\u7528\u7684\u6570\u636e\u5386\u7ecf\u591a\u6b21\u8fed\u4ee3\uff0c\u8d28\u91cf\u4e0d\u65ad\u63d0\u5347\u3002\u6211\u4eec\u5c1d\u8bd5\u5728\u5404\u79cd\u6570\u636e\u96c6\u4e0a\u62df\u5408\u7f29\u653e\u66f2\u7ebf\uff0c\u53d1\u73b0<strong>\u6570\u636e\u8d28\u91cf<\/strong>\u5bf9\u6700\u4f18\u7684\u6a21\u578b \/ \u6570\u636e\u6269\u5bb9\u5206\u914d\u7b56\u7565\u6709\u663e\u8457\u5f71\u54cd\u3002\u6570\u636e\u8d28\u91cf\u8d8a\u9ad8\uff0c\u589e\u52a0\u7684\u8ba1\u7b97\u9884\u7b97\u5c31\u8d8a\u5e94\u5206\u914d\u7ed9\u6a21\u578b\u6269\u5bb9\u3002\u8fd9\u610f\u5473\u7740<strong>\u5728\u76f8\u540c\u7684\u6570\u636e\u89c4\u6a21\u4e0b\uff0c\u9ad8\u8d28\u91cf\u6570\u636e\u80fd\u591f\u63a8\u52a8\u66f4\u5927\u89c4\u6a21\u6a21\u578b\u7684\u8bad\u7ec3<\/strong>\u3002\u6700\u4f18\u6a21\u578b \/ \u6570\u636e\u6269\u5bb9\u5206\u914d\u7b56\u7565\u7684\u5dee\u5f02\uff0c\u4e5f\u53ef\u4f5c\u4e3a\u4e00\u79cd\u95f4\u63a5\u8bc4\u4f30\u6570\u636e\u8d28\u91cf\u7684\u65b9\u6cd5\u3002\u6211\u4eec\u5c06\u7ee7\u7eed\u5bc6\u5207\u5173\u6ce8\u6570\u636e\u8d28\u91cf\u7684\u53d8\u5316\u53ca\u5176\u5bf9\u7f29\u653e\u5b9a\u5f8b\u7684\u5f71\u54cd\uff0c\u5e76\u5728\u672a\u6765\u7684\u7814\u7a76\u4e2d\u63d0\u4f9b\u66f4\u591a\u5206\u6790\u3002<\/p>\n<p>\u7efc\u4e0a\u6240\u8ff0\uff0c\u6211\u4eec\u5728\u7f29\u653e\u5b9a\u5f8b\u65b9\u9762\u7684\u8d21\u732e\u548c\u53d1\u73b0\u53ef\u603b\u7ed3\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>\u6211\u4eec\u5efa\u7acb\u4e86<strong>\u8d85\u53c2\u6570\u7684\u7f29\u653e\u5b9a\u5f8b<\/strong>\uff0c\u4e3a\u786e\u5b9a\u6700\u4f18\u8d85\u53c2\u6570\u63d0\u4f9b\u4e86\u4e00\u4e2a\u5b9e\u8bc1\u6846\u67b6\u3002<\/li>\n<li>\u6211\u4eec\u91c7\u7528\u6bcf\u4e2a\u8bcd\u5143\u7684\u975e\u5d4c\u5165\u6d6e\u70b9\u8fd0\u7b97\u6b21\u6570 <code class=\"katex-inline\">M<\/code> \u6765\u4ee3\u66ff\u6a21\u578b\u53c2\u6570 <code class=\"katex-inline\">N<\/code> \u8868\u793a\u6a21\u578b\u89c4\u6a21\uff0c\u8fd9\u5e26\u6765\u4e86<strong>\u66f4\u51c6\u786e\u7684\u6700\u4f18\u6a21\u578b \/ \u6570\u636e\u6269\u5bb9\u5206\u914d\u7b56\u7565<\/strong>\uff0c\u5e76\u4e14\u80fd\u66f4\u597d\u5730\u9884\u6d4b\u5927\u89c4\u6a21\u6a21\u578b\u7684\u6cdb\u5316\u635f\u5931\u3002<\/li>\n<li>\u9884\u8bad\u7ec3\u6570\u636e\u7684\u8d28\u91cf\u4f1a\u5f71\u54cd\u6700\u4f18\u7684\u6a21\u578b \/ \u6570\u636e\u6269\u5bb9\u5206\u914d\u7b56\u7565\u3002<strong>\u6570\u636e\u8d28\u91cf\u8d8a\u9ad8\uff0c\u589e\u52a0\u7684\u8ba1\u7b97\u9884\u7b97\u5c31\u8d8a\u5e94\u5206\u914d\u7ed9\u6a21\u578b\u6269\u5bb9<\/strong>\u3002<\/li>\n<\/ul>\n<\/blockquote>\n<h4><span class=\"ez-toc-section\" id=\"%E8%AE%BA%E6%96%87%E7%90%86%E8%A7%A3-7\"><\/span>\u8bba\u6587\u7406\u89e3<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<ol>\n<li>\n<p>\u8bad\u7ec3AI\u7684&quot;\u4e09\u539f\u8272&quot;\u539f\u7406 \ud83c\udfa8<\/p>\n<ul>\n<li><strong>\u8ba1\u7b97\u529b\u91cf<\/strong> (C)\uff1a\u8bad\u7ec3AI\u9700\u8981\u7684&quot;\u7535\u529b&quot;<\/li>\n<li><strong>\u6a21\u578b\u5927\u5c0f<\/strong> (N)\uff1aAI\u5927\u8111\u7684&quot;\u795e\u7ecf\u5143\u6570\u91cf&quot;<\/li>\n<li><strong>\u6570\u636e\u91cf<\/strong> (D)\uff1a\u7ed9AI\u770b\u7684&quot;\u6559\u6750\u539a\u5ea6&quot;<\/li>\n<li>\u4f20\u7edf\u516c\u5f0f\uff1a<code>\u8bad\u7ec3\u7535\u529b \u2248 6 \u00d7 \u795e\u7ecf\u5143\u6570 \u00d7 \u6559\u6750\u539a\u5ea6<\/code><\/li>\n<\/ul>\n<\/li>\n<li>\n<p>DeepSeek\u7684\u65b0\u53d1\u73b0 \ud83d\udd0d<\/p>\n<ul>\n<li><strong>\u66f4\u806a\u660e\u7684\u6d4b\u91cf\u5c3a<\/strong> \ud83d\udccf<br \/>\n\u6539\u7528<code>\u6bcf\u4e2a\u5b57\u7684\u8ba1\u7b97\u91cf(M)<\/code>\u4ee3\u66ff\u53c2\u6570\u6570\u91cf\uff0c\u5c31\u50cf\u7528&quot;\u8111\u529b\u52b3\u52a8\u91cf&quot;\u6bd4&quot;\u8111\u7ec6\u80de\u6570\u91cf&quot;\u66f4\u80fd\u53cd\u6620\u771f\u5b9e\u667a\u5546<\/li>\n<li><strong>\u6559\u6750\u8d28\u91cf\u51b3\u5b9a\u57f9\u517b\u65b9\u6848<\/strong> \ud83d\udcda<br \/>\n\u2705 \u597d\u6559\u6750(\u9ad8\u8d28\u91cf\u6570\u636e)\uff1a\u53ef\u4ee5\u57f9\u517b&quot;\u5927\u8111\u53d1\u8fbe&quot;\u7684AI\u5b66\u9738<br \/>\n\u274c \u5dee\u6559\u6750\uff1a\u53ea\u80fd\u57f9\u517b&quot;\u6b7b\u8bb0\u786c\u80cc&quot;\u7684AI\u5b66\u6e23<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<h2><span class=\"ez-toc-section\" id=\"%E5%86%85%E5%AE%B9%E5%B0%8F%E7%BB%93%EF%BC%9A\"><\/span>\u5185\u5bb9\u5c0f\u7ed3\uff1a<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<ol>\n<li>\n<p>AI\u6c5f\u6e56\u73b0\u72b6 \ud83c\udf0d<\/p>\n<ul>\n<li><strong>\u95ed\u6e90\u5927\u4f6c<\/strong>\uff1aChatGPT\u7b49\u50cf&quot;\u7c73\u5176\u6797\u9910\u5385&quot;\uff08\u914d\u65b9\u4fdd\u5bc6\uff0c\u5403\u5f97\u8d77\u4f46\u5b66\u4e0d\u4f1a\uff09<\/li>\n<li><strong>\u5f00\u6e90\u65b0\u79c0<\/strong>\uff1aLLaMA\u7cfb\u5217\u5982&quot;\u7f51\u7ea2\u83dc\u8c31&quot;\uff08\u514d\u8d39\u516c\u5f00\uff0c\u4f46\u9700\u8981\u81ea\u5bb6\u53a8\u623f\u591f\u5927\uff09<\/li>\n<li><strong>\u5f53\u524d\u56f0\u5883<\/strong>\uff1a\u5927\u591a\u6570\u5f00\u6e90\u6a21\u578b\u50cf&quot;\u56fa\u5b9a\u83dc\u5f0f\u5957\u9910&quot;\uff08\u53ea\u67097B\/13B\u7b49\u56fa\u5b9a\u53c2\u6570\u7248\u672c\uff09<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>DeepSeek\u7684\u6838\u5fc3\u7a81\u7834 \ud83d\udca5<\/p>\n<\/li>\n<\/ol>\n<table>\n<thead>\n<tr>\n<th><strong>\u7814\u7a76\u91cd\u70b9<\/strong><\/th>\n<th><strong>\u901a\u4fd7\u89e3\u91ca<\/strong><\/th>\n<th><strong>\u5b9e\u9645\u5e94\u7528<\/strong><\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u7f29\u653e\u5b9a\u5f8b<\/td>\n<td>\u627e\u5230AI\u8bad\u7ec3\u7684&quot;\u9ec4\u91d1\u914d\u6bd4\u516c\u5f0f&quot;<\/td>\n<td>\u77e5\u9053\u4f55\u65f6\u8be5\u52a0&quot;\u8111\u5bb9\u91cf&quot;\u6216&quot;\u8bfb\u4e66\u91cf&quot;<\/td>\n<\/tr>\n<tr>\n<td>\u52a8\u6001\u5b66\u4e60\u7387<\/td>\n<td>\u50cf\u667a\u80fd\u7535\u996d\u7172\u8c03\u8282\u706b\u5019<\/td>\n<td>\u8bad\u7ec3\u66f4\u7a33\u5b9a\uff0c\u6548\u679c\u66f4\u597d<\/td>\n<\/tr>\n<tr>\n<td>\u6570\u636e\u8d28\u91cf\u68c0\u6d4b<\/td>\n<td>\u7ed9\u8bad\u7ec3\u6570\u636e\u505a&quot;\u4f53\u68c0&quot;<\/td>\n<td>\u81ea\u52a8\u8bc6\u522b\u4f18\u8d28\u5b66\u4e60\u8d44\u6599<\/td>\n<\/tr>\n<tr>\n<td>\u5b89\u5168\u9632\u62a4<\/td>\n<td>\u7ed9AI\u88c5&quot;\u5239\u8f66\u7cfb\u7edf&quot;<\/td>\n<td>\u907f\u514d\u8bf4\u9519\u8bdd\/\u505a\u5371\u9669\u4e8b<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<blockquote>\n<p>\u5907\u6ce8\uff1a\u4ee5\u4e0a\u8bba\u6587\u7406\u89e3\u5185\u5bb9\u4ee5\u53ca\u5185\u5bb9\u5c0f\u7ed3\uff0c\u7531Deepseek-R1\u8f85\u52a9\u751f\u6210\u3002\u8bba\u6587\u7ffb\u8bd1\u90e8\u5206\u7531\u8c46\u5305\u7684PDF\u9605\u8bfb\u7ffb\u8bd1\u529f\u80fd\u5b8c\u6210\u3002<\/p>\n<\/blockquote>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00 \u5728\u751f\u6210\u5f0fAI\u6d6a\u6f6e\u5e2d\u5377\u5168\u7403\u7684\u5f53\u4e0b\uff0c\u56fd [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":37841,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"aside","meta":{"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"default","adv-header-id-meta":"","stick-header-meta":"default","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"set","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":""},"categories":[28],"tags":[68],"class_list":["post-37835","post","type-post","status-publish","format-aside","has-post-thumbnail","hentry","category-blog","tag-68","post_format-post-format-aside"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v27.9 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b - \u4e00\u8d77AI\u6280\u672f<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/17aitech.com\/?p=37835\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835\"},\"author\":{\"name\":\"Dongming\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/#\\\/schema\\\/person\\\/3d23bb6f7f115fcefc9ae7803a691739\"},\"headline\":\"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b\",\"datePublished\":\"2025-02-05T06:05:29+00:00\",\"dateModified\":\"2025-02-05T06:29:59+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835\"},\"wordCount\":2341,\"commentCount\":0,\"image\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2025\\\/02\\\/\u5c01\u9762.jpeg\",\"keywords\":[\"\u5927\u6a21\u578b\"],\"articleSection\":[\"\u535a\u5ba2\u6587\u7ae0\"],\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"CommentAction\",\"name\":\"Comment\",\"target\":[\"https:\\\/\\\/17aitech.com\\\/?p=37835#respond\"]}]},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835\",\"url\":\"https:\\\/\\\/17aitech.com\\\/?p=37835\",\"name\":\"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b - \u4e00\u8d77AI\u6280\u672f\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2025\\\/02\\\/\u5c01\u9762.jpeg\",\"datePublished\":\"2025-02-05T06:05:29+00:00\",\"dateModified\":\"2025-02-05T06:29:59+00:00\",\"author\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/#\\\/schema\\\/person\\\/3d23bb6f7f115fcefc9ae7803a691739\"},\"breadcrumb\":{\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/17aitech.com\\\/?p=37835\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#primaryimage\",\"url\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2025\\\/02\\\/\u5c01\u9762.jpeg\",\"contentUrl\":\"https:\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/2025\\\/02\\\/\u5c01\u9762.jpeg\",\"width\":2200,\"height\":1212},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/?p=37835#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\\\/\\\/17aitech.com\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/#website\",\"url\":\"https:\\\/\\\/17aitech.com\\\/\",\"name\":\"\u4e00\u8d77AI\u6280\u672f\",\"description\":\"\u8ba9AI\u77e5\u8bc6\u89e6\u624b\u53ef\u53ca\",\"alternateName\":\"\u4e00\u8d77AI\u6280\u672f\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/17aitech.com\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\\\/\\\/17aitech.com\\\/#\\\/schema\\\/person\\\/3d23bb6f7f115fcefc9ae7803a691739\",\"name\":\"Dongming\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/member\\\/avatars\\\/238a0b923820dcc5.1732798681.jpg\",\"url\":\"\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/member\\\/avatars\\\/238a0b923820dcc5.1732798681.jpg\",\"contentUrl\":\"\\\/\\\/17aitech.com\\\/wp-content\\\/uploads\\\/member\\\/avatars\\\/238a0b923820dcc5.1732798681.jpg\",\"caption\":\"Dongming\"},\"description\":\"\u89c1\u5929\u5730\uff0c\u89c1\u4f17\u751f\uff0c\u89c1\u81ea\u5df1\u3002\",\"sameAs\":[\"http:\\\/\\\/17aitech.com\"],\"url\":\"https:\\\/\\\/17aitech.com\\\/?page_id=33738&user=1\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b - \u4e00\u8d77AI\u6280\u672f","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/17aitech.com\/?p=37835","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/17aitech.com\/?p=37835#article","isPartOf":{"@id":"https:\/\/17aitech.com\/?p=37835"},"author":{"name":"Dongming","@id":"https:\/\/17aitech.com\/#\/schema\/person\/3d23bb6f7f115fcefc9ae7803a691739"},"headline":"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b","datePublished":"2025-02-05T06:05:29+00:00","dateModified":"2025-02-05T06:29:59+00:00","mainEntityOfPage":{"@id":"https:\/\/17aitech.com\/?p=37835"},"wordCount":2341,"commentCount":0,"image":{"@id":"https:\/\/17aitech.com\/?p=37835#primaryimage"},"thumbnailUrl":"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/\u5c01\u9762.jpeg","keywords":["\u5927\u6a21\u578b"],"articleSection":["\u535a\u5ba2\u6587\u7ae0"],"inLanguage":"zh-Hans","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/17aitech.com\/?p=37835#respond"]}]},{"@type":"WebPage","@id":"https:\/\/17aitech.com\/?p=37835","url":"https:\/\/17aitech.com\/?p=37835","name":"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b - \u4e00\u8d77AI\u6280\u672f","isPartOf":{"@id":"https:\/\/17aitech.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/17aitech.com\/?p=37835#primaryimage"},"image":{"@id":"https:\/\/17aitech.com\/?p=37835#primaryimage"},"thumbnailUrl":"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/\u5c01\u9762.jpeg","datePublished":"2025-02-05T06:05:29+00:00","dateModified":"2025-02-05T06:29:59+00:00","author":{"@id":"https:\/\/17aitech.com\/#\/schema\/person\/3d23bb6f7f115fcefc9ae7803a691739"},"breadcrumb":{"@id":"https:\/\/17aitech.com\/?p=37835#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/17aitech.com\/?p=37835"]}]},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/17aitech.com\/?p=37835#primaryimage","url":"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/\u5c01\u9762.jpeg","contentUrl":"https:\/\/17aitech.com\/wp-content\/uploads\/2025\/02\/\u5c01\u9762.jpeg","width":2200,"height":1212},{"@type":"BreadcrumbList","@id":"https:\/\/17aitech.com\/?p=37835#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/17aitech.com\/"},{"@type":"ListItem","position":2,"name":"\u3010\u8bba\u6587\u7b80\u8bfb\u3011DeepSeek LLM\uff1a\u4ee5\u957f\u8fdc\u4e3b\u4e49\u62d3\u5c55\u5f00\u6e90\u8bed\u8a00\u6a21\u578b"}]},{"@type":"WebSite","@id":"https:\/\/17aitech.com\/#website","url":"https:\/\/17aitech.com\/","name":"\u4e00\u8d77AI\u6280\u672f","description":"\u8ba9AI\u77e5\u8bc6\u89e6\u624b\u53ef\u53ca","alternateName":"\u4e00\u8d77AI\u6280\u672f","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/17aitech.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/17aitech.com\/#\/schema\/person\/3d23bb6f7f115fcefc9ae7803a691739","name":"Dongming","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"\/\/17aitech.com\/wp-content\/uploads\/member\/avatars\/238a0b923820dcc5.1732798681.jpg","url":"\/\/17aitech.com\/wp-content\/uploads\/member\/avatars\/238a0b923820dcc5.1732798681.jpg","contentUrl":"\/\/17aitech.com\/wp-content\/uploads\/member\/avatars\/238a0b923820dcc5.1732798681.jpg","caption":"Dongming"},"description":"\u89c1\u5929\u5730\uff0c\u89c1\u4f17\u751f\uff0c\u89c1\u81ea\u5df1\u3002","sameAs":["http:\/\/17aitech.com"],"url":"https:\/\/17aitech.com\/?page_id=33738&user=1"}]}},"_links":{"self":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts\/37835","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=37835"}],"version-history":[{"count":5,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts\/37835\/revisions"}],"predecessor-version":[{"id":37846,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/posts\/37835\/revisions\/37846"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=\/wp\/v2\/media\/37841"}],"wp:attachment":[{"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=37835"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=37835"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/17aitech.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=37835"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}