Home Esplora Aiuto
Accedi
limour
/
limour-blog.github.io
mirror da https://dl.limour.top/cf/https://github.com/limour-blog/limour-blog.github.io.git
1
0
Forka 0
File Problemi 0
Ramo (Branch): gh-pages
Rami (Branch) Tag
limour-blog.git...
/
training-gpt-from-scratch.html

training-gpt-from-scratch.html 65 KB
Permalink Cronologia Originale

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749 
  1. 

  2. 

  3. <!DOCTYPE html>
    4. 

  4. <html lang="en" data-default-color-scheme=auto>
    5. 

  5. 

  6. 

  7. 

  8. <head><!-- hexo injector head_begin start -->
    9. 

  9. <script defer src="https://api.limour.top/vue/0d2f95c1-755d-436b-adf8-eee12a80ed32/script.js"></script>
    10. 

  10. <!-- hexo injector head_begin end -->
    11. 

  11.   <meta charset="UTF-8">
    12. 

  12.   <link rel="apple-touch-icon" sizes="76x76" href="https://img.limour.top/2023/08/29/64ee07361815a.webp">
    13. 

  13.   <link rel="icon" href="https://img.limour.top/2023/08/29/64ee07361815a.webp">
    14. 

  14.   <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
    15. 

  15.   <meta http-equiv="x-ua-compatible" content="ie=edge">
    16. 

  16.   
    17. 

  17.   <meta name="theme-color" content="#2f4154">
    18. 

  18.   <meta name="author" content="Limour">
    19. 

  19.   <meta name="keywords" content="">
    20. 

  20.   
    21. 

  21.     <meta name="description" content="探索整个过程，从在一台搭载1660Ti显卡的笔记本电脑上构建 Tokenizer，定义带有 RoPE 的 Transformer，一直到训练、保存模型和可视化训练过程。沉浸在从零开始训练 GPT 的旅程中，深入了解每一个步骤。跳入深度学习的世界，释放在你的便携1660Ti笔记本上的强大潜能。">
    22. 

  22.   
    23. 

  23.   
    24. 

  24.   
    25. 

  25.   <title>【探索】从零开始训练 GPT - Limour&#39;s Blog</title>
    26. 

  26. 

  27.   <link  rel="stylesheet" href="https://jscdn.limour.top/npm/bootstrap@4.6.1/dist/css/bootstrap.min.css" />
    28. 

  28. 

  29. 

  30. 

  31.   <link  rel="stylesheet" href="https://jscdn.limour.top/npm/github-markdown-css@4.0.0/github-markdown.min.css" />
    32. 

  32. 

  33.   <link  rel="stylesheet" href="https://jscdn.limour.top/npm/hint.css@2.7.0/hint.min.css" />
    34. 

  34. 

  35. 

  36. 

  37. <!-- 主题依赖的图标库，不要自行修改 -->
    38. 

  38. <!-- Do not modify the link that theme dependent icons -->
    39. 

  39. 

  40. <link rel="stylesheet" href="//at.alicdn.com/t/c/font_1749284_5i9bdhy70f8.css">
    41. 

  41. 

  42. 

  43. 

  44. <link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">
    45. 

  45. 

  46. 

  47. <link  rel="stylesheet" href="/css/main.css" />
    48. 

  48. 

  49. 

  50.   <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
    51. 

  51.   
    52. 

  52.     <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
    53. 

  53.   
    54. 

  54. 

  55. 

  56. 

  57.   
    58. 

  58. <link rel="stylesheet" href="/theme-inject/custom.css">
    59. 

  59. <link rel="stylesheet" href="/theme-inject/iconfont.css">
    60. 

  60. 

  61. 

  62. 

  63.   <script id="fluid-configs">
    64. 

  64.     var Fluid = window.Fluid || {};
    65. 

  65.     Fluid.ctx = Object.assign({}, Fluid.ctx)
    66. 

  66.     var CONFIG = {"hostname":"hexo.limour.top","root":"/","version":"1.9.8","typing":{"enable":false,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":"§"},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":false,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"https://jscdn.limour.top/gh/Limour-dev/Sakurairo_Vision/load_svg/inload.svg","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":{"measurement_id":null},"tencent":{"sid":null,"cid":null},"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false},"umami":{"src":null,"website_id":null,"domains":null,"start_time":"2024-01-01T00:00:00.000Z","token":null,"api_server":null},"woyaola":null,"cnzz":null},"search_path":"/local-search.xml","include_content_in_search":true};
    67. 

  67. 

  68.     if (CONFIG.web_analytics.follow_dnt) {
    69. 

  69.       var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
    70. 

  70.       Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    71. 

  71.     }
    72. 

  72.   </script>
    73. 

  73.   <script  src="/js/utils.js" ></script>
    74. 

  74.   <script  src="/js/color-schema.js" ></script>
    75. 

  75.   
    76. 

  76. 

  77. 

  78.   
    79. 

  79.   <link rel="canonical" href="https://hexo.limour.top/training-gpt-from-scratch"/>
    80. 

  80. <meta name="generator" content="Hexo 7.1.1"><link rel="alternate" href="/atom.xml" title="Limour's Blog" type="application/atom+xml">
    81. 

  81. <link rel="alternate" href="/rss2.xml" title="Limour's Blog" type="application/rss+xml">
    82. 

  82. </head>
    83. 

  83. 

  84. 

  85. <body>
    86. 

  86.   
    87. 

  87. 

  88.   <header>
    89. 

  89.     
    90. 

  90. 

  91. <div class="header-inner" style="height: 70vh;">
    92. 

  92.   <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
    93. 

  93.   <div class="container">
    94. 

  94.     <a class="navbar-brand" href="/">
    95. 

  95.       <strong>Limour&#39;s Blog</strong>
    96. 

  96.     </a>
    97. 

  97. 

  98.     <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
    99. 

  99.             data-target="#navbarSupportedContent"
    100. 

  100.             aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
    101. 

  101.       <div class="animated-icon"><span></span><span></span><span></span></div>
    102. 

  102.     </button>
    103. 

  103. 

  104.     <!-- Collapsible content -->
    105. 

  105.     <div class="collapse navbar-collapse" id="navbarSupportedContent">
    106. 

  106.       <ul class="navbar-nav ml-auto text-center">
    107. 

  107.         
    108. 

  108.           
    109. 

  109.           
    110. 

  110.           
    111. 

  111.           
    112. 

  112.             <li class="nav-item">
    113. 

  113.               <a class="nav-link" href="https://hexo.limour.top/" target="_self">
    114. 

  114.                 <i class="iconfont icon-home-fill"></i>
    115. 

  115.                 <span>Home</span>
    116. 

  116.               </a>
    117. 

  117.             </li>
    118. 

  118.           
    119. 

  119.         
    120. 

  120.           
    121. 

  121.           
    122. 

  122.           
    123. 

  123.           
    124. 

  124.             <li class="nav-item">
    125. 

  125.               <a class="nav-link" href="/archives/" target="_self">
    126. 

  126.                 <i class="iconfont icon-archive-fill"></i>
    127. 

  127.                 <span>Archive1</span>
    128. 

  128.               </a>
    129. 

  129.             </li>
    130. 

  130.           
    131. 

  131.         
    132. 

  132.           
    133. 

  133.           
    134. 

  134.           
    135. 

  135.           
    136. 

  136.             <li class="nav-item">
    137. 

  137.               <a class="nav-link" href="https://occdn.limour.top/archives/" target="_self">
    138. 

  138.                 <i class="iconfont icon-archive-fill"></i>
    139. 

  139.                 <span>Archive2</span>
    140. 

  140.               </a>
    141. 

  141.             </li>
    142. 

  142.           
    143. 

  143.         
    144. 

  144.           
    145. 

  145.           
    146. 

  146.           
    147. 

  147.           
    148. 

  148.             <li class="nav-item">
    149. 

  149.               <a class="nav-link" href="https://b.limour.top/archives/" target="_self">
    150. 

  150.                 <i class="iconfont icon-archive-fill"></i>
    151. 

  151.                 <span>Archive3</span>
    152. 

  152.               </a>
    153. 

  153.             </li>
    154. 

  154.           
    155. 

  155.         
    156. 

  156.           
    157. 

  157.           
    158. 

  158.           
    159. 

  159.           
    160. 

  160.             <li class="nav-item">
    161. 

  161.               <a class="nav-link" href="https://od.limour.top/" target="_self">
    162. 

  162.                 <i class="iconfont icon-onedrive"></i>
    163. 

  163.                 <span>Alist</span>
    164. 

  164.               </a>
    165. 

  165.             </li>
    166. 

  166.           
    167. 

  167.         
    168. 

  168.           
    169. 

  169.           
    170. 

  170.           
    171. 

  171.           
    172. 

  172.             <li class="nav-item">
    173. 

  173.               <a class="nav-link" href="https://orcid.org/0000-0001-8897-1685" target="_self">
    174. 

  174.                 <i class="iconfont icon-orcid"></i>
    175. 

  175.                 <span>Orcid</span>
    176. 

  176.               </a>
    177. 

  177.             </li>
    178. 

  178.           
    179. 

  179.         
    180. 

  180.           
    181. 

  181.           
    182. 

  182.           
    183. 

  183.           
    184. 

  184.             <li class="nav-item">
    185. 

  185.               <a class="nav-link" href="/links/" target="_self">
    186. 

  186.                 <i class="iconfont icon-link-fill"></i>
    187. 

  187.                 <span>Links</span>
    188. 

  188.               </a>
    189. 

  189.             </li>
    190. 

  190.           
    191. 

  191.         
    192. 

  192.           
    193. 

  193.           
    194. 

  194.           
    195. 

  195.           
    196. 

  196.             <li class="nav-item">
    197. 

  197.               <a class="nav-link" href="/atom.xml" target="_self">
    198. 

  198.                 <i class="iconfont icon-rss"></i>
    199. 

  199.                 <span>RSS</span>
    200. 

  200.               </a>
    201. 

  201.             </li>
    202. 

  202.           
    203. 

  203.         
    204. 

  204.         
    205. 

  205.           <li class="nav-item" id="search-btn">
    206. 

  206.             <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
    207. 

  207.               <i class="iconfont icon-search"></i>
    208. 

  208.             </a>
    209. 

  209.           </li>
    210. 

  210.           
    211. 

  211.         
    212. 

  212.         
    213. 

  213.           <li class="nav-item" id="color-toggle-btn">
    214. 

  214.             <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">
    215. 

  215.               <i class="iconfont icon-dark" id="color-toggle-icon"></i>
    216. 

  216.             </a>
    217. 

  217.           </li>
    218. 

  218.         
    219. 

  219.       </ul>
    220. 

  220.     </div>
    221. 

  221.   </div>
    222. 

  222. </nav>
    223. 

  223. 

  224.   
    225. 

  225. 

  226. <div id="banner" class="banner" parallax=true
    227. 

  227.      style="background: url('https://img.limour.top/2023/08/29/64ee08e108638.webp') no-repeat center center; background-size: cover;">
    228. 

  228.   <div class="full-bg-img">
    229. 

  229.     <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
    230. 

  230.       <div class="banner-text text-center fade-in-up">
    231. 

  231.         <div class="h2">
    232. 

  232.           
    233. 

  233.             <span id="subtitle">【探索】从零开始训练 GPT</span>
    234. 

  234.           
    235. 

  235.         </div>
    236. 

  236. 

  237.         
    238. 

  238.           
    239. 

  239.   <div class="mt-3">
    240. 

  240.     
    241. 

  241.       <span class="post-meta mr-2">
    242. 

  242.         <i class="iconfont icon-author" aria-hidden="true"></i>
    243. 

  243.         Limour
    244. 

  244.       </span>
    245. 

  245.     
    246. 

  246.     
    247. 

  247.       <span class="post-meta">
    248. 

  248.         <i class="iconfont icon-date-fill" aria-hidden="true"></i>
    249. 

  249.         <time datetime="2024-01-18 22:19" pubdate>
    250. 

  250.           January 18, 2024 pm
    251. 

  251.         </time>
    252. 

  252.       </span>
    253. 

  253.     
    254. 

  254.   </div>
    255. 

  255. 

  256.   <div class="mt-1">
    257. 

  257.     
    258. 

  258.       <span class="post-meta mr-2">
    259. 

  259.         <i class="iconfont icon-chart"></i>
    260. 

  260.         
    261. 

  261.           1.7k words
    262. 

  262.         
    263. 

  263.       </span>
    264. 

  264.     
    265. 

  265. 

  266.     
    267. 

  267.       <span class="post-meta mr-2">
    268. 

  268.         <i class="iconfont icon-clock-fill"></i>
    269. 

  269.         
    270. 

  270.         
    271. 

  271.         
    272. 

  272.           15 mins
    273. 

  273.         
    274. 

  274.       </span>
    275. 

  275.     
    276. 

  276. 

  277.     
    278. 

  278.     
    279. 

  279.   </div>
    280. 

  280. 

  281. 

  282.         
    283. 

  283.       </div>
    284. 

  284. 

  285.       
    286. 

  286.     </div>
    287. 

  287.   </div>
    288. 

  288. </div>
    289. 

  289. 

  290. </div>
    291. 

  291. 

  292.   </header>
    293. 

  293. 

  294.   <main>
    295. 

  295.     
    296. 

  296.       
    297. 

  297. 

  298. <div class="container-fluid nopadding-x">
    299. 

  299.   <div class="row nomargin-x">
    300. 

  300.     <div class="side-col d-none d-lg-block col-lg-2">
    301. 

  301.       
    302. 

  302. 

  303.     </div>
    304. 

  304. 

  305.     <div class="col-lg-8 nopadding-x-md">
    306. 

  306.       <div class="container nopadding-x-md" id="board-ctn">
    307. 

  307.         <div id="board">
    308. 

  308.           <article class="post-content mx-auto">
    309. 

  309.             <h1 id="seo-header">【探索】从零开始训练 GPT</h1>
    310. 

  310.             
    311. 

  311.               <p id="updated-time" class="note note-info" style="">
    312. 

  312.                 
    313. 

  313.                   
    314. 

  314.                     Last updated on March 19, 2024 pm
    315. 

  315.                   
    316. 

  316.                 
    317. 

  317.               </p>
    318. 

  318.             
    319. 

  319.             
    320. 

  320.               <div class="markdown-body">
    321. 

  321.                 
    322. 

  322.                 <p><img src="https://img.limour.top/2024/01/18/65a93c6a8065a.webp" srcset="https://jscdn.limour.top/gh/Limour-dev/Sakurairo_Vision/load_svg/inload.svg" lazyload alt="训练中..."></p>
    323. 

  323. <h2 id="预期结构">预期结构</h2>
    324. 

  324. <ul>
    325. 

  325. <li><a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQ=" rel="noopener external nofollow noreferrer">相关代码已经放到 Github</a></li>
    326. 

  326. </ul>
    327. 

  327. <figure class="highlight txt"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><code class="hljs txt">HelloGPT(<br>  (tok_embeddings): Embedding(32765, 768)<br>  (rotary_emb): RotaryEmbedding(head_dim=64, max_seq_len=1024)<br>  (layers): ModuleList(<br>    (0-11): 12 x Decoder(<br>      (ln1): RMSNorm(hidden_size=768, eps=1e-06)<br>      (attn): Attention(<br>        (q_proj): Linear(in_features=768, out_features=768, bias=False)<br>        (k_proj): Linear(in_features=768, out_features=768, bias=False)<br>        (v_proj): Linear(in_features=768, out_features=768, bias=False)<br>        (o_proj): Linear(in_features=768, out_features=768, bias=False)<br>      )<br>      (ln2): RMSNorm(hidden_size=768, eps=1e-06)<br>      (mlp): MLP(<br>        (gate_proj): Linear(in_features=768, out_features=1536, bias=False)<br>        (up_proj): Linear(in_features=768, out_features=1536, bias=False)<br>        (down_proj): Linear(in_features=1536, out_features=768, bias=False)<br>      )<br>    )<br>  )<br>  (norm): RMSNorm(hidden_size=768, eps=1e-06)<br>  (ln2): Linear(in_features=768, out_features=32765, bias=False)<br>)<br></code></pre></td></tr></table></figure>
    328. 

  328. <h2 id="配置环境">配置环境</h2>
    329. 

  329. <ul>
    330. 

  330. <li><a href="/-ji-lu--an-zhuang-conda-bing-geng-huan-qing-hua-yuan">安装conda</a></li>
    331. 

  331. </ul>
    332. 

  332. <figure class="highlight powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><code class="hljs powershell"><span class="hljs-built_in">cd</span> E:\GPT<br>conda install mamba <span class="hljs-literal">-c</span> conda<span class="hljs-literal">-forge</span><br>mamba create <span class="hljs-literal">-n</span> HelloGPT pytorch pytorch<span class="hljs-literal">-cuda</span>=<span class="hljs-number">12.1</span> <span class="hljs-literal">-c</span> pytorch <span class="hljs-literal">-c</span> nvidia <span class="hljs-literal">-c</span> conda<span class="hljs-literal">-forge</span><br>conda activate HelloGPT<br>conda install numpy transformers tiktoken tensorboard sentencepiece<span class="hljs-literal">-python</span> jieba emoji <span class="hljs-literal">-c</span> conda<span class="hljs-literal">-forge</span><br>pip install opencc<span class="hljs-literal">-python-reimplemented</span> <span class="hljs-literal">-i</span> https://pypi.tuna.tsinghua.edu.cn/simple<br>python test_cuda.py<br>python test_SPDA.py<br>D:\vscode\Code.exe<br></code></pre></td></tr></table></figure>
    333. 

  333. <h2 id="准备数据">准备数据</h2>
    334. 

  334. <ul>
    335. 

  335. <li>下载 <a href="https://hexo.limour.top/go/#aHR0cHM6Ly9odWdnaW5nZmFjZS5jby9jb2xsZWN0aW9ucy9MaW1vdXIvcjE4LW5vdmVscy1nYWxnYW1lLTY1OThmMTY4OTRjYWRjOWNkY2IzZjNhYg==" rel="noopener external nofollow noreferrer">h-corpus-2023</a></li>
    336. 

  336. </ul>
    337. 

  337. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">import</span> os<br><br><span class="hljs-keyword">class</span> <span class="hljs-title class_">Fileset</span>(<span class="hljs-title class_ inherited__">list</span>):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, path, ext=<span class="hljs-string">&#x27;&#x27;</span>, _read=<span class="hljs-literal">None</span></span>):<br>        <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(path, <span class="hljs-built_in">str</span>):<br>            self.root = path<br>            self.extend(f <span class="hljs-keyword">for</span> f <span class="hljs-keyword">in</span> os.listdir(self.root) <span class="hljs-keyword">if</span> f.endswith(ext))<br>            self._read = _read<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__getitem__</span>(<span class="hljs-params">self, index</span>):<br>        <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(index, <span class="hljs-built_in">int</span>):  <span class="hljs-comment"># index是索引</span><br>            <span class="hljs-keyword">if</span> self._read:<br>                <span class="hljs-keyword">return</span> self._read(os.path.join(self.root, <span class="hljs-built_in">super</span>().__getitem__(index)))<br>            <span class="hljs-keyword">else</span>:<br>                <span class="hljs-keyword">return</span> os.path.join(self.root, <span class="hljs-built_in">super</span>().__getitem__(index))<br>        <span class="hljs-keyword">else</span>:  <span class="hljs-comment"># index是切片</span><br>            fileset = Fileset(<span class="hljs-literal">None</span>)<br>            fileset.root = self.root<br>            fileset._read = self._read<br>            fileset.extend(<span class="hljs-built_in">super</span>().__getitem__(index))<br>            <span class="hljs-keyword">return</span> fileset<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">getFileName</span>(<span class="hljs-params">self, index</span>):<br>        fname, ext = os.path.splitext(<span class="hljs-built_in">super</span>().__getitem__(index))<br>        <span class="hljs-keyword">return</span> fname<br><br><br><span class="hljs-keyword">from</span> tokenizer <span class="hljs-keyword">import</span> tokenizer<br>token_eos = <span class="hljs-number">2</span><br><br><br><span class="hljs-keyword">def</span> <span class="hljs-title function_">readOne</span>(<span class="hljs-params">filePath</span>):<br>    retn = []<br>    <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(file=filePath, encoding=<span class="hljs-string">&#x27;utf-8&#x27;</span>) <span class="hljs-keyword">as</span> f:<br>        <span class="hljs-keyword">for</span> line <span class="hljs-keyword">in</span> f:<br>            retn += tokenizer.encode(line).ids<br>    retn.append(token_eos)<br>    <span class="hljs-keyword">return</span> retn<br><br><br><span class="hljs-keyword">class</span> <span class="hljs-title class_">Hcorpus</span>():<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, path, ext=<span class="hljs-string">&#x27;txt&#x27;</span>, fileset_idx=<span class="hljs-number">0</span>, fileset_sub_idx=<span class="hljs-number">0</span></span>):<br>        self.fileset = Fileset(path, ext, readOne)<br>        self.fileset_idx = fileset_idx<br>        self.fileset_sub_idx = fileset_sub_idx<br>        <span class="hljs-keyword">if</span> self.fileset_sub_idx &lt; <span class="hljs-number">0</span>:  <span class="hljs-comment"># 再读上一个太复杂了，直接放弃</span><br>            self.fileset_sub_idx = <span class="hljs-number">0</span><br>        <span class="hljs-keyword">if</span> self.fileset_idx &gt;= <span class="hljs-built_in">len</span>(self.fileset):<br>            self.fileset_idx = <span class="hljs-number">0</span><br>        self.cache = self.fileset[self.fileset_idx]<br>        self.fileset_idx += <span class="hljs-number">1</span><br>        self.cache_idx = self.fileset_sub_idx<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, size=<span class="hljs-number">512</span></span>):<br>        <span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(self.cache) &lt; self.cache_idx + size:<br>            <span class="hljs-keyword">if</span> self.fileset_idx &gt;= <span class="hljs-built_in">len</span>(self.fileset):<br>                self.fileset_idx = <span class="hljs-number">0</span><br>            self.fileset_sub_idx = self.cache_idx - <span class="hljs-built_in">len</span>(self.cache)<br>            self.cache = self.cache[self.cache_idx:] + self.fileset[self.fileset_idx]<br>            self.cache_idx = <span class="hljs-number">0</span><br>            self.fileset_idx += <span class="hljs-number">1</span><br>        retn = self.cache[self.cache_idx:self.cache_idx + size]<br>        self.cache_idx += size<br>        self.fileset_sub_idx += size<br>        <span class="hljs-keyword">return</span> retn<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__repr__</span>(<span class="hljs-params">self</span>):<br>        <span class="hljs-keyword">return</span> <span class="hljs-string">f&quot;Hcorpus(r&#x27;<span class="hljs-subst">&#123;self.fileset.root&#125;</span>&#x27;, fileset_idx=<span class="hljs-subst">&#123;self.fileset_idx-<span class="hljs-number">1</span>&#125;</span>, fileset_sub_idx=<span class="hljs-subst">&#123;self.fileset_sub_idx&#125;</span>)&quot;</span><br></code></pre></td></tr></table></figure>
    338. 

  338. <h2 id="训练Tokenizer">训练Tokenizer</h2>
    339. 

  339. <ul>
    340. 

  340. <li><a href="https://hexo.limour.top/go/#aHR0cHM6Ly9odWdnaW5nZmFjZS5jby9kb2NzL3Rva2VuaXplcnMvcXVpY2t0b3Vy" rel="noopener external nofollow noreferrer">tokenizer 包的文档</a></li>
    341. 

  341. <li>繁体转换成简体：<a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQvYmxvYi9tYWluL3RyYWluX3Rva2VuaXplcl9wcmUucHk=" rel="noopener external nofollow noreferrer">train_tokenizer_pre.py</a></li>
    342. 

  342. <li>获取常用 emoji：<a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQvYmxvYi9tYWluL3RtcF9lbW9qaS5weQ==" rel="noopener external nofollow noreferrer">tmp_emoji.py</a></li>
    343. 

  343. <li>分词统计词频：<a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQvYmxvYi9tYWluL3RyYWluX3Rva2VuaXplcl9qaWViYS5weQ==" rel="noopener external nofollow noreferrer">tokenizer_jieba.py</a></li>
    344. 

  344. <li>区分词性并构造 BPE 语料：<a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQvYmxvYi9tYWluL3RyYWluX3Rva2VuaXplcl9qaWViYV9zdGF0aXN0aWNzLnB5" rel="noopener external nofollow noreferrer">train_tokenizer_jieba_statistics.py</a></li>
    345. 

  345. <li>训练 BPE 模型：<a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQvYmxvYi9tYWluL3RyYWluX3Rva2VuaXplci5weQ==" rel="noopener external nofollow noreferrer">train_tokenizer.py</a></li>
    346. 

  346. <li>最终训练好的 BPE 模型：<a href="https://hexo.limour.top/go/#aHR0cHM6Ly9naXRodWIuY29tL0xpbW91ci1kZXYvSGVsbG9HUFQvYmxvYi9tYWluL0hlbGxvQlBFLnRva2VuaXplci5qc29u" rel="noopener external nofollow noreferrer">HelloBPE.tokenizer.json</a></li>
    347. 

  347. </ul>
    348. 

  348. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer<br>tokenizer = Tokenizer.from_file(<span class="hljs-string">&quot;HelloBPE.tokenizer.json&quot;</span>)<br></code></pre></td></tr></table></figure>
    349. 

  349. <h2 id="定义模型">定义模型</h2>
    350. 

  350. <h3 id="定义-Decoder">定义 Decoder</h3>
    351. 

  351. <h4 id="定义-RMSnorm">定义 RMSnorm</h4>
    352. 

  352. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">RMSNorm</span>(nn.Module):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, dim: <span class="hljs-built_in">int</span>, eps: <span class="hljs-built_in">float</span> = <span class="hljs-number">1e-6</span></span>):<br>        <span class="hljs-built_in">super</span>().__init__()<br>        self.eps = eps<br>        self.weight = nn.Parameter(torch.ones(dim))<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):<br>        x = x * torch.rsqrt(x.<span class="hljs-built_in">pow</span>(<span class="hljs-number">2</span>).mean(-<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>) + self.eps)<br>        <span class="hljs-keyword">return</span> x * self.weight<br></code></pre></td></tr></table></figure>
    353. 

  353. <h4 id="定义-RoPE">定义 RoPE</h4>
    354. 

  354. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">RotaryEmbedding</span>(nn.Module):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, head_dim: <span class="hljs-built_in">int</span>, max_seq_len: <span class="hljs-built_in">int</span>, device=device, theta: <span class="hljs-built_in">float</span> = <span class="hljs-number">10000.0</span></span>):<br>        <span class="hljs-built_in">super</span>().__init__()<br>        self.head_dim = head_dim<br>        self.set_max_seq_len(max_seq_len, device, theta)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">set_max_seq_len</span>(<span class="hljs-params">self, max_seq_len: <span class="hljs-built_in">int</span>, device=device, theta: <span class="hljs-built_in">float</span> = <span class="hljs-number">10000.0</span></span>):<br>        self.max_seq_len = max_seq_len<br>        freqs = <span class="hljs-number">1.0</span> / (theta ** (torch.arange(<span class="hljs-number">0</span>, self.head_dim, <span class="hljs-number">2</span>).<span class="hljs-built_in">float</span>().to(device) / self.head_dim))<br>        t = torch.arange(max_seq_len, device=device)  <span class="hljs-comment"># type: ignore</span><br>        freqs = torch.outer(t, freqs).<span class="hljs-built_in">float</span>()  <span class="hljs-comment"># 外积</span><br>        self.freqs_cis = torch.polar(torch.ones_like(freqs), freqs)  <span class="hljs-comment"># 复数，模 1，角度 freqs</span><br>        self.freqs_cis.requires_grad = <span class="hljs-literal">False</span>  <span class="hljs-comment"># filter(lambda p : p.requires_grad, model.parameters())</span><br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">rotary_emb</span>(<span class="hljs-params">self, x</span>):<br>        x_ = torch.view_as_complex(x.<span class="hljs-built_in">float</span>().reshape(*x.shape[:-<span class="hljs-number">1</span>], -<span class="hljs-number">1</span>, <span class="hljs-number">2</span>))<br>        x_out = torch.view_as_real(x_ * self.local_freqs_cis).flatten(<span class="hljs-number">3</span>)<br>        <span class="hljs-keyword">return</span> x_out.type_as(x)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, start_pos: <span class="hljs-built_in">int</span>, seqlen: <span class="hljs-built_in">int</span></span>):<br>        self.local_freqs_cis = self.freqs_cis[start_pos: start_pos + seqlen].view(<span class="hljs-number">1</span>, seqlen, <span class="hljs-number">1</span>, -<span class="hljs-number">1</span>)  <span class="hljs-comment"># cacheKV 相关，可忽略</span><br>        self.local_freqs_cis.requires_grad = <span class="hljs-literal">False</span><br>        <span class="hljs-keyword">return</span> self.rotary_emb<br></code></pre></td></tr></table></figure>
    355. 

  355. <h4 id="定义-Attention">定义 Attention</h4>
    356. 

  356. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">Attention</span>(nn.Module):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, hidden_size, n_heads, cacheKV, max_batch_size, max_seq_len, device=device</span>):<br>        <span class="hljs-built_in">super</span>().__init__()<br>        self.n_heads = n_heads<br>        self.head_dim = hidden_size // n_heads<br>        self.q_proj = nn.Linear(hidden_size, hidden_size, bias=<span class="hljs-literal">False</span>)<br>        self.k_proj = nn.Linear(hidden_size, hidden_size, bias=<span class="hljs-literal">False</span>)<br>        self.v_proj = nn.Linear(hidden_size, hidden_size, bias=<span class="hljs-literal">False</span>)<br>        self.o_proj = nn.Linear(hidden_size, hidden_size, bias=<span class="hljs-literal">False</span>)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states, rotary_emb, start_pos=<span class="hljs-number">0</span>, mask=<span class="hljs-literal">None</span>, is_causal=<span class="hljs-literal">True</span></span>):<br>        bsz, seqlen, hidden_size = hidden_states.shape<br><br>        q = self.q_proj(hidden_states)<br>        k = self.k_proj(hidden_states)<br>        v = self.v_proj(hidden_states)<br><br>        q = q.view(bsz, seqlen, self.n_heads, self.head_dim)<br>        k = k.view(bsz, seqlen, self.n_heads, self.head_dim)<br>        v = v.view(bsz, seqlen, self.n_heads, self.head_dim)<br><br>        q = rotary_emb(q)<br>        k = rotary_emb(k)<br><br>        q = q.transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>)  <span class="hljs-comment"># (bs, n_heads, seqlen, head_dim)</span><br>        k = k.transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>)  <span class="hljs-comment"># (bs, n_local_heads, cache_len + seqlen, head_dim)</span><br>        v = v.transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>)  <span class="hljs-comment"># (bs, n_local_heads, cache_len + seqlen, head_dim)</span><br><br>        output = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=is_causal)<br><br>        output = output.transpose(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>).contiguous().view(bsz, seqlen, hidden_size)<br>        <span class="hljs-keyword">return</span> self.o_proj(output)<br></code></pre></td></tr></table></figure>
    357. 

  357. <h4 id="定义-MLP">定义 MLP</h4>
    358. 

  358. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">MLP</span>(nn.Module):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, hidden_size</span>):<br>        <span class="hljs-built_in">super</span>().__init__()<br>        intermediate_size = <span class="hljs-built_in">int</span>(<span class="hljs-number">2</span> * hidden_size)<br>        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=<span class="hljs-literal">False</span>)<br>        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=<span class="hljs-literal">False</span>)<br>        self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=<span class="hljs-literal">False</span>)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x</span>):<br>        gate = F.silu(self.gate_proj(x))<br>        intermediate_states = self.up_proj(x)<br>        <span class="hljs-keyword">return</span> self.down_proj(gate * intermediate_states)<br></code></pre></td></tr></table></figure>
    359. 

  359. <h4 id="组装-Decoder">组装 Decoder</h4>
    360. 

  360. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">Decoder</span>(nn.Module):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, hidden_size, n_heads, cacheKV, max_batch_size, max_seq_len</span>):<br>        <span class="hljs-built_in">super</span>().__init__()<br>        self.ln1 = RMSNorm(hidden_size)<br>        self.attn = Attention(hidden_size, n_heads, cacheKV, max_batch_size, max_seq_len)<br>        self.ln2 = RMSNorm(hidden_size)<br>        self.mlp = MLP(hidden_size)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, x, rotary_emb, start_pos, mask=<span class="hljs-literal">None</span>, is_causal=<span class="hljs-literal">True</span></span>):<br>        x = x + self.attn(self.ln1(x), rotary_emb, start_pos, mask, is_causal)<br>        <span class="hljs-keyword">return</span> x + self.mlp(self.ln2(x))<br></code></pre></td></tr></table></figure>
    361. 

  361. <h3 id="组装模型">组装模型</h3>
    362. 

  362. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">HelloGPT</span>(nn.Module):<br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, vocab_size=<span class="hljs-number">32765</span>, hidden_size=<span class="hljs-number">768</span>, n_heads=<span class="hljs-number">12</span>, max_seq_len=<span class="hljs-number">1024</span>, n_layers=<span class="hljs-number">12</span>, cacheKV=<span class="hljs-literal">False</span>, max_batch_size=<span class="hljs-number">1</span></span>):<br>        <span class="hljs-built_in">super</span>().__init__()<br>        <span class="hljs-comment"># hidden_size &gt; 8.33 * ln(vocab_size)</span><br>        self.tok_embeddings = nn.Embedding(vocab_size, hidden_size)<br>        self.rotary_emb = RotaryEmbedding(hidden_size // n_heads, max_seq_len * <span class="hljs-number">2</span>)<br>        self.rotary_emb.requires_grad = <span class="hljs-literal">False</span><br>        self.layers = nn.ModuleList()<br>        <span class="hljs-keyword">for</span> layer_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_layers):<br>            self.layers.append(Decoder(hidden_size, n_heads, cacheKV, max_batch_size, max_seq_len))<br>        self.norm = RMSNorm(hidden_size)<br>        self.ln2 = nn.Linear(hidden_size, vocab_size, bias=<span class="hljs-literal">False</span>)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, input_ids: torch.Tensor, start_pos=<span class="hljs-number">0</span>, no_mask=<span class="hljs-literal">True</span></span>):<br>        _bsz, seqlen = input_ids.shape<br>        h = self.tok_embeddings(input_ids)<br><br>        <span class="hljs-comment"># 预计算，减少每一层的重复计算</span><br>        rotary_emb = self.rotary_emb(start_pos, seqlen)<br>        <span class="hljs-keyword">for</span> layer <span class="hljs-keyword">in</span> self.layers:<br>            h = layer(h, rotary_emb, start_pos)<br><br>        h = self.norm(h)<br>        h = self.ln2(h)<br>        <span class="hljs-keyword">return</span> h.<span class="hljs-built_in">float</span>()<br></code></pre></td></tr></table></figure>
    363. 

  363. <h2 id="训练模型">训练模型</h2>
    364. 

  364. <h3 id="数据载入">数据载入</h3>
    365. 

  365. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><code class="hljs python">data = Hcorpus(<span class="hljs-string">r&#x27;D:\datasets\h-corpus&#x27;</span>)<br><span class="hljs-keyword">def</span> <span class="hljs-title function_">get_batch</span>(<span class="hljs-params">size=<span class="hljs-number">512</span>, bsz=<span class="hljs-number">8</span></span>):<br>    x = []<br>    y = []<br>    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(bsz):<br>        tmp = data(size+<span class="hljs-number">1</span>)<br>        x.append(tmp[:size])<br>        y.append(tmp[<span class="hljs-number">1</span>:])<br>    <span class="hljs-keyword">return</span> torch.tensor(x).to(device), torch.tensor(y).to(device)<br></code></pre></td></tr></table></figure>
    366. 

  366. <h3 id="模型载入">模型载入</h3>
    367. 

  367. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs python">model = HelloGPT(n_layers=<span class="hljs-number">8</span>, max_seq_len=<span class="hljs-number">768</span>)<br>model.to(device)<br></code></pre></td></tr></table></figure>
    368. 

  368. <h3 id="训练模型-2">训练模型</h3>
    369. 

  369. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-comment">## 初始化训练器</span><br>criterion = nn.CrossEntropyLoss()  <span class="hljs-comment"># 交叉熵损失函数</span><br>optimizer = torch.optim.Adam(train_parameters, lr=<span class="hljs-number">6e-4</span>)  <span class="hljs-comment"># Adam 优化器</span><br>scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=<span class="hljs-number">5</span>, T_mult=<span class="hljs-number">2</span>)  <span class="hljs-comment"># 余弦退火学习率</span><br>torch.manual_seed(<span class="hljs-number">1337</span>)  <span class="hljs-comment"># 魔术随机种子</span><br><br>total_loss = <span class="hljs-number">0</span><br>print_iter = <span class="hljs-number">20</span><br><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">100001</span>):<br>    optimizer.zero_grad(set_to_none=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># 清空梯度，节省显存</span><br>    x, y = get_batch(size=<span class="hljs-number">384</span>, bsz=<span class="hljs-number">4</span>)  <span class="hljs-comment"># x 是训练语料 y 是 x 移动了一位，当做预测目标</span><br>    y_ = model(x)  <span class="hljs-comment"># 通过 x 预测的 y</span><br>    loss = criterion(y_.view(-<span class="hljs-number">1</span>, <span class="hljs-number">32765</span>), y.view(-<span class="hljs-number">1</span>))  <span class="hljs-comment"># 计算损失</span><br>    loss.backward()  <span class="hljs-comment"># 反向传播梯度</span><br>    torch.nn.utils.clip_grad_norm_(train_parameters, <span class="hljs-number">0.5</span>)  <span class="hljs-comment"># 梯度裁剪，减轻过拟合</span><br>    optimizer.step()  <span class="hljs-comment"># 通过梯度优化训练参数</span><br>    scheduler.step()  <span class="hljs-comment"># 计算下一步的学习率</span><br>    total_loss += loss  <span class="hljs-comment"># 累计损失</span><br><br>    <span class="hljs-keyword">if</span> epoch % print_iter == <span class="hljs-number">0</span>:<br>        <span class="hljs-built_in">print</span>(data)<br>        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&#x27;epoch: <span class="hljs-subst">&#123;epoch&#125;</span>  lr: <span class="hljs-subst">&#123;scheduler.get_last_lr()[<span class="hljs-number">0</span>]:<span class="hljs-number">.4</span>e&#125;</span> loss: <span class="hljs-subst">&#123;total_loss / print_iter:<span class="hljs-number">.4</span>e&#125;</span>&#x27;</span>)<br>        total_loss = <span class="hljs-number">0</span><br></code></pre></td></tr></table></figure>
    370. 

  370. <h3 id="保存读取">保存读取</h3>
    371. 

  371. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">&#x27;tmp_training.pkl&#x27;</span>, <span class="hljs-string">&#x27;rb&#x27;</span>) <span class="hljs-keyword">as</span> file:<br>    epoch = pickle.load(file)  <span class="hljs-comment"># 读取 epoch 位置</span><br>    tmp_fileset_idx = pickle.load(file)  <span class="hljs-comment"># 读取 data 位置</span><br>    tmp_fileset_sub_idx = pickle.load(file)<br><span class="hljs-comment"># 恢复数据位置</span><br>data = Hcorpus(<span class="hljs-string">r&#x27;D:\datasets\h-corpus&#x27;</span>, fileset_idx=tmp_fileset_idx-<span class="hljs-number">1</span>, fileset_sub_idx=tmp_fileset_sub_idx)<br>model = torch.load(<span class="hljs-string">f&#x27;tmp_model_<span class="hljs-subst">&#123;epoch&#125;</span>.pth&#x27;</span>)  <span class="hljs-comment"># 恢复模型</span><br><span class="hljs-built_in">print</span>(<span class="hljs-string">f&#x27;start from epoch: <span class="hljs-subst">&#123;epoch&#125;</span>   data: <span class="hljs-subst">&#123;data&#125;</span>&#x27;</span>)<br><br>save_iter = <span class="hljs-number">5000</span><br><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">100001</span>):<br>    <span class="hljs-keyword">pass</span><br>    <span class="hljs-keyword">if</span> epoch % save_iter == <span class="hljs-number">0</span>:<br>        optimizer.zero_grad(set_to_none=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># 清空梯度，节省显存</span><br>        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">&#x27;tmp_training.pkl&#x27;</span>, <span class="hljs-string">&#x27;wb&#x27;</span>) <span class="hljs-keyword">as</span> file:<br>            pickle.dump(epoch, file)  <span class="hljs-comment"># 保存 epoch 位置</span><br>            pickle.dump(data.fileset_idx, file)  <span class="hljs-comment"># 保存 data 位置</span><br>            pickle.dump(data.fileset_sub_idx, file)<br>        torch.save(model, <span class="hljs-string">f&#x27;tmp_model_<span class="hljs-subst">&#123;epoch&#125;</span>.pth&#x27;</span>)  <span class="hljs-comment"># 保存模型</span><br>        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&#x27;save to tmp_model_<span class="hljs-subst">&#123;epoch&#125;</span>.pth&#x27;</span>)<br></code></pre></td></tr></table></figure>
    372. 

  372. <h3 id="可视化">可视化</h3>
    373. 

  373. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><code class="hljs python">writer = SummaryWriter(<span class="hljs-string">&#x27;logs&#x27;</span>)  <span class="hljs-comment"># tensorboard --logdir logs</span><br><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">100001</span>):<br>    <span class="hljs-keyword">pass</span><br>    writer.add_scalar(<span class="hljs-string">&#x27;lr&#x27;</span>, scheduler.get_last_lr()[<span class="hljs-number">0</span>], epoch)<br>    writer.add_scalar(<span class="hljs-string">&#x27;loss&#x27;</span>, loss, epoch)<br>    <span class="hljs-keyword">if</span> epoch % print_iter == <span class="hljs-number">0</span>:<br>        <span class="hljs-keyword">pass</span><br>        writer.add_scalar(<span class="hljs-string">&#x27;total_loss&#x27;</span>, total_loss / print_iter, epoch)<br>writer.close()<br></code></pre></td></tr></table></figure>
    374. 

  374. <h2 id="附加-streaming-llm">附加 streaming_llm</h2>
    375. 

  375. <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><code class="hljs python"><span class="hljs-keyword">class</span> <span class="hljs-title class_">RotaryEmbedding</span>(nn.Module):<br>    <span class="hljs-keyword">pass</span><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">inverse_rotary_emb</span>(<span class="hljs-params">self, x</span>):<br>        x_ = torch.view_as_complex(x.<span class="hljs-built_in">float</span>().reshape(*x.shape[:-<span class="hljs-number">1</span>], -<span class="hljs-number">1</span>, <span class="hljs-number">2</span>))<br>        x_out = torch.view_as_real(x_ * self.local_freqs_cis_inverse).flatten(<span class="hljs-number">3</span>)<br>        <span class="hljs-keyword">return</span> x_out.type_as(x)<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">inverse_forward</span>(<span class="hljs-params">self, start_pos: <span class="hljs-built_in">int</span>, seqlen: <span class="hljs-built_in">int</span></span>):<br>        self.local_freqs_cis_inverse = self.freqs_cis[start_pos: start_pos + seqlen].view(<span class="hljs-number">1</span>, seqlen, <span class="hljs-number">1</span>, -<span class="hljs-number">1</span>)  <span class="hljs-comment"># cacheKV 相关，可忽略</span><br>        self.local_freqs_cis_inverse = self.local_freqs_cis_inverse.conj()  <span class="hljs-comment"># 乘上共轭就旋转回去了</span><br>        self.local_freqs_cis.requires_grad = <span class="hljs-literal">False</span><br>        <span class="hljs-keyword">return</span> self.inverse_rotary_emb<br><br><span class="hljs-keyword">class</span> <span class="hljs-title class_">Attention</span>(nn.Module):<br>    <span class="hljs-keyword">pass</span><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states, rotary_emb, start_pos=<span class="hljs-number">0</span>, mask=<span class="hljs-literal">None</span>, is_causal=<span class="hljs-literal">True</span></span>):<br>        <span class="hljs-keyword">pass</span><br>        <span class="hljs-keyword">if</span> self.cacheKV:  <span class="hljs-comment"># cacheKV 相关，可忽略</span><br>            self.cache_k[:bsz, start_pos: start_pos + seqlen] = k<br>            self.cache_v[:bsz, start_pos: start_pos + seqlen] = v<br>            k = self.cache_k[:bsz, : start_pos + seqlen]<br>            v = self.cache_v[:bsz, : start_pos + seqlen]<br><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">streaming_llm</span>(<span class="hljs-params">self, start_pos, seqlen, to_pos, inverse_rotary_emb, rotary_emb, bsz</span>):<br>        k = self.cache_k[:bsz, start_pos: start_pos + seqlen]<br>        v = self.cache_v[:bsz, start_pos: start_pos + seqlen]<br>        k = inverse_rotary_emb(k)<br>        k = rotary_emb(k)<br>        self.cache_k[:bsz, to_pos: to_pos + seqlen] = k<br>        self.cache_v[:bsz, to_pos: to_pos + seqlen] = v<br><br><span class="hljs-keyword">class</span> <span class="hljs-title class_">HelloGPT</span>(nn.Module):<br>    <span class="hljs-keyword">pass</span><br>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">streaming_llm</span>(<span class="hljs-params">self, start_pos, seqlen, to_pos, max_batch_size=<span class="hljs-number">1</span></span>):<br>        rotary_emb = self.rotary_emb(to_pos, seqlen)<br>        inverse_rotary_emb = self.rotary_emb.inverse_forward(start_pos, seqlen)<br>        <span class="hljs-keyword">for</span> layer <span class="hljs-keyword">in</span> self.layers:<br>            layer.attn.streaming_llm(start_pos, seqlen, to_pos, inverse_rotary_emb, rotary_emb, max_batch_size)<br></code></pre></td></tr></table></figure>
    376. 

  376.                 
    377. 

  377.               </div>
    378. 

  378.             
    379. 

  379.             <hr/>
    380. 

  380.             <div>
    381. 

  381.               <div class="post-metas my-3">
    382. 

  382.   
    383. 

  383.   
    384. 

  384.     <div class="post-meta">
    385. 

  385.       <i class="iconfont icon-tags"></i>
    386. 

  386.       
    387. 

  387.         <a href="/tags/%E6%8E%A2%E7%B4%A2/" class="print-no-link">#探索</a>
    388. 

  388.       
    389. 

  389.         <a href="/tags/llama/" class="print-no-link">#llama</a>
    390. 

  390.       
    391. 

  391.     </div>
    392. 

  392.   
    393. 

  393. </div>
    394. 

  394. 

  395. 

  396.               
    397. 

  397.   
    398. 

  398. 

  399.   <div class="license-box my-3">
    400. 

  400.     <div class="license-title">
    401. 

  401.       <div>【探索】从零开始训练 GPT</div>
    402. 

  402.       <div>https://hexo.limour.top/training-gpt-from-scratch</div>
    403. 

  403.     </div>
    404. 

  404.     <div class="license-meta">
    405. 

  405.       
    406. 

  406.         <div class="license-meta-item">
    407. 

  407.           <div>Author</div>
    408. 

  408.           <div>Limour</div>
    409. 

  409.         </div>
    410. 

  410.       
    411. 

  411.       
    412. 

  412.         <div class="license-meta-item license-meta-date">
    413. 

  413.           <div>Posted on</div>
    414. 

  414.           <div>January 18, 2024</div>
    415. 

  415.         </div>
    416. 

  416.       
    417. 

  417.       
    418. 

  418.         <div class="license-meta-item license-meta-date">
    419. 

  419.           <div>Updated on</div>
    420. 

  420.           <div>March 19, 2024</div>
    421. 

  421.         </div>
    422. 

  422.       
    423. 

  423.       
    424. 

  424.         <div class="license-meta-item">
    425. 

  425.           <div>Licensed under</div>
    426. 

  426.           <div>
    427. 

  427.             
    428. 

  428.               
    429. 

  429.               
    430. 

  430.                 <a class="print-no-link" target="_blank" href="https://creativecommons.org/licenses/by-nc-sa/4.0/">
    431. 

  431.                   <span class="hint--top hint--rounded" aria-label="BY - Attribution">
    432. 

  432.                     <i class="iconfont icon-cc-by"></i>
    433. 

  433.                   </span>
    434. 

  434.                 </a>
    435. 

  435.               
    436. 

  436.                 <a class="print-no-link" target="_blank" href="https://creativecommons.org/licenses/by-nc-sa/4.0/">
    437. 

  437.                   <span class="hint--top hint--rounded" aria-label="NC - Non-commercial">
    438. 

  438.                     <i class="iconfont icon-cc-nc"></i>
    439. 

  439.                   </span>
    440. 

  440.                 </a>
    441. 

  441.               
    442. 

  442.                 <a class="print-no-link" target="_blank" href="https://creativecommons.org/licenses/by-nc-sa/4.0/">
    443. 

  443.                   <span class="hint--top hint--rounded" aria-label="SA - Share-alike">
    444. 

  444.                     <i class="iconfont icon-cc-sa"></i>
    445. 

  445.                   </span>
    446. 

  446.                 </a>
    447. 

  447.               
    448. 

  448.             
    449. 

  449.           </div>
    450. 

  450.         </div>
    451. 

  451.       
    452. 

  452.     </div>
    453. 

  453.     <div class="license-icon iconfont"></div>
    454. 

  454.   </div>
    455. 

  455. 

  456. 

  457. 

  458.               
    459. 

  459.                 <div class="post-prevnext my-3">
    460. 

  460.                   <article class="post-prev col-6">
    461. 

  461.                     
    462. 

  462.                     
    463. 

  463.                       <a href="/Convert-BlueLM-7B-Chat-to-the-standard-GGUF-model" title="【探索】将BlueLM-7B-Chat转换为标准的GGUF模型">
    464. 

  464.                         <i class="iconfont icon-arrowleft"></i>
    465. 

  465.                         <span class="hidden-mobile">【探索】将BlueLM-7B-Chat转换为标准的GGUF模型</span>
    466. 

  466.                         <span class="visible-mobile">Previous</span>
    467. 

  467.                       </a>
    468. 

  468.                     
    469. 

  469.                   </article>
    470. 

  470.                   <article class="post-next col-6">
    471. 

  471.                     
    472. 

  472.                     
    473. 

  473.                       <a href="/Azure-AI-prevents-reverse-wool-shearing" title="【避坑】Azure AI 避免反向薅羊毛">
    474. 

  474.                         <span class="hidden-mobile">【避坑】Azure AI 避免反向薅羊毛</span>
    475. 

  475.                         <span class="visible-mobile">Next</span>
    476. 

  476.                         <i class="iconfont icon-arrowright"></i>
    477. 

  477.                       </a>
    478. 

  478.                     
    479. 

  479.                   </article>
    480. 

  480.                 </div>
    481. 

  481.               
    482. 

  482.             </div>
    483. 

  483. 

  484.             
    485. 

  485.   
    486. 

  486.   
    487. 

  487.     <article id="comments" lazyload>
    488. 

  488.       
    489. 

  489.   <div id="waline"></div>
    490. 

  490.   <script type="text/javascript">
    491. 

  491.     Fluid.utils.loadComments('#waline', function() {
    492. 

  492.       Fluid.utils.createCssLink('https://cdn.staticfile.org/waline/2.15.5/waline.css')
    493. 

  493.       Fluid.utils.createScript('https://cdn.staticfile.org/waline/2.15.5/waline.js', function() {
    494. 

  494.         var options = Object.assign(
    495. 

  495.           {"serverURL":"https://comments.limour.top","path":"window.location.pathname","meta":["nick","mail","link"],"requiredMeta":["nick"],"lang":"zh-CN","emoji":["https://jscdn.limour.top/gh/walinejs/emojis/weibo"],"dark":"html[data-user-color-scheme=\"dark\"]","wordLimit":0,"pageSize":10},
    496. 

  496.           {
    497. 

  497.             el: '#waline',
    498. 

  498.             path: window.location.pathname
    499. 

  499.           }
    500. 

  500.         )
    501. 

  501.         Waline.init(options);
    502. 

  502.         Fluid.utils.waitElementVisible('#waline .vcontent', () => {
    503. 

  503.           var imgSelector = '#waline .vcontent img:not(.vemoji)';
    504. 

  504.           Fluid.plugins.imageCaption(imgSelector);
    505. 

  505.           Fluid.plugins.fancyBox(imgSelector);
    506. 

  506.         })
    507. 

  507.       });
    508. 

  508.     });
    509. 

  509.   </script>
    510. 

  510.   <noscript>Please enable JavaScript to view the comments</noscript>
    511. 

  511. 

  512. 

  513.     </article>
    514. 

  514.   
    515. 

  515. 

  516. 

  517.           </article>
    518. 

  518.         </div>
    519. 

  519.       </div>
    520. 

  520.     </div>
    521. 

  521. 

  522.     <div class="side-col d-none d-lg-block col-lg-2">
    523. 

  523.       
    524. 

  524.   <aside class="sidebar" style="margin-left: -1rem">
    525. 

  525.     <div id="toc">
    526. 

  526.   <p class="toc-header">
    527. 

  527.     <i class="iconfont icon-list"></i>
    528. 

  528.     <span>Table of Contents</span>
    529. 

  529.   </p>
    530. 

  530.   <div class="toc-body" id="toc-body"></div>
    531. 

  531. </div>
    532. 

  532. 

  533. 

  534. 

  535.   </aside>
    536. 

  536. 

  537. 

  538.     </div>
    539. 

  539.   </div>
    540. 

  540. </div>
    541. 

  541. 

  542. 

  543. 

  544. 

  545. 

  546.   
    547. 

  547. 

  548. 

  549. 

  550.   
    551. 

  551. 

  552. 

  553. 

  554.   
    555. 

  555. 

  556. 

  557. 

  558. 

  559.   
    560. 

  560. 

  561. 

  562. 

  563. 

  564. 

  565. 

  566. 

  567.     
    568. 

  568. 

  569.     
    570. 

  570.       <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
    571. 

  571.         <i class="iconfont icon-arrowup" aria-hidden="true"></i>
    572. 

  572.       </a>
    573. 

  573.     
    574. 

  574. 

  575.     
    576. 

  576.       <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
    577. 

  577.      aria-hidden="true">
    578. 

  578.   <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    579. 

  579.     <div class="modal-content">
    580. 

  580.       <div class="modal-header text-center">
    581. 

  581.         <h4 class="modal-title w-100 font-weight-bold">Search</h4>
    582. 

  582.         <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
    583. 

  583.           <span aria-hidden="true">&times;</span>
    584. 

  584.         </button>
    585. 

  585.       </div>
    586. 

  586.       <div class="modal-body mx-3">
    587. 

  587.         <div class="md-form mb-5">
    588. 

  588.           <input type="text" id="local-search-input" class="form-control validate">
    589. 

  589.           <label data-error="x" data-success="v" for="local-search-input">Keyword</label>
    590. 

  590.         </div>
    591. 

  591.         <div class="list-group" id="local-search-result"></div>
    592. 

  592.       </div>
    593. 

  593.     </div>
    594. 

  594.   </div>
    595. 

  595. </div>
    596. 

  596. 

  597.     
    598. 

  598. 

  599.     
    600. 

  600.   </main>
    601. 

  601. 

  602.   <footer>
    603. 

  603.     <div class="footer-inner">
    604. 

  604.   
    605. 

  605.     <div class="footer-content">
    606. 

  606.        <a target="_blank" rel="nofollow noopener" href="http://www.beian.gov.cn/portal/registerSystemInfo?recordcode=43130202000203"><img src="https://img.limour.top/2023/08/27/64eadeb81d6a0.webp" srcset="https://jscdn.limour.top/gh/Limour-dev/Sakurairo_Vision/load_svg/inload.svg" lazyload>湘公网安备43130202000203号  </a> <a target="_blank" rel="nofollow noopener" href="https://beian.miit.gov.cn/">湘ICP备20008299号 </a> <a target="_blank" rel="nofollow noopener" href="https://icp.gov.moe/?keyword=20210128">萌ICP备20210128号</a> <br> <a href="https://www.foreverblog.cn/" target="_blank"> <img src="https://img.foreverblog.cn/logo_en_default.png" srcset="https://jscdn.limour.top/gh/Limour-dev/Sakurairo_Vision/load_svg/inload.svg" lazyload alt="" style="width:auto;height:24px"> </a> <br> <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/limour-blog/limour-blog.github.io" target="_blank" rel="nofollow noopener"><span>SRC</span></a> <i class="iconfont icon-love"></i> <a href="https://web.archive.org/web/20231130095837/https://effectiveacceleration.tech/" target="_blank" rel="nofollow noopener"><span>e/Acc</span></a> 
    607. 

  607.     </div>
    608. 

  608.   
    609. 

  609.   
    610. 

  610.   
    611. 

  611. </div>
    612. 

  612. 

  613.   </footer>
    614. 

  614. 

  615.   <!-- Scripts -->
    616. 

  616.   
    617. 

  617.   <script  src="https://jscdn.limour.top/npm/nprogress@0.2.0/nprogress.min.js" ></script>
    618. 

  618.   <link  rel="stylesheet" href="https://jscdn.limour.top/npm/nprogress@0.2.0/nprogress.min.css" />
    619. 

  619. 

  620.   <script>
    621. 

  621.     NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    622. 

  622.     NProgress.start()
    623. 

  623.     window.addEventListener('load', function() {
    624. 

  624.       NProgress.done();
    625. 

  625.     })
    626. 

  626.   </script>
    627. 

  627. 

  628. 

  629. <script  src="https://jscdn.limour.top/npm/jquery@3.6.4/dist/jquery.min.js" ></script>
    630. 

  630. <script  src="https://jscdn.limour.top/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js" ></script>
    631. 

  631. <script  src="/js/events.js" ></script>
    632. 

  632. <script  src="/js/plugins.js" ></script>
    633. 

  633. 

  634. 

  635. 

  636. 

  637. 

  638.   
    639. 

  639.     <script  src="/js/img-lazyload.js" ></script>
    640. 

  640.   
    641. 

  641. 

  642. 

  643. 

  644. 

  645.   
    646. 

  646. <script>
    647. 

  647.   Fluid.utils.createScript('https://jscdn.limour.top/npm/tocbot@4.20.1/dist/tocbot.min.js', function() {
    648. 

  648.     var toc = jQuery('#toc');
    649. 

  649.     if (toc.length === 0 || !window.tocbot) { return; }
    650. 

  650.     var boardCtn = jQuery('#board-ctn');
    651. 

  651.     var boardTop = boardCtn.offset().top;
    652. 

  652. 

  653.     window.tocbot.init(Object.assign({
    654. 

  654.       tocSelector     : '#toc-body',
    655. 

  655.       contentSelector : '.markdown-body',
    656. 

  656.       linkClass       : 'tocbot-link',
    657. 

  657.       activeLinkClass : 'tocbot-active-link',
    658. 

  658.       listClass       : 'tocbot-list',
    659. 

  659.       isCollapsedClass: 'tocbot-is-collapsed',
    660. 

  660.       collapsibleClass: 'tocbot-is-collapsible',
    661. 

  661.       scrollSmooth    : true,
    662. 

  662.       includeTitleTags: true,
    663. 

  663.       headingsOffset  : -boardTop,
    664. 

  664.     }, CONFIG.toc));
    665. 

  665.     if (toc.find('.toc-list-item').length > 0) {
    666. 

  666.       toc.css('visibility', 'visible');
    667. 

  667.     }
    668. 

  668. 

  669.     Fluid.events.registerRefreshCallback(function() {
    670. 

  670.       if ('tocbot' in window) {
    671. 

  671.         tocbot.refresh();
    672. 

  672.         var toc = jQuery('#toc');
    673. 

  673.         if (toc.length === 0 || !tocbot) {
    674. 

  674.           return;
    675. 

  675.         }
    676. 

  676.         if (toc.find('.toc-list-item').length > 0) {
    677. 

  677.           toc.css('visibility', 'visible');
    678. 

  678.         }
    679. 

  679.       }
    680. 

  680.     });
    681. 

  681.   });
    682. 

  682. </script>
    683. 

  683. 

  684. 

  685.   <script src=https://lib.baomitu.com/clipboard.js/2.0.11/clipboard.min.js></script>
    686. 

  686. 

  687.   <script>Fluid.plugins.codeWidget();</script>
    688. 

  688. 

  689. 

  690.   
    691. 

  691. <script>
    692. 

  692.   Fluid.utils.createScript('https://jscdn.limour.top/npm/anchor-js@4.3.1/anchor.min.js', function() {
    693. 

  693.     window.anchors.options = {
    694. 

  694.       placement: CONFIG.anchorjs.placement,
    695. 

  695.       visible  : CONFIG.anchorjs.visible
    696. 

  696.     };
    697. 

  697.     if (CONFIG.anchorjs.icon) {
    698. 

  698.       window.anchors.options.icon = CONFIG.anchorjs.icon;
    699. 

  699.     }
    700. 

  700.     var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    701. 

  701.     var res = [];
    702. 

  702.     for (var item of el) {
    703. 

  703.       res.push('.markdown-body > ' + item.trim());
    704. 

  704.     }
    705. 

  705.     if (CONFIG.anchorjs.placement === 'left') {
    706. 

  706.       window.anchors.options.class = 'anchorjs-link-left';
    707. 

  707.     }
    708. 

  708.     window.anchors.add(res.join(', '));
    709. 

  709. 

  710.     Fluid.events.registerRefreshCallback(function() {
    711. 

  711.       if ('anchors' in window) {
    712. 

  712.         anchors.removeAll();
    713. 

  713.         var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    714. 

  714.         var res = [];
    715. 

  715.         for (var item of el) {
    716. 

  716.           res.push('.markdown-body > ' + item.trim());
    717. 

  717.         }
    718. 

  718.         if (CONFIG.anchorjs.placement === 'left') {
    719. 

  719.           anchors.options.class = 'anchorjs-link-left';
    720. 

  720.         }
    721. 

  721.         anchors.add(res.join(', '));
    722. 

  722.       }
    723. 

  723.     });
    724. 

  724.   });
    725. 

  725. </script>
    726. 

  726. 

  727. 

  728.   <script>Fluid.plugins.imageCaption();</script>
    729. 

  729. 

  730.   <script  src="/js/local-search.js" ></script>
    731. 

  731. 

  732. 

  733. 

  734. 

  735. 

  736. <!-- 主题的启动项，将它保持在最底部 -->
    737. 

  737. <!-- the boot of the theme, keep it at the bottom -->
    738. 

  738. <script  src="/js/boot.js" ></script>
    739. 

  739. 

  740. 

  741.   
    742. 

  742. 

  743.   <noscript>
    744. 

  744.     <div class="noscript-warning">Blog works best with JavaScript enabled</div>
    745. 

  745.   </noscript>
    746. 

  746. <!-- hexo injector body_end start -->
    747. 

  747.   <script defer src="/theme-inject/timeliness.js"></script>
    748. 

  748. <!-- hexo injector body_end end --></body>
    749. 

  749. </html>
    750.