-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathindex.html
More file actions
1371 lines (971 loc) · 81.6 KB
/
Copy pathindex.html
File metadata and controls
1371 lines (971 loc) · 81.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 6.3.0">
<link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
<link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
<link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
<link rel="mask-icon" href="/images/logo.svg" color="#222">
<meta name="google-site-verification" content="IuBNKxWYReEztWCHo-65FPRKh6hr4eEu2OdizQdEg0c">
<meta name="baidu-site-verification" content="code-X3mIYM4Zeg">
<link rel="stylesheet" href="/css/main.css">
<link rel="stylesheet" href="https://fonts.lug.ustc.edu.cn/css?family=Noto Serif SC:300,300italic,400,400italic,700,700italic|Lobster Two:300,300italic,400,400italic,700,700italic|EB Garamond:300,300italic,400,400italic,700,700italic|Roboto Slab:300,300italic,400,400italic,700,700italic|Source Code Pro:300,300italic,400,400italic,700,700italic&display=swap&subset=latin,latin-ext">
<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/gh/fancyapps/fancybox@3/dist/jquery.fancybox.min.css">
<link rel="stylesheet" href="/lib/pace/pace-theme-mac-osx.min.css">
<script src="/lib/pace/pace.min.js"></script>
<script id="hexo-configurations">
var NexT = window.NexT || {};
var CONFIG = {"hostname":"mapicccy.github.io","root":"/","scheme":"Pisces","version":"7.8.0","exturl":false,"sidebar":{"position":"left","width":240,"display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":true,"show_result":false,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":true},"bookmark":{"enable":true,"color":"#222","save":"auto"},"fancybox":true,"mediumzoom":false,"lazyload":true,"pangu":true,"comments":{"style":"tabs","active":"gitalk","storage":true,"lazyload":false,"nav":null,"activeClass":"gitalk"},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":true,"trigger":"auto","top_n_per_article":3,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
</script>
<meta property="og:type" content="website">
<meta property="og:title" content="mapicccy">
<meta property="og:url" content="https://mapicccy.github.io/index.html">
<meta property="og:site_name" content="mapicccy">
<meta property="og:locale" content="zh_CN">
<meta property="article:author" content="mapicccy">
<meta name="twitter:card" content="summary">
<link rel="canonical" href="https://mapicccy.github.io/">
<script id="page-configurations">
// https://hexo.io/docs/variables.html
CONFIG.page = {
sidebar: "",
isHome : true,
isPost : false,
lang : 'zh-CN'
};
</script>
<title>mapicccy</title>
<script async src="https://www.googletagmanager.com/gtag/js?id=G-JBNFVGV93B"></script>
<script data-pjax>
if (CONFIG.hostname === location.hostname) {
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-JBNFVGV93B');
}
</script>
<script data-pjax>
var _hmt = _hmt || [];
(function() {
var hm = document.createElement("script");
hm.src = "https://hm.baidu.com/hm.js?85b9fb02b9e2859d61f68382e3dcb1c1";
var s = document.getElementsByTagName("script")[0];
s.parentNode.insertBefore(hm, s);
})();
</script>
<noscript>
<style>
.use-motion .brand,
.use-motion .menu-item,
.sidebar-inner,
.use-motion .post-block,
.use-motion .pagination,
.use-motion .comments,
.use-motion .post-header,
.use-motion .post-body,
.use-motion .collection-header { opacity: initial; }
.use-motion .site-title,
.use-motion .site-subtitle {
opacity: initial;
top: initial;
}
.use-motion .logo-line-before i { left: initial; }
.use-motion .logo-line-after i { right: initial; }
</style>
</noscript>
<link rel="alternate" href="/atom.xml" title="mapicccy" type="application/atom+xml">
</head>
<body itemscope itemtype="http://schema.org/WebPage">
<div class="container use-motion">
<div class="headband"></div>
<header class="header" itemscope itemtype="http://schema.org/WPHeader">
<div class="header-inner"><div class="site-brand-container">
<div class="site-nav-toggle">
<div class="toggle" aria-label="切换导航栏">
<span class="toggle-line toggle-line-first"></span>
<span class="toggle-line toggle-line-middle"></span>
<span class="toggle-line toggle-line-last"></span>
</div>
</div>
<div class="site-meta">
<a href="/" class="brand" rel="start">
<span class="logo-line-before"><i></i></span>
<h1 class="site-title">mapicccy</h1>
<span class="logo-line-after"><i></i></span>
</a>
<p class="site-subtitle" itemprop="description">男人有实力，才有魅力</p>
</div>
<div class="site-nav-right">
<div class="toggle popup-trigger">
<i class="fa fa-search fa-fw fa-lg"></i>
</div>
</div>
</div>
<nav class="site-nav">
<ul id="menu" class="main-menu menu">
<li class="menu-item menu-item-home">
<a href="/" rel="section"><i class="fa fa-home fa-fw"></i>首页</a>
</li>
<li class="menu-item menu-item-about">
<a href="/about/" rel="section"><i class="fa fa-user fa-fw"></i>关于</a>
</li>
<li class="menu-item menu-item-tags">
<a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>标签</a>
</li>
<li class="menu-item menu-item-categories">
<a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>分类</a>
</li>
<li class="menu-item menu-item-archives">
<a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>归档</a>
</li>
<li class="menu-item menu-item-sitemap">
<a href="/sitemap.xml" rel="section"><i class="fa fa-sitemap fa-fw"></i>站点地图</a>
</li>
<li class="menu-item menu-item-search">
<a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
</a>
</li>
</ul>
</nav>
<div class="search-pop-overlay">
<div class="popup search-popup">
<div class="search-header">
<span class="search-icon">
<i class="fa fa-search"></i>
</span>
<div class="search-input-container">
<input autocomplete="off" autocapitalize="off"
placeholder="搜索..." spellcheck="false"
type="search" class="search-input">
</div>
<span class="popup-btn-close">
<i class="fa fa-times-circle"></i>
</span>
</div>
<div id="search-result">
<div id="no-result">
<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
</div>
</div>
</div>
</div>
</div>
</header>
<div class="back-to-top">
<i class="fa fa-arrow-up"></i>
<span>0%</span>
</div>
<div class="reading-progress-bar"></div>
<a role="button" class="book-mark-link book-mark-link-fixed"></a>
<main class="main">
<div class="main-inner">
<div class="content-wrap">
<div class="content index posts-expand">
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/538b6f3e/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/538b6f3e/" class="post-title-link" itemprop="url">The Llama 3 Herd of Models——论文翻译</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2024-07-29 04:54:58 / 修改时间：05:22:31" itemprop="dateCreated datePublished" datetime="2024-07-29T04:54:58+00:00">2024-07-29</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>6.7k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>11 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p>Llama 3 模型群 —— Llama 团队，Meta 的 AI 分支</p>
<p>现代人工智能（AI）系统的核心驱动力是基础模型。本文介绍了一组名为 Llama 3 的全新基础模型系列。这是一群原生支持多语言能力、编程、推理以及工具使用的语言模型。我们最大的模型是一个拥有 4050 亿参数和最多可达 12.8 万令牌上下文窗口的密集型 Transformer。本文详细展示了 Llama 3 的大规模实证评估。我们发现，Llama 3 在众多任务上提供了可与顶尖语言模型如 GPT-4 相媲美的质量表现。我们公开发布了 Llama 3，包括预训练和后训练版本的 4050 亿参数语言模型，以及我们的 Llama Guard 3 模型，用于输入和输出的安全保障。</p>
<p>此外，本文还展示了我们通过组合方法将图像、视频和语音能力融入 Llama 3 的实验结果。我们观察到，这种方法在图像、视频和语音识别任务上与最先进水平竞争表现优异。然而，这些融合了多媒体能力的模型目前尚未广泛发布，因为它们仍在开发阶段。</p>
<h3 id="1-引言"><a href="#1-引言" class="headerlink" title="1 引言"></a>1 引言</h3><p>基础模型是指那些通用的语言、视觉、语音或其他模态的模型，旨在支撑广泛的人工智能任务。它们构成了许多现代AI系统的基石。</p>
<p>现代基础模型的发展历经两大核心阶段：(1) 预训练阶段，模型通过大规模数据集进行训练，采用诸如下一个词预测或图片描述等直接任务；(2) 后训练阶段，模型被微调以遵循指令、与人类偏好对齐及提升特定能力，例如编程和推理。</p>
<p>本文介绍了一组新的语言基础模型——Llama 3系列。Llama 3模型群原生支持多语言、编程、推理及工具运用能力。其中最大的模型是一个密集型Transformer架构，拥有4050亿参数，能在长达12.8万个令牌的上下文窗口中处理信息。模型群的每个成员在表1中列出。本文所展示的所有成果均基于Llama 3.1模型，为简洁起见，我们将统一简称其为Llama 3。</p>
<p>我们认为，高质量基础模型的开发有三个关键要素：数据、规模以及复杂度管理。在开发过程中，我们致力于针对这三个要素进行优化：</p>
<ul>
<li><p><strong>数据</strong>：相较于Llama的早期版本（Touvron等人，2023a,b），我们在预训练和后训练中使用的数据量和质量都有所提升。这些改进包括开发更加细致的预处理和精选管道以优化预训练数据，以及建立更严格的品质保证和过滤方法来处理后训练数据。Llama 3的预训练基于约1.5万亿个多语言令牌的数据集，相比之下，Llama 2使用的数据集为1.8万亿令牌。</p>
</li>
<li><p><strong>规模</strong>: 我们在远超以往Llama模型的规模上进行了训练：我们的旗舰语言模型预训练消耗了大约3.8×10^25次浮点运算（FLOPs），几乎是Llama 2最大版本的近50倍。具体而言，我们对一个拥有4050亿可训练参数的旗舰模型进行了预训练，使用了1.56万亿文本令牌的数据集。正如基础模型的规模法则所预期的那样，我们的旗舰模型在相同的训练流程下超越了规模较小的模型。尽管规模法则表明，对于我们的训练预算，旗舰模型大约处于计算最优的规模，但我们对小型模型的训练时间也远超计算最优时长。由此产生的模型在相同的推理预算下，表现优于计算最优模型。我们还利用旗舰模型在后训练阶段进一步提升了这些小型模型的质量。</p>
</li>
<li><p><strong>管理复杂性</strong>: 我们在设计选择上力求最大化模型开发过程的可扩展性。例如，我们选择了标准的密集型Transformer模型架构（Vaswani等人，2017年），仅做轻微调整，而不是混合专家模型（MoE，Shazeer等人，2017年），以此来最大限度地提高训练稳定性。同样地，我们采用了相对简单的后训练流程，基于监督微调（SFT）、拒绝采样（RS）和直接偏好优化（DPO，Rafailov等人，2023年），而非采用更复杂的强化学习算法（如Ouyang等人，2022年；Schulman等人，2017年），后者往往稳定性较差且难以规模化。</p>
</li>
</ul>
<p>我们的工作成果是Llama 3：一组包含三个多语言模型的“羊驼群”，分别具有80亿、700亿和4050亿个参数。我们对Llama 3在涵盖广泛语言理解任务的关键基准数据集上的性能进行了评估。此外，我们还进行了深入的人类评估，将Llama 3与竞争对手的模型进行比较。旗舰Llama 3模型在关键基准上的性能概览见表2。实验评估表明，我们的旗舰模型在多种任务上与诸如GPT-4（OpenAI，2023a）等领先语言模型的表现相当，并接近达到当前技术前沿。我们的小型模型也是同类最佳，优于具有相似参数数量的替代模型（Bai等人，2023；Jiang等人，2023）。Llama 3相比其前身在有用性和无害性之间取得了更好的平衡（Touvron等人，2023b）。我们将在第5.4节详细介绍Llama 3的安全性分析。</p>
<p>我们正根据更新版的Llama 3社区许可协议公开发布所有三个Llama 3模型；详情请见<a target="_blank" rel="noopener" href="https://llama.meta.com.这包括我们4050亿参数语言模型的预训练和后训练版本,以及输入和输出安全性的llama/">https://llama.meta.com。这包括我们4050亿参数语言模型的预训练和后训练版本，以及输入和输出安全性的Llama</a> Guard模型新版本（Inan等人，2023）。我们希望通过旗舰模型的开放发布激发研究界的创新浪潮，并加速负责任地迈向人工智能（AGI）总体发展的道路。</p>
<p>作为Llama 3开发过程的一部分，我们还开发了模型的多模态扩展，使模型具备图像识别、视频识别和语音理解的能力。这些多模态模型仍在积极开发中，尚未准备发布。除了语言建模的结果外，本文还展示了我们使用这些多模态模型的初步实验结果。</p>
<h3 id="2-总体概述"><a href="#2-总体概述" class="headerlink" title="2 总体概述"></a>2 总体概述</h3><p>Llama 3的模型架构如图1所示。我们Llama 3语言模型的开发包含两个主要阶段：</p>
<h4 id="语言模型预训练"><a href="#语言模型预训练" class="headerlink" title="语言模型预训练"></a>语言模型预训练</h4><p>我们首先将大型多语言文本语料库转换为离散的令牌，并在此基础上预训练一个大型语言模型（LLM），以执行下一个令牌的预测任务。在语言模型预训练阶段，模型学习语言结构，并从其“阅读”的文本中获取大量关于世界的知识。为了有效实现这一目标，预训练是在大规模数据上进行的：我们使用8K令牌的上下文窗口，在15.6万亿个令牌的数据集上预训练一个拥有4050亿参数的模型。这一标准预训练阶段之后是持续预训练阶段，将支持的上下文窗口扩大到128K令牌。详情请参阅第3节。</p>
<h4 id="语言模型后训练"><a href="#语言模型后训练" class="headerlink" title="语言模型后训练"></a>语言模型后训练</h4><p>预训练后的语言模型虽然对语言有了丰富的理解，但还不能遵循指令或像我们期望的助手那样行为。我们通过几轮人类反馈对其校准，每轮涉及在指令调优数据上的监督微调（SFT）和直接偏好优化（DPO，Rafailov等人，2024）。在这一后训练阶段，我们还整合了新能力，比如工具使用，并在编码、推理等领域观察到了显著的进步。详情请见第4节。最后，安全缓解措施也在后训练阶段被整合进模型中，具体细节在第5.4节描述。</p>
<p>最终产生的模型具备一系列丰富的能力。它们能够至少用八种语言回答问题、编写高质量的代码、解决复杂的推理问题，并能开箱即用或以零样本方式使用工具。</p>
<p>我们还进行了实验，通过一种组合方法向Llama 3添加了图像、视频和语音处理能力。我们研究的方法包括图28中所示的三个额外阶段：</p>
<h4 id="多模态编码器预训练"><a href="#多模态编码器预训练" class="headerlink" title="多模态编码器预训练"></a>多模态编码器预训练</h4><p>我们分别为图像和语音训练独立的编码器。我们的图像编码器在大量的图像-文本对上进行训练，这教会模型视觉内容与其自然语言描述之间的关系。语音编码器则采用自监督方法进行训练，该方法会遮蔽语音输入的部分内容，并尝试通过离散令牌表示来重建被遮蔽的部分，从而使模型学习到语音信号的结构。图像编码器的详细信息请参见第7节，语音编码器的详细信息请参见第8节。</p>
<h4 id="视觉适配器训练"><a href="#视觉适配器训练" class="headerlink" title="视觉适配器训练"></a>视觉适配器训练</h4><p>我们训练了一个适配器，将预训练的图像编码器集成到预训练的语言模型中。该适配器包含一系列交叉注意力层，用于将图像编码器的表示馈入语言模型。适配器在文本-图像对上进行训练，这使得图像表示与语言表示对齐。在适配器训练期间，我们也会更新图像编码器的参数，但有意不更新语言模型的参数。我们还在图像适配器的基础上，使用配对的视频-文本数据训练了一个视频适配器，使模型能够在帧间聚合信息。更多细节请参阅第7节。</p>
<h4 id="语音适配器训练"><a href="#语音适配器训练" class="headerlink" title="语音适配器训练"></a>语音适配器训练</h4><p>最后，我们通过一个适配器将语音编码器集成到模型中，该适配器将语音编码转换为可以直接输入到微调过的语言模型中的令牌表示。适配器和编码器的参数在监督微调阶段联合更新，以实现高质量的语音理解能力。在语音适配器训练期间，我们不对语言模型进行更改。我们还集成了一个文本转语音系统。详细信息请参见第8节。</p>
<p>我们的多模态实验产生了能够识别图像和视频内容，并通过语音接口支持交互的模型。这些模型仍处于开发阶段，尚未准备好发布。</p>
<h3 id="3-预训练"><a href="#3-预训练" class="headerlink" title="3 预训练"></a>3 预训练</h3><p>语言模型预训练包括以下步骤：(1) 大规模训练语料库的策划与过滤，(2) 模型架构的开发及确定模型大小的相应规模法则，(3) 大规模高效预训练技术的开发，以及(4) 预训练方案的制定。下面我们将逐一介绍这些组成部分。</p>
<h4 id="3-1-预训练数据"><a href="#3-1-预训练数据" class="headerlink" title="3.1 预训练数据"></a>3.1 预训练数据</h4><p>我们从包含截至2023年底知识的多种数据源创建语言模型预训练数据集。对每个数据源应用多种去重方法和数据清洗机制，以获得高质量的令牌。我们移除含有大量个人可识别信息(PII)的领域，以及已知含有成人内容的领域。</p>
<h4 id="3-1-1-网络数据策划"><a href="#3-1-1-网络数据策划" class="headerlink" title="3.1.1 网络数据策划"></a>3.1.1 网络数据策划</h4><p>我们使用的大部分数据来源于网络，以下是我们的数据清洗流程。</p>
<h5 id="PII和安全过滤"><a href="#PII和安全过滤" class="headerlink" title="PII和安全过滤"></a>PII和安全过滤</h5><p>除其他缓解措施外，我们实施了旨在移除可能含有不安全内容或大量PII的网站数据的过滤器，移除按照Meta安全标准被评为有害的域名，以及已知含有成人内容的域名。</p>
<h5 id="文本提取与清洗"><a href="#文本提取与清洗" class="headerlink" title="文本提取与清洗"></a>文本提取与清洗</h5><p>我们处理非截断网页文档的原始HTML内容，以提取高质量且多样化的文本。为此，我们构建了一个自定义解析器，用于提取HTML内容，并优化去除模板内容的精确度和内容召回率。我们通过人工评估来验证解析器的质量，将其与优化文章类内容的第三方HTML解析器进行对比，结果表现良好。我们仔细处理含有数学和代码内容的HTML页面，以保留这类内容的结构。我们保留图像alt属性文本，因为在数学内容常以预渲染图像形式出现时，数学内容也会在alt属性中提供。我们实验性地评估了不同的清洗配置，发现相比于纯文本，Markdown格式对主要在网页数据上训练的模型性能不利，因此我们移除了所有Markdown标记。</p>
<h5 id="去重"><a href="#去重" class="headerlink" title="去重"></a>去重</h5><p>我们在URL、文档和行级别上应用了几轮去重：</p>
<ul>
<li>URL级去重：在整个数据集中执行URL级别的去重，为每个URL对应的页面保留最新版本。</li>
<li>文档级去重：在整个数据集上执行全局MinHash（Broder, 1997）去重，移除近似重复的文档。</li>
<li>行级去重：执行类似于ccNet（Wenzek等人，2019）的激进行列级去重。移除在每个3000万文档桶中出现超过6次的行。尽管手动定性分析显示，行级去重不仅去除了来自各网站的剩余模板内容（如导航菜单、Cookie警告），也可能移除了频繁出现的高质量文本，但我们的实证评估显示了显著的性能提升。</li>
</ul>
<h5 id="启发式过滤"><a href="#启发式过滤" class="headerlink" title="启发式过滤"></a>启发式过滤</h5><p>我们开发启发式方法来删除更多低质量文档、异常值和重复内容过多的文档。一些启发式例子包括：</p>
<ul>
<li>我们采用重复n-gram覆盖率比例（Rae等人，2021）来删除包含重复内容如日志或错误信息的行。这些行可能非常长且独特，因此无法通过行去重过滤。</li>
<li>我们使用“脏词”计数（Raffel等人，2020）来过滤不受域名屏蔽列表覆盖的成人网站。</li>
<li>我们使用令牌分布的Kullback-Leibler散度来过滤与训练语料库分布相比含有过多异常令牌的文档。</li>
</ul>
<h5 id="基于模型的质量过滤"><a href="#基于模型的质量过滤" class="headerlink" title="基于模型的质量过滤"></a>基于模型的质量过滤</h5><p>我们进一步尝试应用多种基于模型的质量分类器来筛选高质量的文本片段。这包括使用快速分类器，如fasttext（Joulin等人，2017年），该分类器被训练以识别给定文本是否会被维基百科引用（Touvron等人，2023a），以及计算密集型的Roberta-based分类器（Liu等人，2019a），这些分类器在Llama 2的预测数据上进行训练。为了基于Llama 2训练一个质量分类器，我们创建了一个经过清理的网页文档训练集，并描述了质量要求，然后指导Llama 2的聊天模型判断文档是否满足这些要求。出于效率考虑，我们使用DistilRoberta（Sanh等人，2019年）为每个文档生成质量分数。我们通过实验评估了不同质量过滤配置的有效性。</p>
<h5 id="代码与推理数据处理"><a href="#代码与推理数据处理" class="headerlink" title="代码与推理数据处理"></a>代码与推理数据处理</h5><p>类似于DeepSeek-AI等（2024年）的工作，我们构建了针对特定领域的管道，用于提取包含代码和数学相关内容的网页。具体而言，无论是代码还是推理分类器，都是基于DistilledRoberta模型，该模型在由Llama 2标注的网络数据上进行训练。与上述通用质量分类器不同，我们进行了提示调整，以便针对性地搜索包含数学推导、STEM领域中的推理以及自然语言中穿插的代码的网页。鉴于代码和数学的令牌分布与自然语言有显著差异，这些管道实现了领域特定的HTML抽取、定制的文本特征和过滤启发式规则。</p>
<h5 id="多语言数据处理"><a href="#多语言数据处理" class="headerlink" title="多语言数据处理"></a>多语言数据处理</h5><p>与上述针对英语的数据处理流程相仿，我们也实施了过滤机制，以移除可能包含个人身份信息（PII）或不安全内容的网站数据。我们的多语言文本处理管道具备以下独特特性：</p>
<p>• 利用基于fasttext的语言识别模型，将文档归类到176种语言中。<br>• 针对每种语言的数据，分别在文档级别和行级别执行去重复操作。<br>• 应用针对各语言的特定启发式规则及基于模型的过滤器，以排除低质量文档。</p>
<p>此外，我们采用基于多语言Llama 2的分类器对多语言文档进行质量排序，确保优先考虑高质量内容。通过实验确定预训练中使用的多语言令牌数量，以平衡模型在英语和多语言基准测试上的性能表现。</p>
<h4 id="3-1-2-数据组合的确定"><a href="#3-1-2-数据组合的确定" class="headerlink" title="3.1.2 数据组合的确定"></a>3.1.2 数据组合的确定</h4><p>为了获得高质量的语言模型，仔细决定预训练数据混合中不同数据来源的比例至关重要。我们确定此数据混合的主要工具是知识分类和规模法则实验。</p>
<p><strong>知识分类。</strong> 我们开发了一种分类器，用于分类网络数据中所含信息的类型，以便更有效地确定数据组合。我们利用该分类器对网络上过度代表的数据类别进行下采样，例如艺术与娱乐类内容。</p>
<p><strong>数据混合的规模法则。</strong> 为确定最佳数据组合，我们执行规模法则实验，即在不同的数据组合上训练多个小型模型，并以此预测大型模型在该组合上的性能（参见第3.2.1节）。我们为不同的数据组合多次重复这一过程，以选择新的数据组合候选。随后，在这个候选数据组合上训练一个更大的模型，并评估其在几个关键基准测试上的表现。</p>
<p><strong>数据组合概述。</strong> 最终的数据组合大约包含50%的一般知识令牌，25%的数学与推理令牌，17%的代码令牌，以及8%的多语言令牌。</p>
<h4 id="3-1-3-数据退火"><a href="#3-1-3-数据退火" class="headerlink" title="3.1.3 数据退火"></a>3.1.3 数据退火</h4><p>实验上，我们发现对少量高质量的代码和数学数据进行退火（参见第3.4.3节）可以提升预训练模型在关键基准上的性能。类似于Li等人（2024b），我们在特定领域内对高质量数据进行上采样的数据混合上执行退火。我们的退火数据中不包含任何来自常用基准测试的训练集。这使我们能够评估Llama 3真正的少量样本学习能力和跨领域的泛化能力。</p>
<p>效仿OpenAI（2023a），我们在GSM8k（Cobbe等人，2021）和MATH（Hendrycks等人，2021b）训练集上进行退火，并评估退火的效果。我们发现，退火使预训练的Llama 3 8B模型在GSM8k和MATH验证集上的性能分别提高了24.0%和6.4%。然而，对于405B模型，改进微乎其微，表明我们的旗舰模型拥有强大的上下文内学习和推理能力，不需要特定领域的训练样本即可获得强劲的性能。</p>
<p><strong>利用退火评估数据质量。</strong> 类似于Blakeney等人（2024），我们发现退火使我们能够评判小规模领域特定数据集的价值。我们通过线性地将一个已训练50%的Llama 3 8B模型的学习率退火至0，处理400亿个令牌，以此来衡量这类数据集的价值。在这些实验中，新数据集被赋予30%的权重，剩余70%的权重分配给默认数据混合。相比为每个小数据集执行规模法则实验，使用退火评估新数据源更为高效。</p>
<h3 id="3-2-模型架构"><a href="#3-2-模型架构" class="headerlink" title="3.2 模型架构"></a>3.2 模型架构</h3><p>Llama 3采用了标准的稠密Transformer架构（Vaswani等人，2017年）。就模型架构而言，它与Llama及Llama 2（Touvron等人，2023a，b）并无显著偏离；我们的性能提升主要得益于数据质量和多样性的改进，以及训练规模的扩大。</p>
<p>相较于Llama 3，我们做出了一些较小的修改：</p>
<p>• 我们采用了分组查询注意力（Grouped Query Attention，简称GQA；Ainslie等人，2023年），并设置了8个键值头，以提高推理速度并减少解码期间键值缓存的大小。<br>• 我们使用了一种注意力掩码，防止序列中不同文档之间的自注意力。我们发现在标准预训练过程中，这一改动的影响有限，但在针对非常长序列的持续预训练中，我们发现它非常重要。</p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/db9f248e/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/db9f248e/" class="post-title-link" itemprop="url">ARM64使用kernel 6.6.7启动日志</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2024-03-05 04:54:01 / 修改时间：04:56:05" itemprop="dateCreated datePublished" datetime="2024-03-05T04:54:01+00:00">2024-03-05</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>46k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>1:17</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p>– BIOS boot done –<br>[ 0.000000] Booting Linux on physical CPU 0x0100000000 [0x410fd8e0]<br>[ 0.000000] Linux version 6.6.7+ (<a href="mailto:root@j66e01291.sqa.eu95">root@j66e01291.sqa.eu95</a>) (gcc (GCC) 10.2.1 20200825 (Alibaba 10.2.1-3 2.17), GNU ld version 2.35-12.2.alios7) #13 SMP Wed Feb 21 15:11:08 CST 2024<br>[ 0.000000] KASLR disabled due to lack of seed<br>[ 0.000000] earlycon: pl11 at MMIO 0x000000002a401000 (options ‘’)<br>[ 0.000000] printk: bootconsole [pl11] enabled<br>[ 0.000000] efi: need at least one option<br>[ 0.000000] Malformed early option ‘efi’<br>[ 0.000000] efi: EFI v2.7 by EDK II<br>[ 0.000000] efi: SMBIOS 3.0=0xf9810000 ACPI 2.0=0xf4530018 MEMATTR=0xf7ca7018 INITRD=0xf40f7f18 MEMRESERVE=0xf40f7f98<br>[ 0.000000] ACPI: Early table checksum verification disabled<br>[ 0.000000] ACPI: RSDP 0x00000000F4530018 000024 (v02 PTG )<br>[ 0.000000] ACPI: XSDT 0x00000000F453FE98 000094 (v01 PTG PTG01 00000000 “ “ 01000013)<br>[ 0.000000] ACPI: FACP 0x00000000F453FB98 000114 (v06 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: DSDT 0x00000000F4537518 006A14 (v02 PTG PTG01 00000000 INTL 20230331)<br>[ 0.000000] ACPI: AGDI 0x00000000F453FF98 000030 (v00 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: DBG2 0x00000000F453FA98 00005E (v00 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: GTDT 0x00000000F453E998 000104 (v03 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: MCFG 0x00000000F453FE18 00003C (v01 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: SLIT 0x00000000F453F998 00003C (v01 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: SPCR 0x00000000F453FA18 000050 (v02 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: SDEI 0x00000000F453FD18 000024 (v01 PTG PTG01 00000000 INTL 20230331)<br>[ 0.000000] ACPI: APIC 0x00000000F453F598 000334 (v05 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: SRAT 0x00000000F453EB18 000140 (v03 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: IORT 0x00000000F453F318 00012C (v00 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: PPTT 0x00000000F453EC98 0003E6 (v02 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: MPAM 0x00000000F4530098 0004A4 (v00 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: SPMI 0x00000000F453FD98 000041 (v05 PTG PTG01 00000000 PTG 20200717)<br>[ 0.000000] ACPI: SPCR: console: pl011,mmio32,0x2a400000,115200<br>[ 0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x80000000-0xffffffff]<br>[ 0.000000] ACPI: SRAT: Node 0 PXM 0 [mem 0x900000000-0x187fffffff]<br>[ 0.000000] NUMA: NODE_DATA [mem 0x1877fc1b00-0x1877fc6fff]</p>
<!--noindex-->
<div class="post-button">
<a class="btn" href="/posts/db9f248e/#more" rel="contents">
阅读全文 »
</a>
</div>
<!--/noindex-->
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/4b2d89b0/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/4b2d89b0/" class="post-title-link" itemprop="url">dummy syscall test in linux kernel</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2023-07-27 06:39:18 / 修改时间：15:40:04" itemprop="dateCreated datePublished" datetime="2023-07-27T06:39:18+00:00">2023-07-27</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>1.4k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>2 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p>This is an example about dummy syscall test in linux kernel</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br></pre></td><td class="code"><pre><span class="line">#include <sys/utsname.h></span><br><span class="line">#include <errno.h></span><br><span class="line">#include <time.h></span><br><span class="line">#include <sched.h></span><br><span class="line">#include <stdalign.h></span><br><span class="line">#include <unistd.h></span><br><span class="line">#include <stdlib.h></span><br><span class="line">#include <stdio.h></span><br><span class="line">#include <syscall.h></span><br><span class="line">#include <signal.h></span><br><span class="line">#include <stdint.h></span><br><span class="line">#include <errno.h></span><br><span class="line">#include <string.h></span><br><span class="line">#include <sys/wait.h></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">int main(void)</span><br><span class="line">{</span><br><span class="line"> unsigned long i = 0;</span><br><span class="line"> int nr_mysyscall;</span><br><span class="line"> struct utsname buffer;</span><br><span class="line"> struct timespec tstart={0,0}, tend={0,0};</span><br><span class="line"></span><br><span class="line"> errno = 0;</span><br><span class="line"> if (uname(&buffer) != 0) {</span><br><span class="line"> perror("uname");</span><br><span class="line"> exit(EXIT_FAILURE);</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> if (buffer.release[0] == '5') {</span><br><span class="line"> printf("5.10 kernel\n");</span><br><span class="line"> clock_gettime(CLOCK_MONOTONIC, &tstart);</span><br><span class="line"> for (i=0; i<30000000; i++)</span><br><span class="line"> syscall(335);</span><br><span class="line"> clock_gettime(CLOCK_MONOTONIC, &tend);</span><br><span class="line"> printf("took about %.5f seconds\n",</span><br><span class="line"> ((double)tend.tv_sec + 1.0e-9*tend.tv_nsec) -</span><br><span class="line"> ((double)tstart.tv_sec + 1.0e-9*tstart.tv_nsec));</span><br><span class="line"> } else {</span><br><span class="line"> printf("4.19 kernel\n");</span><br><span class="line"> clock_gettime(CLOCK_MONOTONIC, &tstart);</span><br><span class="line"> for (i=0; i<30000000; i++)</span><br><span class="line"> syscall(438);</span><br><span class="line"> clock_gettime(CLOCK_MONOTONIC, &tend);</span><br><span class="line"> printf("took about %.5f seconds\n",</span><br><span class="line"> ((double)tend.tv_sec + 1.0e-9*tend.tv_nsec) -</span><br><span class="line"> ((double)tstart.tv_sec + 1.0e-9*tstart.tv_nsec));</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> return 0;</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/72968cc7/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/72968cc7/" class="post-title-link" itemprop="url">Prompts used with stable diffusion</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2023-04-05 13:14:36 / 修改时间：13:25:47" itemprop="dateCreated datePublished" datetime="2023-04-05T13:14:36+00:00">2023-04-05</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>3.6k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>6 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p>This article collects prompts for Stable Diffusion’s automatic drawing tool.</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">1girl, the whole body, standing, thin waist, slender waist, fully naked, (completely nude:1.2), collarbone, medium breasts, showing breasts, round breasts, nipples, glowing nipples, pussy, vaginal, no bra, no panties, no bikini, smile, outdoor, best quality ,masterpiece, illustration, an extremely delicate and beautiful, extremely detailed ,CG ,unity ,8k wallpaper, Amazing, finely detail, best quality, official art, extremely detailed CG unity 8k wallpaper, absurdres, huge filesize , ultra-detailed, highres, beautiful detailed girl, extremely detailed eyes and face, beautiful detailed eyes, light on face, realistic, perfect fingers, <perfectPussy_newb0001:1> , <breastinclassBetter_v13:0.5></span><br><span class="line"></span><br><span class="line">multiple breasts, (mutated hands and fingers:1.5 ), (long body :1.3), (mutation, poorly drawn :1.2) , black-white, bad anatomy, liquid body, liquid tongue, disfigured, malformed, mutated, anatomical nonsense, text font ui, error, malformed hands, long neck, blurred, lowers, lowres, bad anatomy, bad proportions, bad shadow, uncoordinated body, unnatural body, fused breasts, bad breasts, huge breasts, poorly drawn breasts, extra breasts, liquid breasts, heavy breasts, missing breasts, huge haunch, huge thighs, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, fused ears, bad ears, poorly drawn ears, extra ears, liquid ears, heavy ears, missing ears, fused animal ears, bad animal ears, poorly drawn animal ears, extra animal ears, liquid animal ears, heavy animal ears, missing animal ears, text, ui, error, missing fingers, missing limb, fused fingers, one hand with more than 5 fingers, one hand with less than 5 fingers, one hand with more than 5 digit, one hand with less than 5 digit, extra digit, fewer digits, fused digit, missing digit, bad digit, liquid digit, colorful tongue, black tongue, cropped, watermark, username, blurry, JPEG artifacts, signature, 3D, 3D game, 3D game scene, 3D character, malformed feet, extra feet, bad feet, poorly drawn feet, fused feet, missing feet, extra shoes, bad shoes, fused shoes, more than two shoes, poorly drawn shoes, bad gloves, poorly drawn gloves, fused gloves, bad cum, poorly drawn cum, fused cum, bad hairs, poorly drawn hairs, fused hairs, big muscles, ugly, bad face, fused face, poorly drawn face, cloned face, big face, long face, bad eyes, fused eyes poorly drawn eyes, extra eyes, malformed limbs, more than 2 nipples, missing nipples, different nipples, fused nipples, bad nipples, poorly drawn nipples, black nipples, colorful nipples, gross proportions. short arm, (((missing arms))), missing thighs, missing calf, missing legs, mutation, duplicate, morbid, mutilated, poorly drawn hands, more than 1 left hand, more than 1 right hand, deformed, (blurry), disfigured, missing legs, extra arms, extra thighs, more than 2 thighs, extra calf, fused calf, extra legs, bad knee, extra knee, more than 2 legs, bad tails, bad mouth, fused mouth, poorly drawn mouth, bad tongue, tongue within mouth, too long tongue, black tongue, big mouth, cracked mouth, bad mouth, dirty face, dirty teeth, dirty pantie, fused pantie, poorly drawn pantie, fused cloth, poorly drawn cloth, bad pantie, yellow teeth, thick lips, bad cameltoe, colorful cameltoe, bad asshole, poorly drawn asshole, fused asshole, missing asshole, bad anus, bad pussy, bad crotch, bad crotch seam, fused anus, fused pussy, fused anus, fused crotch, poorly drawn crotch, fused seam, poorly drawn anus, poorly drawn pussy, poorly drawn crotch, poorly drawn crotch seam, bad thigh gap, missing thigh gap, fused thigh gap, liquid thigh gap, poorly drawn thigh gap, poorly drawn anus, bad collarbone, fused collarbone, missing collarbone, liquid collarbone, strong girl, obesity, worst quality, low quality, normal quality, liquid tentacles, bad tentacles, poorly drawn tentacles, split tentacles, fused tentacles, missing clit, bad clit, fused clit, colorful clit, black clit, liquid clit, QR code, bar code, censored, safety panties, safety knickers, beard, furry ,pony, pubic hair, mosaic, excrement, faeces, shit</span><br></pre></td></tr></table></figure>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/7c673635/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/7c673635/" class="post-title-link" itemprop="url">Understand kprobe through talking with ChatGPT4</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2023-04-01 05:30:03 / 修改时间：05:30:41" itemprop="dateCreated datePublished" datetime="2023-04-01T05:30:03+00:00">2023-04-01</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>2.5k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>4 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<h1 id="Kprobes-in-Linux-Kernel"><a href="#Kprobes-in-Linux-Kernel" class="headerlink" title="Kprobes in Linux Kernel"></a>Kprobes in Linux Kernel</h1><p>Kprobes is a dynamic tracing and analysis tool in the Linux kernel, which allows developers to insert breakpoints into running kernel code without stopping or recompiling the kernel. Kprobes is designed to minimize its impact on the performance of the target code.</p>
<h2 id="kprobe-insn-page"><a href="#kprobe-insn-page" class="headerlink" title="kprobe_insn_page"></a>kprobe_insn_page</h2><p><code>kprobe_insn_page</code> is a memory page used to store copies of the original instructions that are replaced by Kprobes breakpoints. It serves as a cache for these instructions so that they can be executed without affecting the original code location.</p>
<h2 id="ModR-x2F-M-Byte"><a href="#ModR-x2F-M-Byte" class="headerlink" title="ModR/M Byte"></a>ModR/M Byte</h2><p>The ModR/M byte is part of the x86 instruction set architecture, providing a compact way to encode certain instructions that involve registers and memory operands. The ModR/M byte is used in combination with other bytes to specify the complete behavior of an instruction.</p>
<h2 id="Kprobes-Event-Handling"><a href="#Kprobes-Event-Handling" class="headerlink" title="Kprobes Event Handling"></a>Kprobes Event Handling</h2><p>Kprobes inserts a breakpoint (e.g., INT3 instruction) at the target code address. When the breakpoint is triggered, the processor transfers the execution to a predefined Kprobes handler function. The handler function modifies the instruction pointer register (<code>regs->ip</code>) to point to the memory location containing the copy of the original instruction. The CPU executes the original instruction copy, triggering the INT3 breakpoint again, allowing Kprobes to take control once more before resuming execution.</p>
<h2 id="kprobe-ctlblk"><a href="#kprobe-ctlblk" class="headerlink" title="kprobe_ctlblk"></a>kprobe_ctlblk</h2><p><code>kprobe_ctlblk</code> (Kprobe Control Block) is a kernel data structure used to store control information for each CPU during Kprobes event handling. Each CPU has an instance of <code>kprobe_ctlblk</code>, which contains temporary state and information such as:</p>
<ul>
<li>Status flags indicating the current state of Kprobes event handling.</li>
<li>Saved register values, such as the instruction pointer register (<code>ip</code>).</li>
<li>Other temporary data related to the Kprobes handler function.</li>
</ul>
<p>The main purpose of <code>kprobe_ctlblk</code> is to save and restore CPU register values and other context information during Kprobes event handling.</p>
<h2 id="KPROBE-HIT-SS"><a href="#KPROBE-HIT-SS" class="headerlink" title="KPROBE_HIT_SS"></a>KPROBE_HIT_SS</h2><p><code>KPROBE_HIT_SS</code> is a Kprobes status flag that indicates a Single Step Trap was triggered during Kprobe event handling on x86 architecture. This flag is used to signify this situation so that the Kprobes handler function can correctly handle the single step trap.</p>
<p>When a single step trap is detected during Kprobes event handling, the <code>KPROBE_HIT_SS</code> flag is set to <code>true</code>. The Kprobes handler function can then use this flag to determine how to handle the single step trap, such as whether specific actions need to be performed or how to resume execution after event handling is complete.</p>
<p>In summary, the <code>KPROBE_HIT_SS</code> flag is used during Kprobes event handling to indicate a triggered single step trap so that the Kprobes handler function can correctly handle the situation. This helps ensure that target code execution can continue without being affected by Kprobes after the handler function is complete.</p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/6eda6b08/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/6eda6b08/" class="post-title-link" itemprop="url">Launch VM through QEMU</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2023-03-21 02:15:51" itemprop="dateCreated datePublished" datetime="2023-03-21T02:15:51+00:00">2023-03-21</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar-check"></i>
</span>
<span class="post-meta-item-text">更新于</span>
<time title="修改时间：2023-03-23 09:56:54" itemprop="dateModified" datetime="2023-03-23T09:56:54+00:00">2023-03-23</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>2.4k</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>4 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p>We can lauch vm through qemu quickly. These steps are:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">qemu-system-x86_64 -nographic -enable-kvm -cpu host,+x2apic,pmu=on -smp 32,sockets=2,cores=16,threads=1 -m 64G -device virtio-net-pci,netdev=n1 -netdev user,id=n1,hostfwd=tcp::1688-:22 -drive file=/home/zhaoguanjun.zgj/aliyun_3_x64_20G_nocloud_alibase_20211025.qcow2,if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 -monitor pty -cpu host -machine q35 -drive file=/dev/sdb1,if=none,id=drive-virtio-disk2,format=raw,cache=none,aio=native -device virtio-blk-pci,scsi=off,num-queues=4,config-wce=off,drive=drive-virtio-disk2,id=virtio-disk1 -gdb tcp::1111</span><br></pre></td></tr></table></figure>
<p>Some params and the related meanings:</p>
<ul>
<li>-nographic, Disable graphical output and redirect serial I/Os to console</li>
<li>-daemonize, Daemonize the qemu thread</li>
</ul>
<p>If you want to add a mdev device, you should:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">1. uuid # For example, a6613d90-c944-11ed-a67e-00e04c84d2c4</span><br><span class="line">2. echo "a6613d90-c944-11ed-a67e-00e04c84d2c4" > /sys/bus/pci/devices/0000:6d:00.0/mdev_supported_types/dlb2-dlb/create</span><br><span class="line">3. add "-device vfio-pci,sysfsdev=/sys/bus/pci/devices/0000:6d:00.0/a6613d90-c944-11ed-a67e-00e04c84d2c4/" to the qemu boot line</span><br></pre></td></tr></table></figure>
<p>Live migration from A to B:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">A: Launch VM, source</span><br><span class="line">- qemu-system-x86_64 -nographic -enable-kvm -cpu host,+x2apic,pmu=on -smp 32,sockets=2,cores=16,threads=1 -m 64G -device virtio-net-pci,netdev=n1 -netdev user,id=n1,hostfwd=tcp::1688-:22 -drive file=/home/zhaoguanjun.zgj/aliyun_3_x64_20G_nocloud_alibase_20211025.qcow2,if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 -monitor pty -cpu host -machine q35 -drive file=/dev/sdb1,if=none,id=drive-virtio-disk2,format=raw,cache=none,aio=native -device virtio-blk-pci,scsi=off,num-queues=4,config-wce=off,drive=drive-virtio-disk2,id=virtio-disk1 -gdb tcp::1111</span><br><span class="line"></span><br><span class="line">B: Migrate to, destination</span><br><span class="line">- qemu-system-x86_64 -nographic -enable-kvm -cpu host,+x2apic,pmu=on -smp 32,sockets=2,cores=16,threads=1 -m 64G -device virtio-net-pci,netdev=n1 -netdev user,id=n1,hostfwd=tcp::1688-:22 -drive file=/home/zhaoguanjun.zgj/aliyun_3_x64_20G_nocloud_alibase_20211025.qcow2,if=none,id=virtio-disk0 -device virtio-blk-pci,drive=virtio-disk0 -monitor pty -cpu host -machine q35 -drive file=/dev/sdb1,if=none,id=drive-virtio-disk2,format=raw,cache=none,aio=native -device virtio-blk-pci,scsi=off,num-queues=4,config-wce=off,drive=drive-virtio-disk2,id=virtio-disk1 -gdb tcp::1111 -incoming tcp:(A IP):(A PORT)</span><br><span class="line">- ./qemu-system-x86_64 -monitor stdio</span><br><span class="line"> (qemu) migrate tcp:(A IP):(A PORT)</span><br><span class="line"> (qemu) info migrate</span><br></pre></td></tr></table></figure>
<p>Then, if migration status echo ‘completed’, the VM migrate from A to B successfully.</p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/1aa54f31/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/1aa54f31/" class="post-title-link" itemprop="url">How to use docker to run your task</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2023-03-10 11:42:16" itemprop="dateCreated datePublished" datetime="2023-03-10T11:42:16+00:00">2023-03-10</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar-check"></i>
</span>
<span class="post-meta-item-text">更新于</span>
<time title="修改时间：2023-04-01 02:02:04" itemprop="dateModified" datetime="2023-04-01T02:02:04+00:00">2023-04-01</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>646</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>1 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<p>A example about how to use docker to run your task. That is:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">1. Pull the images or repository from registry</span><br><span class="line"> docker pull reg.docker.alibaba-inc.com/alinux/kdevel-x64:3.230303.1</span><br><span class="line"></span><br><span class="line">2. List the images</span><br><span class="line"> docker images</span><br><span class="line"></span><br><span class="line">3. Start the container and interact through terminal</span><br><span class="line"> docker run -itd --net host -v /mnt/sdb/zhaoguanjun.zgj/:/root/workspace reg.docker.alibaba-inc.com/alinux/kdevel-x64 bash</span><br><span class="line"></span><br><span class="line">4. List all the containers</span><br><span class="line"> docker ps -a</span><br><span class="line"></span><br><span class="line">5. Execute the container</span><br><span class="line"> docker exec -it [container-id] bash</span><br><span class="line"></span><br><span class="line">6. Exit the container</span><br><span class="line"> Execute 'exit' in docker terminal</span><br><span class="line"></span><br><span class="line">7. Kill the container</span><br><span class="line"> docker kill [container-id]</span><br><span class="line"></span><br><span class="line">8. Remove the container or images</span><br><span class="line"> docker rm [container-id]</span><br></pre></td></tr></table></figure>
<p>This document will be continuously improved.</p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/679b99cc/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/679b99cc/" class="post-title-link" itemprop="url">20220918回复妹子</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2022-09-18 05:16:11" itemprop="dateCreated datePublished" datetime="2022-09-18T05:16:11+00:00">2022-09-18</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar-check"></i>
</span>
<span class="post-meta-item-text">更新于</span>
<time title="修改时间：2022-09-19 09:10:24" itemprop="dateModified" datetime="2022-09-19T09:10:24+00:00">2022-09-19</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>615</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>1 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
这是一篇加密文章，需要密码才能继续阅读。
<!--noindex-->
<div class="post-button">
<a class="btn" href="/posts/679b99cc/#more" rel="contents">
阅读全文 »
</a>
</div>
<!--/noindex-->
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</article>
<article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
<link itemprop="mainEntityOfPage" href="https://mapicccy.github.io/posts/57813cdd/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="image" content="/images/meizi.jpg">
<meta itemprop="name" content="mapicccy">
<meta itemprop="description" content="">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="mapicccy">
</span>
<header class="post-header">
<h2 class="post-title" itemprop="name headline">
<a href="/posts/57813cdd/" class="post-title-link" itemprop="url">20220912被一个妹子吓到了</a>
</h2>
<div class="post-meta">
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar"></i>
</span>
<span class="post-meta-item-text">发表于</span>
<time title="创建时间：2022-09-12 12:10:22" itemprop="dateCreated datePublished" datetime="2022-09-12T12:10:22+00:00">2022-09-12</time>
</span>
<span class="post-meta-item">
<span class="post-meta-item-icon">
<i class="far fa-calendar-check"></i>
</span>
<span class="post-meta-item-text">更新于</span>
<time title="修改时间：2022-09-18 06:47:19" itemprop="dateModified" datetime="2022-09-18T06:47:19+00:00">2022-09-18</time>
</span>
<br>
<span class="post-meta-item" title="本文字数">
<span class="post-meta-item-icon">
<i class="far fa-file-word"></i>
</span>
<span class="post-meta-item-text">本文字数：</span>
<span>329</span>
</span>
<span class="post-meta-item" title="阅读时长">
<span class="post-meta-item-icon">
<i class="far fa-clock"></i>
</span>
<span class="post-meta-item-text">阅读时长 ≈</span>
<span>1 分钟</span>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
这是一篇加密文章，需要密码才能继续阅读。
<!--noindex-->
<div class="post-button">
<a class="btn" href="/posts/57813cdd/#more" rel="contents">
阅读全文 »
</a>
</div>
<!--/noindex-->
</div>