-
Notifications
You must be signed in to change notification settings - Fork 3.4k
/
Copy pathInstrProfData.inc
1023 lines (945 loc) · 39.4 KB
/
InstrProfData.inc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
|*
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|* See https://llvm.org/LICENSE.txt for license information.
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|*
\*===----------------------------------------------------------------------===*/
/*
* This is the main file that defines all the data structure, signature,
* constant literals that are shared across profiling runtime library,
* compiler (instrumentation), and host tools (reader/writer). The entities
* defined in this file affect the profile runtime ABI, the raw profile format,
* or both.
*
* The file has two identical copies. The primary copy lives in LLVM and
* the other one sits in compiler-rt/lib/profile directory. To make changes
* in this file, first modify the primary copy and copy it over to compiler-rt.
* Testing of any change in this file can start only after the two copies are
* synced up.
*
* The first part of the file includes macros that defines types, names, and
* initializers for the member fields of the core data structures. The field
* declarations for one structure is enabled by defining the field activation
* macro associated with that structure. Only one field activation record
* can be defined at one time and the rest definitions will be filtered out by
* the preprocessor.
*
* Examples of how the template is used to instantiate structure definition:
* 1. To declare a structure:
*
* struct ProfData {
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* Type Name;
* #include "llvm/ProfileData/InstrProfData.inc"
* };
*
* 2. To construct LLVM type arrays for the struct type:
*
* Type *DataTypes[] = {
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* LLVMType,
* #include "llvm/ProfileData/InstrProfData.inc"
* };
*
* 4. To construct constant array for the initializers:
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* Initializer,
* Constant *ConstantVals[] = {
* #include "llvm/ProfileData/InstrProfData.inc"
* };
*
*
* The second part of the file includes definitions all other entities that
* are related to runtime ABI and format. When no field activation macro is
* defined, this file can be included to introduce the definitions.
*
\*===----------------------------------------------------------------------===*/
/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
* the compiler runtime. */
#ifndef INSTR_PROF_VISIBILITY
#define INSTR_PROF_VISIBILITY
#endif
// clang-format off:consider re-enabling clang-format if auto-formatted C macros
// are readable (e.g., after `issue #82426` is fixed)
/* INSTR_PROF_DATA start. */
/* Definition of member fields of the per-function control structure. */
#ifndef INSTR_PROF_DATA
#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
Inc->getHash()->getZExtValue()))
INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
/* This is used to map function pointers for the indirect call targets to
* function name hashes during the conversion from raw to merged profile
* data.
*/
INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
FunctionAddr)
INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
ValuesPtrExpr)
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */
/* For a virtual table object, record the name hash to associate profiled
* addresses with global variables, and record {starting address, size in bytes}
* to map the profiled virtual table (which usually have an offset from the
* starting address) back to a virtual table object. */
#ifndef INSTR_PROF_VTABLE_DATA
#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_VTABLE_DATA_DEFINED
#endif
INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \
VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
IndexedInstrProf::ComputeHash(PGOVTableName)))
INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
VTablePointer, VTableAddr)
INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
VTableSizeVal))
#undef INSTR_PROF_VTABLE_DATA
/* INSTR_PROF_VTABLE_DATA end. */
/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
* to be used in unit test.
*
* typedef struct ValueProfNode {
* // InstrProfValueData VData;
* uint64_t Value;
* uint64_t Count;
* struct ValueProfNode *Next;
* } ValueProfNode;
*/
/* INSTR_PROF_VALUE_NODE start. */
#ifndef INSTR_PROF_VALUE_NODE
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0))
#undef INSTR_PROF_VALUE_NODE
/* INSTR_PROF_VALUE_NODE end. */
/* INSTR_PROF_RAW_HEADER start */
/* Definition of member fields of the raw profile header data structure. */
/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating
raw profile format. */
#ifndef INSTR_PROF_RAW_HEADER
#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
(uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
(uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
/* VALUE_PROF_FUNC_PARAM start */
/* Definition of parameter types of the runtime API used to do value profiling
* for a given value site.
*/
#ifndef VALUE_PROF_FUNC_PARAM
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
#define INSTR_PROF_COMMA
#else
#define INSTR_PROF_DATA_DEFINED
#define INSTR_PROF_COMMA ,
#endif
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
#undef VALUE_PROF_FUNC_PARAM
#undef INSTR_PROF_COMMA
/* VALUE_PROF_FUNC_PARAM end */
/* VALUE_PROF_KIND start */
#ifndef VALUE_PROF_KIND
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
/* For indirect function call value profiling, the addresses of the target
* functions are profiled by the instrumented code. The target addresses are
* written in the raw profile data and converted to target function name's MD5
* hash by the profile reader during deserialization. Typically, this happens
* when the raw profile data is read during profile merging.
*
* For this remapping the ProfData is used. ProfData contains both the function
* name hash and the function address.
*/
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
/* For virtual table address profiling, the address point of the virtual table
* (i.e., the address contained in objects pointing to a virtual table) are
* profiled. Note this may not be the address of the per C++ class virtual table
* object (e.g., there might be an offset).
*
* The profiled addresses are stored in raw profile, together with the following
* two types of information.
* 1. The (starting and ending) addresses of per C++ class virtual table objects.
* 2. The (compressed) virtual table object names.
* RawInstrProfReader converts profiled virtual table addresses to virtual table
* objects' MD5 hash.
*/
VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
#undef COVMAP_V2_OR_V3
#ifdef COVMAP_V2
#define COVMAP_V2_OR_V3
#endif
#ifdef COVMAP_V3
#define COVMAP_V2_OR_V3
#endif
/* COVMAP_FUNC_RECORD start */
/* Definition of member fields of the function record structure in coverage
* map.
*/
#ifndef COVMAP_FUNC_RECORD
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
#ifdef COVMAP_V1
COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
llvm::PointerType::getUnqual(Ctx)))
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
NameValue.size()))
#endif
#ifdef COVMAP_V2_OR_V3
COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
llvm::ConstantInt::get( \
llvm::Type::getInt64Ty(Ctx), NameHash))
#endif
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
llvm::ConstantInt::get( \
llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
llvm::ConstantInt::get( \
llvm::Type::getInt64Ty(Ctx), FuncHash))
#ifdef COVMAP_V3
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
llvm::ConstantInt::get( \
llvm::Type::getInt64Ty(Ctx), FilenamesRef))
COVMAP_FUNC_RECORD(const char, \
llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
CoverageMapping.size()), \
CoverageMapping,
llvm::ConstantDataArray::getRaw( \
CoverageMapping, CoverageMapping.size(), \
llvm::Type::getInt8Ty(Ctx)))
#endif
#undef COVMAP_FUNC_RECORD
/* COVMAP_FUNC_RECORD end. */
/* COVMAP_HEADER start */
/* Definition of member fields of coverage map header.
*/
#ifndef COVMAP_HEADER
#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
llvm::ConstantInt::get(Int32Ty, NRecords))
COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
llvm::ConstantInt::get(Int32Ty, FilenamesSize))
COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
COVMAP_HEADER(uint32_t, Int32Ty, Version, \
llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
#undef COVMAP_HEADER
/* COVMAP_HEADER end. */
/* COVINIT_FUNC start */
#ifndef COVINIT_FUNC
#define COVINIT_FUNC(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \
WriteoutF)
COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \
ResetF)
#undef COVINIT_FUNC
/* COVINIT_FUNC end */
#ifdef INSTR_PROF_SECT_ENTRY
#define INSTR_PROF_DATA_DEFINED
INSTR_PROF_SECT_ENTRY(IPSK_data, \
INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
INSTR_PROF_DATA_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
INSTR_PROF_CNTS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
INSTR_PROF_BITS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vname, \
INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
INSTR_PROF_VNAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
INSTR_PROF_VTAB_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_covname, \
INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \
INSTR_PROF_COVINIT_COFF, "__LLVM_COV,")
#undef INSTR_PROF_SECT_ENTRY
#endif
#ifdef INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_DATA_DEFINED
#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
/*!
* This is the header of the data structure that defines the on-disk
* layout of the value profile data of a particular kind for one function.
*/
typedef struct ValueProfRecord {
/* The kind of the value profile record. */
uint32_t Kind;
/*
* The number of value profile sites. It is guaranteed to be non-zero;
* otherwise the record for this kind won't be emitted.
*/
uint32_t NumValueSites;
/*
* The first element of the array that stores the number of profiled
* values for each value site. The size of the array is NumValueSites.
* Since NumValueSites is greater than zero, there is at least one
* element in the array.
*/
uint8_t SiteCountArray[1];
/*
* The fake declaration is for documentation purpose only.
* Align the start of next field to be on 8 byte boundaries.
uint8_t Padding[X];
*/
/* The array of value profile data. The size of the array is the sum
* of all elements in SiteCountArray[].
InstrProfValueData ValueData[];
*/
#ifdef __cplusplus
/*!
* Return the number of value sites.
*/
uint32_t getNumValueSites() const { return NumValueSites; }
/*!
* Read data from this record and save it to Record.
*/
void deserializeTo(InstrProfRecord &Record,
InstrProfSymtab *SymTab);
/*
* In-place byte swap:
* Do byte swap for this instance. \c Old is the original order before
* the swap, and \c New is the New byte order.
*/
void swapBytes(llvm::endianness Old, llvm::endianness New);
#endif
} ValueProfRecord;
/*!
* Per-function header/control data structure for value profiling
* data in indexed format.
*/
typedef struct ValueProfData {
/*
* Total size in bytes including this field. It must be a multiple
* of sizeof(uint64_t).
*/
uint32_t TotalSize;
/*
*The number of value profile kinds that has value profile data.
* In this implementation, a value profile kind is considered to
* have profile data if the number of value profile sites for the
* kind is not zero. More aggressively, the implementation can
* choose to check the actual data value: if none of the value sites
* has any profiled values, the kind can be skipped.
*/
uint32_t NumValueKinds;
/*
* Following are a sequence of variable length records. The prefix/header
* of each record is defined by ValueProfRecord type. The number of
* records is NumValueKinds.
* ValueProfRecord Record_1;
* ValueProfRecord Record_N;
*/
#if __cplusplus
/*!
* Return the total size in bytes of the on-disk value profile data
* given the data stored in Record.
*/
static uint32_t getSize(const InstrProfRecord &Record);
/*!
* Return a pointer to \c ValueProfData instance ready to be streamed.
*/
static std::unique_ptr<ValueProfData>
serializeFrom(const InstrProfRecord &Record);
/*!
* Check the integrity of the record.
*/
Error checkIntegrity();
/*!
* Return a pointer to \c ValueProfileData instance ready to be read.
* All data in the instance are properly byte swapped. The input
* data is assumed to be in little endian order.
*/
static Expected<std::unique_ptr<ValueProfData>>
getValueProfData(const unsigned char *SrcBuffer,
const unsigned char *const SrcBufferEnd,
llvm::endianness SrcDataEndianness);
/*!
* Swap byte order from \c Endianness order to host byte order.
*/
void swapBytesToHost(llvm::endianness Endianness);
/*!
* Swap byte order from host byte order to \c Endianness order.
*/
void swapBytesFromHost(llvm::endianness Endianness);
/*!
* Return the total size of \c ValueProfileData.
*/
uint32_t getSize() const { return TotalSize; }
/*!
* Read data from this data and save it to \c Record.
*/
void deserializeTo(InstrProfRecord &Record,
InstrProfSymtab *SymTab);
void operator delete(void *ptr) { ::operator delete(ptr); }
#endif
} ValueProfData;
/*
* The closure is designed to abstact away two types of value profile data:
* - InstrProfRecord which is the primary data structure used to
* represent profile data in host tools (reader, writer, and profile-use)
* - value profile runtime data structure suitable to be used by C
* runtime library.
*
* Both sources of data need to serialize to disk/memory-buffer in common
* format: ValueProfData. The abstraction allows compiler-rt's raw profiler
* writer to share the same format and code with indexed profile writer.
*
* For documentation of the member methods below, refer to corresponding methods
* in class InstrProfRecord.
*/
typedef struct ValueProfRecordClosure {
const void *Record;
uint32_t (*GetNumValueKinds)(const void *Record);
uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
/*
* After extracting the value profile data from the value profile record,
* this method is used to map the in-memory value to on-disk value. If
* the method is null, value will be written out untranslated.
*/
uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
uint32_t S);
ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
} ValueProfRecordClosure;
INSTR_PROF_VISIBILITY ValueProfRecord *
getFirstValueProfRecord(ValueProfData *VPD);
INSTR_PROF_VISIBILITY ValueProfRecord *
getValueProfRecordNext(ValueProfRecord *VPR);
INSTR_PROF_VISIBILITY InstrProfValueData *
getValueProfRecordValueData(ValueProfRecord *VPR);
INSTR_PROF_VISIBILITY uint32_t
getValueProfRecordHeaderSize(uint32_t NumValueSites);
#undef INSTR_PROF_VALUE_PROF_DATA
#endif /* INSTR_PROF_VALUE_PROF_DATA */
#ifdef INSTR_PROF_COMMON_API_IMPL
#define INSTR_PROF_DATA_DEFINED
#ifdef __cplusplus
#define INSTR_PROF_INLINE inline
#define INSTR_PROF_NULLPTR nullptr
#else
#define INSTR_PROF_INLINE
#define INSTR_PROF_NULLPTR NULL
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
// clang-format on
/*!
* Return the \c ValueProfRecord header size including the
* padding bytes.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
getValueProfRecordHeaderSize(uint32_t NumValueSites) {
uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
sizeof(uint8_t) * NumValueSites;
/* Round the size to multiple of 8 bytes. */
Size = (Size + 7) & ~7;
return Size;
}
/*!
* Return the total size of the value profile record including the
* header and the value data.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {
return getValueProfRecordHeaderSize(NumValueSites) +
sizeof(InstrProfValueData) * NumValueData;
}
/*!
* Return the pointer to the start of value data array.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *
getValueProfRecordValueData(ValueProfRecord *This) {
return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
This->NumValueSites));
}
/*!
* Return the total number of value data for \c This record.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
getValueProfRecordNumValueData(ValueProfRecord *This) {
uint32_t NumValueData = 0;
uint32_t I;
for (I = 0; I < This->NumValueSites; I++)
NumValueData += This->SiteCountArray[I];
return NumValueData;
}
/*!
* Use this method to advance to the next \c This \c ValueProfRecord.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
getValueProfRecordNext(ValueProfRecord *This) {
uint32_t NumValueData = getValueProfRecordNumValueData(This);
return (ValueProfRecord *)((char *)This +
getValueProfRecordSize(This->NumValueSites,
NumValueData));
}
/*!
* Return the first \c ValueProfRecord instance.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
getFirstValueProfRecord(ValueProfData *This) {
return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
}
/* Closure based interfaces. */
/*!
* Return the total size in bytes of the on-disk value profile data
* given the data stored in Record.
*/
INSTR_PROF_VISIBILITY uint32_t
getValueProfDataSize(ValueProfRecordClosure *Closure) {
uint32_t Kind;
uint32_t TotalSize = sizeof(ValueProfData);
const void *Record = Closure->Record;
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
if (!NumValueSites)
continue;
TotalSize += getValueProfRecordSize(NumValueSites,
Closure->GetNumValueData(Record, Kind));
}
return TotalSize;
}
/*!
* Extract value profile data of a function for the profile kind \c ValueKind
* from the \c Closure and serialize the data into \c This record instance.
*/
INSTR_PROF_VISIBILITY void
serializeValueProfRecordFrom(ValueProfRecord *This,
ValueProfRecordClosure *Closure,
uint32_t ValueKind, uint32_t NumValueSites) {
uint32_t S;
const void *Record = Closure->Record;
This->Kind = ValueKind;
This->NumValueSites = NumValueSites;
InstrProfValueData *DstVD = getValueProfRecordValueData(This);
for (S = 0; S < NumValueSites; S++) {
uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
This->SiteCountArray[S] = ND;
Closure->GetValueForSite(Record, DstVD, ValueKind, S);
DstVD += ND;
}
}
/*!
* Extract value profile data of a function from the \c Closure
* and serialize the data into \c DstData if it is not NULL or heap
* memory allocated by the \c Closure's allocator method. If \c
* DstData is not null, the caller is expected to set the TotalSize
* in DstData.
*/
INSTR_PROF_VISIBILITY ValueProfData *
serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
ValueProfData *DstData) {
uint32_t Kind;
uint32_t TotalSize =
DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
ValueProfData *VPD =
DstData ? DstData : Closure->AllocValueProfData(TotalSize);
VPD->TotalSize = TotalSize;
VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
ValueProfRecord *VR = getFirstValueProfRecord(VPD);
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
if (!NumValueSites)
continue;
serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
VR = getValueProfRecordNext(VR);
}
return VPD;
}
#undef INSTR_PROF_COMMON_API_IMPL
#endif /* INSTR_PROF_COMMON_API_IMPL */
/*============================================================================*/
// clang-format off:consider re-enabling clang-format if auto-formatted C macros
// are readable (e.g., after `issue #82426` is fixed)
#ifndef INSTR_PROF_DATA_DEFINED
#ifndef INSTR_PROF_DATA_INC
#define INSTR_PROF_DATA_INC
/* Helper macros. */
#define INSTR_PROF_SIMPLE_QUOTE(x) #x
#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
/* Magic number to detect file format and endianness.
* Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
* so that utilities, like strings, don't grab it as a string. 129 is also
* invalid UTF-8, and high enough to be interesting.
* Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
* for 32-bit platforms.
*/
#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
#define INSTR_PROF_RAW_VERSION 10
/* Indexed profile format version (start from 1). */
#define INSTR_PROF_INDEX_VERSION 12
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 6
/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
* version for other variants of profile. We set the 8th most significant bit
* (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
* generated profile, and 0 if this is a Clang FE generated profile.
* 1 in bit 57 indicates there are context-sensitive records in the profile.
* The 54th bit indicates whether to always instrument loop entry blocks.
* The 58th bit indicates whether to always instrument function entry blocks.
* The 59th bit indicates whether to use debug info to correlate profiles.
* The 60th bit indicates single byte coverage instrumentation.
* The 61st bit indicates function entry instrumentation only.
* The 62nd bit indicates whether memory profile information is present.
* The 63rd bit indicates if this is a temporal profile.
*/
#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
/* The variable that holds the name of the profile data
* specified via command line. */
#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
/* section name strings common to all targets other
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_COVDATA_COMMON __llvm_covdata
#define INSTR_PROF_COVNAME_COMMON __llvm_covnames
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
#define INSTR_PROF_COVINIT_COMMON __llvm_covinit
/* Windows section names. Because these section names contain dollar characters,
* they must be quoted.
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
#define INSTR_PROF_VNAME_COFF ".lprfvn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_BITS_COFF ".lprfb$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
#define INSTR_PROF_VTAB_COFF ".lprfvt$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
/* Since cov data and cov names sections are not allocated, we don't need to
* access them at runtime.
*/
#define INSTR_PROF_COVDATA_COFF ".lcovd"
#define INSTR_PROF_COVNAME_COFF ".lcovn"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
// FIXME: Placeholder for Windows. Windows currently does not initialize
// the GCOV functions in the runtime.
#define INSTR_PROF_COVINIT_COFF ".lcovd$M"
#ifdef _WIN32
/* Runtime section names and name strings. */
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF
#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF
/* Array of pointers. Each pointer points to a list
* of value nodes associated with one value site.
*/
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
/* Value profile nodes section. */
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF
#else
/* Runtime section names and name strings. */
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON)
#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON)
/* Array of pointers. Each pointer points to a list
* of value nodes associated with one value site.
*/
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
/* Value profile nodes section. */
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
/* Order file instrumentation. */
#define INSTR_PROF_ORDERFILE_SECT_NAME \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON)
#endif
#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
/* Macros to define start/stop section symbol for a given
* section on Linux. For instance
* INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
* expand to __start___llvm_prof_data
*/
#define INSTR_PROF_SECT_START(Sect) \
INSTR_PROF_CONCAT(__start_,Sect)
#define INSTR_PROF_SECT_STOP(Sect) \
INSTR_PROF_CONCAT(__stop_,Sect)
/* Value Profiling API linkage name. */
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
/* The data structure that represents a tracked value by the
* value profiler.
*/
typedef struct InstrProfValueData {
/* Profiled value. */
uint64_t Value;
/* Number of times the value appears in the training run. */
uint64_t Count;
} InstrProfValueData;
#endif /* INSTR_PROF_DATA_INC */
#ifndef INSTR_ORDER_FILE_INC
/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
#define INSTR_ORDER_FILE_BUFFER_BITS 17
#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
#endif /* INSTR_ORDER_FILE_INC */
#else
#undef INSTR_PROF_DATA_DEFINED
#endif
#undef COVMAP_V2_OR_V3
#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
#ifdef __cplusplus
#define INSTR_PROF_INLINE inline
#else
#define INSTR_PROF_INLINE
#endif
/* The value range buckets (22 buckets) for the memop size value profiling looks
* like:
*
* [0, 0]
* [1, 1]
* [2, 2]
* [3, 3]
* [4, 4]
* [5, 5]
* [6, 6]
* [7, 7]
* [8, 8]
* [9, 15]
* [16, 16]
* [17, 31]
* [32, 32]
* [33, 63]
* [64, 64]
* [65, 127]
* [128, 128]
* [129, 255]
* [256, 256]
* [257, 511]
* [512, 512]
* [513, UINT64_MAX]
*
* Each range has a 'representative value' which is the lower end value of the
* range and used to store in the runtime profile data records and the VP
* metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
*/
#define INSTR_PROF_NUM_BUCKETS 22
/*
* Clz and Popcount. This code was copied from
* compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
* llvm/include/llvm/Support/MathExtras.h.
*/
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfClzll(unsigned long long X) {
unsigned long LeadZeroIdx = 0;
#if !defined(_M_ARM64) && !defined(_M_X64)
// Scan the high 32 bits.
if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
// from the MSB.
// Scan the low 32 bits.
if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
return (int)(63 - LeadZeroIdx);
#else
if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
#endif
return 64;
}
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfPopcountll(unsigned long long X) {
// This code originates from https://reviews.llvm.org/rG30626254510f.
unsigned long long v = X;
v = v - ((v >> 1) & 0x5555555555555555ULL);
v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
}
#else
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
#endif /* defined(_MSC_VER) && !defined(__clang__) */
// clang-format on
/* Map an (observed) memop size value to the representative value of its range.
* For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
InstrProfGetRangeRepValue(uint64_t Value) {
if (Value <= 8)
// The first ranges are individually tracked. Use the value as is.
return Value;
else if (Value >= 513)
// The last range is mapped to its lowest value.
return 513;
else if (InstProfPopcountll(Value) == 1)
// If it's a power of two, use it as is.