Kernel Memory

From pressy's brainbackup
Jump to: navigation, search

From time to time I see some solaris server using a lot of memory "just" for the kernel. You could take a deeper look to see where the kernel memory is used:

root@server:~# mdb -k
Loading modules: [ unix genunix specfs dtrace zfs scsi_vhci sd mpt_sas mac px ldc ip hook neti ds arp usba kssl stmf stmf_sbd random sockfs md niumx idm
cpc crypto fcip fctl fcp mdesc vldc smbsrv nfs zvmm ufs logindmux ptm ii nsctl sppp nsmb rdc sdbc sv lofs ipc ]

>
> ::kmastat
cache                        buf    buf    buf    memory     alloc alloc
name                        size in use  total    in use   succeed  fail
------------------------- ------ ------ ------ ---------- --------- -----
kmem_magazine_1               16  13787  39546     638976B   2212036     0
kmem_magazine_3               32  13293 149017    4825088B  20882637     0
kmem_magazine_7               64  63818 687708   44711936B  74367950     0
kmem_magazine_15             128  81728 154728   20119552B  40689829     0
kmem_magazine_31             256      0      0          0B         0     0
kmem_magazine_47             384      0      0          0B         0     0
kmem_magazine_63             512      0      0          0B         0     0
kmem_magazine_95             768      0      0          0B         0     0
kmem_magazine_127           1024      0      0          0B         0     0
kmem_magazine_143           1152      0      0          0B         0     0
kmem_magazine_179           1440      0      0          0B         0     0
kmem_magazine_255           2048      0      0          0B         0     0
kmem_magazine_361           2896      0      0          0B         0     0
kmem_magazine_492           3944      0      0          0B         0     0
kmem_slab_cache               72  21964  41888    3063808B   1905292     0
kmem_bufctl_cache             24  60474  84162    2039808B   1966131     0
kmem_bufctl_audit_cache      128      0      0          0B         0     0
kmem_va_8192                8192   6617  15040  123207680B     92929     0
kmem_va_16384              16384     12     32     524288B        22     0
kmem_va_24576              24576     11     50    1310720B       147     0
kmem_va_32768              32768      6     64    2097152B      1152     0
kmem_va_40960              40960      9    192    8388608B      4001     0
kmem_va_49152              49152      9    200   10485760B      7542     0
kmem_va_57344              57344      1     48    3145728B      1143     0
kmem_va_65536              65536     18    144    9437184B      4846     0
kmem_alloc_8                   8 225785 229390    1851392B 1460654149     0
kmem_alloc_16                 16  70050  74022    1196032B 1802895553     0
kmem_alloc_24                 24 176486 244712    5931008B 4193514113     0
kmem_alloc_32                 32  40268  45034    1458176B 3716814702     0
kmem_alloc_40                 40  31285  47705    1925120B 1528348140     0
kmem_alloc_48                 48  67277 128102    6209536B 1790577690     0
kmem_alloc_56                 56  91174 141955    8019968B 858181106     0
kmem_alloc_64                 64 151639 323946   21061632B 1933863414     0
kmem_alloc_80                 80 187667 261994   21250048B 596556789     0
kmem_alloc_96                 96   9275  29064    2834432B 1763944514     0
kmem_alloc_112               112   8553  16056    1826816B 242740674     0
kmem_alloc_128               128 108755 111636   14516224B 241660346     0
kmem_alloc_160               160   4200  18000    2949120B 378366571     0
kmem_alloc_192               192  60314 130536   25460736B 516119668     0
kmem_alloc_224               224   1682   2232     507904B 498796577     0
kmem_alloc_256               256   2237  51305   13557760B 1199808091     0
kmem_alloc_320               320   4686  18475    6053888B 4005106805     0
kmem_alloc_384               384    664   2289     892928B 1743958062     0
kmem_alloc_448               448   1487   1890     860160B 156543851     0
kmem_alloc_512               512   6052   6645    3629056B 283536972     0
kmem_alloc_640               640  48054  63216   43155456B  49035869     0
kmem_alloc_768               768    181    510     417792B 533346028     0
kmem_alloc_896               896    106    423     385024B   2201063     0
kmem_alloc_1152             1152   3164   3563    4169728B 758869183     0
kmem_alloc_1344             1344    449    768    1048576B 134708639     0
kmem_alloc_1600             1600    130    490     802816B  12434972     0
kmem_alloc_2048             2048    862   1080    2211840B  49268582     0
kmem_alloc_2688             2688    147    471    1286144B 228164685     0
kmem_alloc_4096             4096   2311   2738   11214848B 599289299     0
kmem_alloc_8192             8192   6843   7355   60252160B 823767889     0
kmem_alloc_12288           12288     24    286    3514368B  94265288     0
kmem_alloc_16384           16384    448   4864   79691776B 372203661     0
kmem_alloc_24576           24576    247    358    8798208B  20182582     0
kmem_alloc_32768           32768    625    829   27164672B  67372957     0
kmem_alloc_40960           40960    292    497   20357120B  10594602     0
kmem_alloc_49152           49152    264    424   20840448B  18226119     0
kmem_alloc_57344           57344     57    140    8028160B   7752775     0
kmem_alloc_65536           65536     83    146    9568256B  68801457     0
kmem_alloc_73728           73728    254    318   23445504B   1224400     0
kmem_alloc_81920           81920     12     75    6144000B    855948     0
kmem_alloc_90112           90112     16     65    5857280B    772093     0
kmem_alloc_98304           98304     72    130   12779520B    673798     0
kmem_alloc_106496         106496      5     60    6389760B    649065     0
kmem_alloc_114688         114688      9     63    7225344B    796342     0
kmem_alloc_122880         122880      3     59    7249920B    388702     0
kmem_alloc_131072         131072     20    352   46137344B  46679183     0
kmem_alloc_262144         262144     25     89   23330816B     22305     0
kmem_alloc_524288         524288     10     91   47710208B    292417     0
kmem_alloc_1048576        1048576     14     83   87031808B    151526     0
streams_mblk                  64  11628  17892    1163264B 4222106588     0
streams_dblk_64              192   1880   3948     770048B 1560260262     0
streams_dblk_128             256      6    744     196608B 2094673871     0
streams_dblk_192             320      2    500     163840B 407759363     0
streams_dblk_256             384    206   1911     745472B 384014686     0
streams_dblk_320             448      0    360     163840B 916550415     0
streams_dblk_512             640      0    312     212992B 122220096     0
streams_dblk_1024           1152      0    322     376832B  60312175     0
streams_dblk_1536           1664      2    468     851968B 4237545508     0
streams_dblk_1920           2048      0    180     368640B   2582208     0
streams_dblk_2560           2688      0    276     753664B   9535544     0
streams_dblk_4096           4224      0     81     368640B  10214770     0
streams_dblk_8192            128      0    693      90112B 319193334     0
streams_dblk_12288         12416      0     98    1261568B  17628988     0
streams_dblk_16384           128      0    315      40960B   2089403     0
streams_dblk_20480         20608      0     56    1179648B   9157299     0
streams_dblk_24576           128      0    315      40960B     23840     0
streams_dblk_28672         28800      0     28     819200B     37109     0
streams_dblk_32768           128      0    567      73728B     72012     0
streams_dblk_36864         36992      0     56    2097152B   1752035     0
streams_dblk_40960           128      0    378      49152B     89538     0
streams_dblk_45056         45184      0     21     958464B    122283     0
streams_dblk_49152           128      0    378      49152B     72414     0
streams_dblk_53248         53376      0     49    2637824B  36207829     0
streams_dblk_57344           128      0     63       8192B        80     0
streams_dblk_61440         61568      0     21    1302528B     30352     0
streams_dblk_65536           128      0     63       8192B       136     0
streams_dblk_69632         69760      0      0          0B         0     0
streams_dblk_73728           128      0      0          0B         0     0
streams_dblk_esb             128   2048   2709     352256B 310293725     0
streams_dblk_mdc             128      0      0          0B         0     0
streams_fthdr                408      0      0          0B         0     0
streams_ftblk                376      0      0          0B         0     0
multidata                    248      0      0          0B         0     0
multidata_pdslab            7112      0      0          0B         0     0
multidata_pattbl              32      0      0          0B         0     0
log_cons_cache                48     29   1352      65536B    943715     0
taskq_ent_cache               56  17219  36685    2072576B 105416892     0
taskq_cache                  280    307    348      98304B       417     0
id32_cache                    32      7    759      24576B  61202888     0
One_wallet_cache              68   3257   4480     327680B  93172593     0
Pac_nopredictor_pool      505536      1      7    3538944B         1     0
Mo_cache                     288      0      0          0B         0     0
Monode_prealloc_cache        104      0      0          0B         0     0
Mo_policy_cache               72      0      0          0B         0     0
Mo_resident_cache             72      0      0          0B         0     0
Mo_capture_cache             376      0      0          0B         0     0
Mo_caphead_cache              64      0      0          0B         0     0
Mw_later_cache               128      0      0          0B         0     0
Mw_cache                     128      0      0          0B         0     0
fakemw                        88      0    460      40960B 306059268     0
mvec_tracking                  8      0   2030      16384B 1104075934     0
mvec_tag                      48      0      0          0B         0     0
Memseg_cache                  64      0      0          0B         0     0
bp_map_8192                 8192      0      0          0B         0     0
bp_map_16384               16384      0     80    1310720B      1823     0
bp_map_24576               24576      0     80    2097152B      2443     0
bp_map_32768               32768      0     80    2621440B      3048     0
bp_map_40960               40960      0      0          0B         0     0
bp_map_49152               49152      0      0          0B         0     0
bp_map_57344               57344      0      0          0B         0     0
bp_map_65536               65536      0      0          0B         0     0
mod_hash_entries              24   1635   3042      73728B  11631688     0
ipp_mod                      304      0      0          0B         0     0
ipp_action                   368      0      0          0B         0     0
ipp_packet                    64      0      0          0B         0     0
mmuctxdom_cache              696      8     11       8192B         8     0
sfmmuid_cache               1176    370    636     868352B 132755208     0
sfmmu_tsbinfo_cache           64    384   1638     106496B 277806748     0
sfmmu_tsb8k_cache           8192      0      0          0B         0     0
sfmmu_tsb_cache             8192    104    218    1785856B 130478770     0
sfmmu8_cache                 320 359954 380900  124813312B 230714082     0
sfmmu1_cache                  96 1160768 1277472  124583936B 975785584     0
pa_hment_cache                64    384   1764     114688B 152854640     0
ism_blk_cache                336      0      0          0B         0     0
ism_ment_cache                32      0      0          0B         0     0
srd_cache                   2192    120    407     909312B  36959896     0
region_cache                 144    186    840     122880B  38085849     0
scd_cache                   2192      0      0          0B         0     0
seg_cache                    112  27575  33264    3784704B 2966412882     0
seg_pcache                   104      0    624      65536B     52128     0
vfs_cache                    240    100    561     139264B     24676     0
vn_cache                     216  99646 140492   37126144B 346774691     0
shadow_cache                  72      0      0          0B         0     0
vsk_anchor_cache              40     39    203       8192B       138     0
nep_cache                    384      5    294     114688B      1429     0
dev_info_node_cache          760    283    310     253952B       830     0
ndi_fm_entry_cache            32   5392   6831     221184B 170484658     0
kcf_sreq_cache                56      0    126       8192B   1542463     0
kcf_areq_cache               296      0     25       8192B        25     0
kcf_context_cache            112      0      0          0B         0     0
object_handle                 80 2798905 3152109  255664128B 931834012     0
object_debug_handle          216      0      0          0B         0     0
object_event                  40      0      0          0B         0     0
segkmem_ppa_262144        262144      0     12    3145728B        16     0
segkp_8192                  8192    375    512    4194304B  73688442     0
segkp_16384                16384      0      0          0B         0     0
segkp_24576                24576      0      0          0B         0     0
segkp_32768                32768      0      0          0B         0     0
segkp_40960                40960   3323   3363  146931712B    507168     0
umem_np_8192                8192      0    128    1048576B    555389     0
umem_np_16384              16384      0     80    1310720B     24633     0
umem_np_24576              24576      0      0          0B         0     0
umem_np_32768              32768      0    104    3407872B    484312     0
umem_np_40960              40960      0     90    3932160B    459456     0
umem_np_49152              49152      0      0          0B         0     0
umem_np_57344              57344      0      0          0B         0     0
umem_np_65536              65536      0     68    4456448B     24633     0
thread_cache                1040    915   1545    1687552B 107833858     0
wbuf32_cache                 512    741    900     491520B  82343977     0
wbuf64_cache                1024    775   1141    1335296B   1121736     0
lwp_cache                   1048   1516   1785    1949696B   5360544     0
turnstile_cache               64   3245   4788     311296B  87013637     0
rw_reentrd_cache             136   3245   4130     573440B  93172717     0
tslabel_cache                 48      2    169       8192B         2     0
cred_cache                   184   1137   3520     655360B 111662850     0
proc_ac_cache                 64    301   1512      98304B  81421934     0
rctl_cache                    48   6392   8957     434176B 1288617008     0
rctl_val_cache                64  13614  18018    1171456B 2940692401     0
task_cache                   160    160    800     131072B   1083893     0
kmem_defrag_cache            224      2     36       8192B         2     0
kmem_move_cache               56      0  18705    1056768B   6862115     0
i_dmahdl                    2648      0      0          0B         0     0
timeout_request              128      0      0          0B         0     0
cyclic_id_cache               80    263    303      24576B       329     0
callout_cachebabecafe         80   3239   3276     425984B      3239     0
callout_lcachebabecafe        48  10163  10206     663552B     10163     0
bounds_predictor          505536      3      7    3538944B         3     0
dnlc_space_cache              24      0      0          0B         0     0
file_cache                    72   5236   7392     540672B 2852162077     0
stream_head_cache            376    518    798     311296B 122978665     0
queue_cache                  664   1176   1536    1048576B 128578727     0
syncq_cache                  168     81    432      73728B     20512     0
qband_cache                   64      2    126       8192B         2     0
linkinfo_cache                48     50    507      24576B      5064     0
ciputctrl_cache             1024      0      0          0B         0     0
serializer_cache              64     55   1008      65536B    163607     0
as_cache                     352    371    667     237568B 132754981     0
marker_cache                 128      0    378      49152B    715850     0
anon_cache                    48 164399 212771   10313728B 2491485009     0
anonmap_cache                120  15546  19363    2367488B 3655840909     0
segvn_cache                  224  27575  32724    7446528B 2756160318     0
segvn_szc_cache1              64      0    882      57344B 772856386     0
segvn_szc_cache2             512      0      0          0B         0     0
segvn_szc_cache3            4096      0     88     360448B  35897056     0
segvn_szc_cache4           32768      0      0          0B         0     0
segvn_szc_cache5          262144      0      0          0B         0     0
segvn_szc_cache6          2097152      0      0          0B         0     0
flk_edges                     48      0    169       8192B       409     0
fdb_cache                    104      0      0          0B         0     0
timer_cache                  176      4     46       8192B        31     0
vmu_bound_cache               56   6650   7540     425984B     15900     0
vmu_object_cache              88   2305   2484     221184B      3234     0
physio_buf_cache             248      0    416     106496B     74677     0
process_cache               4168    383    468    2129920B  92796816     0
numaio_obj_cache             328    112    192      65536B       425     0
numaio_grp_cache             144      9     56       8192B        27     0
mac_impl_cache             13488      6      9     122880B         8     0
mac_ring_cache               480      8     30      16384B        12     0
mac_block_cache              152      0      0          0B         0     0
mac_descriptor_cache          64      0      0          0B         0     0
mac_packet_pool_cache       1184      0      0          0B         0     0
mac_magazine_cache           552      0      0          0B         0     0
flow_tab_cache_0             184      5     42       8192B         7     0
flow_entry_cache_0         22440     12     20     450560B        26     0
mac_bcast_grp_cache           80      7    101       8192B        19     0
mac_client_impl_cache       2064      6     11      24576B         8     0
mac_promisc_impl_cache       120      0      0          0B         0     0
ip_minor_arena_sa_1            1     46    384        384B   1721432     0
ip_minor_arena_la_1            1     65   1088       1088B    959164     0
ip_conn_cache                744      3     60      49152B      1193     0
tcp_conn_cache              2120    314    693    1548288B    737336     0
udp_conn_cache              1256     69    324     442368B   2592397     0
rawip_conn_cache            1096      0    140     163840B     17159     0
rts_conn_cache               816      8     27      24576B        30     0
ire_cache                    352    153    168      65536B       202     0
ncec_cache                   200    110    279      73728B      6158     0
nce_cache                    112    117    378      49152B      6454     0
rt_entry                     152    133    168      32768B       180     0
radix_mask                    32      8    253       8192B        15     0
radix_node                   120      5     67       8192B         5     0
ipsec_actions                 88      0      0          0B         0     0
ipsec_selectors               80      0      0          0B         0     0
ipsec_policy                  80      0      0          0B         0     0
tcp_timercache                88   1239   2024     180224B    726650     0
tcp_notsack_blk_cache         24      1   1690      40960B   2083719     0
squeue_cache                 168    100    126      24576B       100     0
sctp_conn_cache             2608      0      0          0B         0     0
sctp_faddr_cache             472      0      0          0B         0     0
sctp_set_cache                24      0      0          0B         0     0
sctp_ftsn_set_cache           16      0      0          0B         0     0
dce_cache                    152    179    265      40960B       239     0
ire_gw_secattr_cache          24      0      0          0B         0     0
ldc_memhdl_cache              48      0      0          0B         0     0
ldc_memseg_cache              64      0      0          0B         0     0
fnode_cache                  176      9     84      16384B    121213     0
pipe_cache                   320     66    350     114688B  57668150     0
snode_cache                  152    858   1590     245760B 208309593     0
clnt_clts_endpnt_cache        88      0      0          0B         0     0
bpmap_cache                  200      0      0          0B         0     0
zio_cache                    912     26  76848   78692352B 2230982187     0
zio_link_cache                48      0  82472    3997696B 3836987868     0
sa_cache                      56  87949 132675    7495680B 197787650     0
dnode_t                      696 954000 954371  710746112B 104337805     0
dmu_buf_impl_t               216 1769940 2097530  464404480B 206659141     0
arc_elink_t                   32 967948 1389982   45006848B 3414386709     0
arc_buf_t                    168 992393 1401744  239230976B 559040333     0
arc_ref_t                     72 1769981 2097760  153436160B  54845411     0
arc_ghost_t                   64 314633 317646   20652032B 208588997     0
arc_meta                     184 956283 1262932  235134976B  57748154     0
arc_data                     184 915092 924264  172081152B 109836972     0
arc_data_512                  16 743953 756951   12230656B 122178970     0
arc_meta_512                  16 843315 1134159   18325504B  82077912     0
arc_data_1024                 16   3656  12168     196608B   1870065     0
arc_meta_1024                 16    380   2535      40960B    291201     0
arc_data_1536                 16   2409   8112     131072B   1191893     0
arc_meta_1536                 16     72   1521      24576B     43830     0
arc_data_2048                 16   1994   8112     131072B   1125267     0
arc_meta_2048                 16    307   2028      32768B     76657     0
arc_data_3072                 16   2210  11154     180224B   3490145     0
arc_meta_3072                 16     50   1521      24576B     20353     0
arc_data_4096                 16   1369  15210     245760B   5126190     0
arc_meta_4096                 16   5662   7098     114688B   1601065     0
arc_data_6144                 16   1864  10647     172032B   2382865     0
arc_meta_6144                 16     20   1521      24576B     40232     0
arc_data_8192                 16    998   7605     122880B   1823731     0
arc_meta_8192                 16     15   1014      16384B      3434     0
arc_data_12288                16    958   7605     122880B   1551591     0
arc_meta_12288                16     32   1014      16384B      8738     0
arc_data_16384                16    513   5577      90112B   1210128     0
arc_meta_16384                16  90237 305214    4931584B  15250828     0
arc_data_24576                16    458  20787     335872B  10999937     0
arc_meta_24576                16     19   1014      16384B      5337     0
arc_data_32768                16    428  25857     417792B   7619052     0
arc_meta_32768                16     10   1014      16384B      2976     0
arc_data_40960                16    163  22308     360448B  15904627     0
arc_meta_40960                16     13   1521      24576B     22600     0
arc_data_49152                16    227  19266     311296B   7580554     0
arc_meta_49152                16      1    507       8192B       205     0
arc_data_57344                16   2956  13182     212992B   3697718     0
arc_meta_57344                16      0    507       8192B       175     0
arc_data_65536                16   1638   3042      49152B    967486     0
arc_meta_65536                16      0    507       8192B       359     0
arc_data_73728                16     94   2028      32768B    569502     0
arc_meta_73728                16      4    507       8192B       356     0
arc_data_81920                16    448   1521      24576B    528592     0
arc_meta_81920                16      3    507       8192B       542     0
arc_data_90112                16   1492   2535      40960B    675906     0
arc_meta_90112                16      1    507       8192B       204     0
arc_data_98304                16     68   1521      24576B    428205     0
arc_meta_98304                16      1    507       8192B       151     0
arc_data_106496               16     47   1521      24576B    381247     0
arc_meta_106496               16      1    507       8192B       248     0
arc_data_114688               16     32   1521      24576B    387053     0
arc_meta_114688               16      1   1014      16384B      5162     0
arc_data_122880               16     23   1521      24576B    396002     0
arc_meta_122880               16      1    507       8192B       263     0
arc_data_131072               16 126094 142467    2301952B 3965382018     0
arc_meta_131072               16     29   1521      24576B    316793     0
arc_data_139264               16      0      0          0B         0     0
arc_meta_139264               16      0      0          0B         0     0
arc_data_262144               16      0      0          0B         0     0
arc_meta_262144               16      0      0          0B         0     0
arc_data_524288               16      0      0          0B         0     0
arc_meta_524288               16      0      0          0B         0     0
arc_data_1048576              16      0      0          0B         0     0
arc_meta_1048576              16      0      0          0B         0     0
l2arc_seg_t                   96      0      0          0B         0     0
l2arc_buf_t                   80      0      0          0B         0     0
zfetch_trigger_t              80    189   2525     204800B  61215486     0
space_seg_cache               64  35260 482076   31342592B 2637153477     0
dsl_share_t                  328      1     24       8192B         1     0
dsl_share_state_t             48      0      0          0B         0     0
zil_lwb_cache                208      6    624     131072B    736315     0
zil_train_cache               64      2   1386      90112B   2295082     0
zil_car_cache                 56      2   1885     106496B   3394521     0
zil_ian_cache                 80     32   1818     147456B  63267405     0
vdev_disk_cache              256      0    341      90112B 882654057     0
zfs_znode_cache              328  87949  91200   31129600B 243237064     0
dls_link_cache               344      6     23       8192B         9     0
dls_devnet_cache             368      6     22       8192B         8     0
px0_px0_0_cache1            8192     12     32     262144B 112535534     0
px0_px0_0_cache2           16384      3      8     131072B      9249     0
px0_px0_0_cache8           65536      2      2     131072B         2     0
dv_node_cache                184    664    748     139264B      1211     0
px0_mpt_sas0_2_cache1       8192      2     32     262144B 296721559     0
px0_mpt_sas0_2_cache2      16384      0     16     262144B  99973488     0
pkt_cache_mpt_sas_0          720      0    110      81920B 290898182     0
px0_mpt_sas1_3_cache1       8192      4     48     393216B 737561670     0
px0_mpt_sas1_3_cache2      16384      0     32     524288B 281684410     0
pkt_cache_mpt_sas_1          720      2    132      98304B 763554977     0
sdev_node_cache              248    905    992     253952B     51623     0
audit_proc                    48    385   1521      73728B  80880005     0
drv_secobj_cache             296      0      0          0B         0     0
dld_str_cache                320     11    150      49152B    147472     0
exacct_object_cache           40      0      0          0B         0     0
rw_numa_cache                128  19911  20853    2711552B   2551080     0
kssl_cache                  1624      0      0          0B         0     0
stmf_task_event_cache         64      0      0          0B         0     0
stmf_task_cache             3296      0      0          0B         0     0
stmf_ref_node_cache           16      2    507       8192B         6     0
sbd_task_cache              1304      0      0          0B         0     0
namefs_inodes_1                1     47   1152       1152B     74469     0
port_cache                    80     13    101       8192B        51     0
socket_cache                 792    321    720     589824B   1700687     0
socktpi_cache               1096      0      7       8192B         6     0
socktpi_unix_cache          1096     32    350     409600B    170699     0
sock_sod_cache               656      0      0          0B         0     0
tl_cache                     448     78    396     180224B    171292     0
keysock_1                      1      0     64         64B         1     0
spdsock_1                      1      0     64         64B         5     0
rds_alloc_cache               88      0      0          0B         0     0
dtrace_state_cache        262144      0     14    3670016B        29     0
idm_buf_cache                256      0      0          0B         0     0
idm_task_cache              1408      0      0          0B         0     0
idm_tx_pdu_cache             400      0      0          0B         0     0
idm_rx_pdu_cache             596      0      0          0B         0     0
softmac_cache                568      5     14       8192B         7     0
softmac_upper_cache          232      0      0          0B         0     0
fctl_cache                   112      0      0          0B         0     0
vldc_cookie_buf_cache     262144      0     65   17039360B  33963567     0
authkern_cache                72      0    784      57344B 205205946     0
authnone_cache                72      0      0          0B         0     0
authloopback_cache            72      0      0          0B         0     0
authdes_cache_handle          80      0      0          0B         0     0
rnode_cache                  680    982    990     737280B     61171     0
nfs_access_cache              56    182   3190     180224B    145036     0
client_handle_cache           32     17    506      16384B      1850     0
rnode4_cache                1032      0      0          0B         0     0
svnode_cache                  40      0      0          0B         0     0
nfs4_access_cache             56      0      0          0B         0     0
client_handle4_cache          32      0      0          0B         0     0
nfs4_ace4vals_cache           48      0      0          0B         0     0
nfs4_ace4_list_cache         264      0      0          0B         0     0
NFS_idmap_cache               56      0      0          0B         0     0
lm_xprt_10003c62cf40          32      0      0          0B         0     0
lm_vnode_10003c62cf40        184      0      0          0B         0     0
lm_sysid_10003c62cf40        160      0     50       8192B         1     0
lm_client_10003c62cf40       128      0     63       8192B         1     0
lm_async_10003c62cf40         32      0      0          0B         0     0
lm_sleep_10003c62cf40         96      0      0          0B         0     0
lm_config_10003c62cf40        80      2    101       8192B         2     0
uvfs_uvnode_cache            392      0      0          0B         0     0
uvfs_task_sync_cache          16      0      0          0B         0     0
uvfs_task_rootvp_cache        16      0      0          0B         0     0
uvfsvfs_cache                280      0      0          0B         0     0
ufs_inode_cache              368      0      0          0B         0     0
directio_buf_cache           272      0      0          0B         0     0
lufs_save                     24      0      0          0B         0     0
lufs_bufs                    256      0      0          0B         0     0
lufs_mapentry_cache          112      0      0          0B         0     0
pty_map                       64     54    756      49152B     10729     0
sppptun_map                  440      0      0          0B         0     0
Hex0x100030bf3428_minor_1      1      0      0          0B         0     0
Hex0x100030bf3430_minor_1      1      0      0          0B         0     0
px0_igb3_4_cache1           8192   1793   1840   15073280B 169264446     0
px0_igb3_4_cache2          16384      1     24     393216B 194088248     0
px0_igb0_5_cache1           8192   1793   2000   16384000B 3804956026     0
px0_igb0_5_cache2          16384      1     24     393216B 653588623     0
iscsit_status_pdu_cache      400      0      0          0B         0     0
stp_2_0_987                 1712      0      0          0B         0     0
stp_m2_0_987                  56      0      0          0B         0     0
audit_buffer                 152      0    212      32768B       690     0
lnode_cache                   32     12   1265      40960B 247872400     0
flow_tab_cache_1             184      1     42       8192B         1     0
flow_entry_cache_1         22440      1      4      90112B         1     0
lm_xprt_10003c62fa40          32      0      0          0B         0     0
lm_vnode_10003c62fa40        184      0      0          0B         0     0
lm_sysid_10003c62fa40        160      0      0          0B         0     0
lm_client_10003c62fa40       128      0      0          0B         0     0
lm_async_10003c62fa40         32      0      0          0B         0     0
lm_sleep_10003c62fa40         96      0      0          0B         0     0
lm_config_10003c62fa40        80      1    101       8192B         1     0
vnic_cache                  1544      1      5       8192B         1     0
crypto_session_cache         104      0      0          0B         0     0
sdp_generic_table             32      0      0          0B         0     0
sdp_advt_cache                80      0      0          0B         0     0
sdp_advt_table                24      0      0          0B         0     0
sdp_conn_cache              1944      0      0          0B         0     0
Hex0x10004c7d1428_minor_1      1      0      0          0B         0     0
Hex0x10004c7d1430_minor_1      1      0      0          0B         0     0
stp_3_1_3841                1712      0      0          0B         0     0
stp_m3_1_3841                 56      0      0          0B         0     0
stp_2_1_3866                1712      0      0          0B         0     0
stp_m2_1_3866                 56      0      0          0B         0     0
fcsm_job_cache               104      0      0          0B         0     0
aggr_port_cache             1032      0      0          0B         0     0
aggr_grp_cache              1008      0      0          0B         0     0
iptun_cache                  288      0      0          0B         0     0
smb_shr_notify_cache          72      0      0          0B         0     0
smb_share_cache              168      1     48       8192B         1     0
smb_vfs_cache                 48      0      0          0B         0     0
smb_mc_cache                  96      0      0          0B         0     0
smb_uio_cache                752      0      0          0B         0     0
smb_node_cache               752      0      0          0B         0     0
smb_txreq                  66592      0      0          0B         0     0
vxlan_grp_cache              200      0      0          0B         0     0
vxlan_cache                  904      0      0          0B         0     0
zvsdir_zvnode_cache           96      0      0          0B         0     0
px0_igb1_6_cache1           8192      0     32     262144B      3584     0
px0_igb1_6_cache2          16384      0      8     131072B         1     0
px0_igb2_7_cache1           8192      0     32     262144B      3584     0
px0_igb2_7_cache2          16384      0      8     131072B         1     0
------------------------- ------ ------ ------ ---------- --------- -----
Total [hat_memload]                             124813312B 230714082     0
Total [kmem_msb]                                 76464128B 148885992     0
Total [kmem_va]                                 158597120B    111782     0
Total [kmem_default]                           3346472960B 2897004240     0
Total [bp_map]                                    6029312B      7314     0
Total [kmem_tsb_default]                          1785856B 130478770     0
Total [hat_memload1]                            124583936B 975785584     0
Total [segkmem_ppa]                               3145728B        16     0
Total [umem_np]                                  14155776B   1548423     0
Total [id32]                                        24576B  61202888     0
Total [segkp]                                   151126016B  74195610     0
Total [ip_minor_arena_sa]                             384B   1721432     0
Total [ip_minor_arena_la]                            1088B    959164     0
Total [px0_px0_0_vmem_top]                          524288B 112544785     0
Total [px0_mpt_sas0_2_vmem_top]                          524288B 396695047     0
Total [px0_mpt_sas1_3_vmem_top]                          917504B 1019246080     0
Total [namefs_inodes]                                1152B     74469     0
Total [keysock]                                        64B         1     0
Total [spdsock]                                        64B         5     0
Total [px0_igb3_4_vmem_top]                        15466496B 363352694     0
Total [px0_igb0_5_vmem_top]                        16777216B 163577353     0
Total [px0_igb1_6_vmem_top]                          393216B      3585     0
Total [px0_igb2_7_vmem_top]                          393216B      3585     0
------------------------- ------ ------ ------ ---------- --------- -----

vmem                         memory     memory    memory     alloc alloc
name                         in use      total    import   succeed  fail
------------------------- ---------- ----------- ---------- --------- -----
heap                      4418913714176B 17592186044416B         0B    445761     0
    vmem_metadata          14204928B   14417920B  14417920B      1625     0
        vmem_seg           42156032B   42156032B  42156032B      5141     0
        vmem_hash           5945344B    5955584B   5955584B       378     0
        vmem_vmem            443992B     513592B    475136B       171     0
    heap_alloc                61760B      65536B     65536B       198     0
    hat_memload           124813312B  124813312B 124813312B     17940     0
    kstat                   1317944B    1351680B   1286144B   1091062     0
    kmem_metadata          61620224B   97779712B  97779712B    115733     0
        kmem_msb           76464128B   76464128B  76464128B    130218     0
        kmem_audit                0B          0B         0B         0     0
        kmem_cache          2236176B    3907584B   3907584B       552     0
        kmem_hash            530944B     540672B    540672B      1038     0
    kmem_log                4204640B    4210688B   4210688B         6     0
    kmem_firewall_va              0B          0B         0B         0     0
        kmem_firewall             0B          0B         0B         0     0
    mod_sysfile                   8B       8192B      8192B         1     0
    kmem_oversize         20114661376B 20122435584B 20122435584B    333208     0
    kmem_va               210501632B  210501632B 210501632B     43084     0
        kmem_default      3346472960B 3925639168B 3925639168B   5790429     0
    little_endian           1162048B    1228800B   1228800B 104879936     0
    big_endian             36849813B   58744832B  58744832B  60100613     0
    bp_map                  6029312B    6029312B   6029312B      6402     0
    ksyms                   3267768B    3317760B   3317760B       418     0
    ctf                      278772B     327680B    327680B       420     0
    kmem_bigtsb                   0B          0B         0B         0     0
        kmem_bigtsb_default         0B          0B         0B         0     0
    kmem_tsb               12582912B   12582912B  12582912B       610     0
        kmem_tsb_default   10174464B   12582912B  12582912B  28176218   579
    hat_memload1          124583936B  124583936B 124583936B     15214     0
    KOM firewall                  0B          0B         0B         0     0
    segkmem_ppa             3145728B    3145728B   3145728B         3     0
    umem_np                14155776B   14155776B  14155776B     24604     0
    contig_mem_arena       30666816B  222298112B         0B 133475242     0
    contig_mem_arena_le           0B          0B         0B         0     0
    defdump_arena         1147854848B 1147854848B 1147854848B         6     0
        defdump_metadata_arena 1147854848B 1147854848B 1147854848B         6     0
lppool                    154394624B  154394624B         0B     31840 138056
heap32                     10772480B  134217728B         0B       126     0
    id32                      24576B      24576B     24576B         3     0
    module_data             8638543B    8814592B   8290304B       596     0
    promplat                      0B          0B         0B        79     0
    trapstat                      0B          0B         0B         0     0
heaptext                   33562624B  134217728B         0B        17     0
    module_text            33551680B   37863424B         0B       421     0
logminor_space                   61B     262137B         0B    942755     0
taskq_id_arena                  146B 2147483647B         0B       240     0
heap_lp                   4026531840B 4397241204736B         0B        16     0
    kmem_lp               4026531840B 4026531840B 4026531840B      7466  4150
segkp                     151420928B 2147483648B         0B      4905     0
rctl_ids                         44B      32767B         0B        44     0
zoneid_space                      1B       9998B         0B         1     0
taskid_space                    160B     999999B         0B   1048209     0
pool_ids                          0B     999998B         0B         0     0
contracts                       159B 2147483646B         0B    853997     0
regspec                     9175040B 5368709120B         0B        31     0
mac_minor_ids                   116B     130070B         0B       792     0
ip_minor_arena_sa               384B     262140B         0B         6     0
ip_minor_arena_la              1088B 4294705152B         0B        17     0
px0_px0_0_vmem_top           655360B 1878917120B         0B      2135     8
    px0_px0_0_vmem_16             0B          0B         0B      2125     0
px0_px0_0_vmem_c                  0B  268435456B         0B         0     9
px0_mpt_sas0_2_vmem_top     1441792B  939393024B         0B  41273125     8
    px0_mpt_sas0_2_vmem_16    262144B     262144B    262144B  35835132     0
px0_mpt_sas0_2_vmem_c             0B  134217728B         0B         0     9
px0_mpt_sas1_3_vmem_top     1835008B  939393024B         0B 110855090     8
    px0_mpt_sas1_3_vmem_16    262144B     262144B    262144B  94440293     0
px0_mpt_sas1_3_vmem_c             0B  134217728B         0B         0     9
lib_va_32                   7954432B 2031599616B         0B        20     0
lib_va_64                 283328512B 2251793356234752B         0B       211     0
namefs_inodes                  1152B      65536B         0B        18     0
tl_minor_space                   78B     262138B         0B    163787     0
keysock                          64B 4294967295B         0B         1     0
spdsock                          64B 4294967295B         0B         1     0
dtrace                       104429B 4294967295B         0B   1219732     0
dtrace_minor                      0B 4294967293B         0B        28     0
syseventd_channel                15B        101B         0B     24474     0
syseventd_channel                 1B          2B         0B         1     0
idm_taskid_space                  0B      65536B         0B         0     0
module_text_holesrc_2             0B    4194304B         0B         0     0
    ktext_hole_2            1862280B    4194304B         0B       156     0
module_text_holesrc_0             0B    4194304B         0B         0     0
    ktext_hole_0            2155704B    4194304B         0B        20     0
ibcm_local_sid                    0B 4294967295B         0B         0     0
ibcm_ip_sid                       0B      65535B         0B         0     0
lmsysid_space                     1B      16383B         0B         3     0
module_text_holesrc_3             0B    4194304B         0B         0     0
    ktext_hole_3            1924624B    4194304B         0B       154     0
module_text_holesrc_4             0B    4194304B         0B         0     0
    ktext_hole_4            2109924B    4194304B         0B        72     0
module_text_holesrc_1             0B    4194304B         0B         0     0
    ktext_hole_1             236992B    4194304B         0B        32     0
logdmux_minor                    34B        256B         0B      5028     0
ptms_minor                       54B        128B         0B     10703     3
sppptun_minor                     0B         16B         0B         0     0
syseventconfd_door                1B        101B         0B         1     0
syseventconfd_door                1B          2B         0B         1     0
devfsadm_event_channel            1B        101B         0B         1     0
devfsadm_event_channel            1B          2B         0B         1     0
Hex0x100030bf3428_minor           0B 4294967294B         0B         0     0
Hex0x100030bf3430_minor           0B 4294967294B         0B         0     0
px0_igb3_4_vmem_top        15597568B  939393024B         0B  12335257     8
    px0_igb3_4_vmem_16            0B          0B         0B  12334856     0
px0_igb3_4_vmem_c                 0B  134217728B         0B         0     9
px0_igb0_5_vmem_top        16908288B  939393024B         0B    362737     8
    px0_igb0_5_vmem_16            0B          0B         0B    362449     0
px0_igb0_5_vmem_c                 0B  134217728B         0B         0     9
iscsit_tsih_pool                  0B      65535B         0B         0     0
ipnet_minor_space                 6B     262141B         0B         6     0
crypto                            0B         16B         0B    136281     0
lofi_id                           0B      16383B         0B         0     0
ds_minors                         0B     262140B         0B         0     0
Hex0x10004c7d1428_minor           0B 4294967294B         0B         0     0
Hex0x10004c7d1430_minor           0B 4294967294B         0B         0     0
semids                           90B        128B         0B        90     0
mdesc_minor                       0B        256B         0B    120953     0
aggr_portids                      0B      65534B         0B         0     0
aggr_key_ids                      0B      64535B         0B         0     0
zvmm_minor_space                  0B     262142B         0B         0     0
px0_igb1_6_vmem_top          524288B  939393024B         0B       115     8
    px0_igb1_6_vmem_16            0B          0B         0B         0     0
px0_igb1_6_vmem_c                 0B  134217728B         0B         0     9
px0_igb2_7_vmem_top          524288B  939393024B         0B       115     8
    px0_igb2_7_vmem_16            0B          0B         0B         0     0
px0_igb2_7_vmem_c                 0B  134217728B         0B         0     9
msqids                            0B        128B         0B         0     0
shmids                            0B        128B         0B         0     0
------------------------- ---------- ----------- ---------- --------- -----
>
>
> ::kmastat !awk '!/Total/ {print $4 " " $1}' | sort -n | tail
76464128B kmem_msb
97779712B kmem_metadata
124583936B hat_memload1
124813312B hat_memload
210501632B kmem_va
1147854848B defdump_arena
1147854848B defdump_metadata_arena
3925639168B kmem_default
4026531840B kmem_lp
20122435584B kmem_oversize
>
>
> ::vmem ! grep kmem_oversize
000003000008a000   kmem_oversize        20114661376  20122435584    333212     0
> 000003000008a000::print vmem_t vm_kstat.vk_free.value.l
vm_kstat.vk_free.value.l = 0x50304
> 000003000008a000::print vmem_t vm_kstat.vk_alloc.value.l
vm_kstat.vk_alloc.value.l = 0x5159d

In newer solaris releases we see the ZFS buffers as own lines in mdb... that's part of the kernel memory:

> ::memstat
Page Summary                 Pages             Bytes  %Tot
----------------- ----------------  ----------------  ----
Kernel                     3129843             23.8G   76%
Defdump prealloc            140119              1.0G    3%
ZFS Metadata                292857              2.2G    7%
ZFS File Data              2251900             17.1G   55%
Anon                        158898              1.2G    4%
Exec and libs                 6226             48.6M    0%
Page cache                    9878             77.1M    0%
failed to read 'mrp_svc'; module not present
Free (cachelist)                52              416k    0%
Free (freelist)             121576            949.8M    3%
Total                      4128768             31.5G
>


What is happening there? Well... IO IO IO :-)

root@server:~#  dtrace -n 'fbt::vmem_alloc:entry { @[args[0]->vm_name] = sum(arg1); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C

  ip_minor_arena_la                                                 1
  namefs_inodes                                                     1
  ip_minor_arena_sa                                                 2
  logminor_space                                                    2
  little_endian                                                119168
  segkp                                                        188416
  big_endian                                                  1397837
  heap                                                        3145728
  kmem_oversize                                               3145728
  px0_mpt_sas0_2_vmem_16                                      5767168
  px0_mpt_sas0_2_vmem_top                                     5767168
  px0_mpt_sas1_3_vmem_16                                     15990784
  px0_mpt_sas1_3_vmem_top                                    15990784
root@server:~#
root@server:~# dtrace -n 'fbt::vmem_alloc:entry /args[0]->vm_name == "px0_mpt_sas1_3_vmem_top"/ { @[stack()] = count(); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C


              genunix`vmem_xalloc+0x670
              genunix`vmem_alloc+0x21c
              px`px_dvma_pool_default_dvma_alloc+0x180
              px`px_atu_dvma_alloc+0x94
              px`px_dvma_map+0x54
              px`px_dma_bindhdl+0xbc
              genunix`ddi_dma_buf_bind_handle+0x54
              scsi`scsi_cache_bind+0x24
              scsi`scsi_cache_init_pkt+0x2d4
              scsi`scsi_init_pkt+0x4c
              scsi_vhci`vhci_bind_transport+0x9ac
              scsi_vhci`vhci_scsi_start+0x350
              sd`sd_start_cmds+0x3a4
              sd`sd_core_iostart+0x228
              sd`sd_mapblockaddr_iostart+0x210
              sd`xbuf_iostart+0x20c
              zfs`vdev_disk_strategy+0x30
              zfs`vdev_disk_io_start+0x26c
              zfs`zio_execute+0xf4
              zfs`vdev_queue_io_done+0xb4
                9

              genunix`vmem_xalloc+0x670
              genunix`vmem_alloc+0x21c
              px`px_dvma_pool_default_dvma_alloc+0x180
              px`px_atu_dvma_alloc+0x94
              px`px_dvma_map+0x54
              px`px_dma_bindhdl+0xbc
              genunix`ddi_dma_buf_bind_handle+0x54
              scsi`scsi_cache_bind+0x24
              scsi`scsi_cache_init_pkt+0x2d4
              scsi`scsi_init_pkt+0x4c
              scsi_vhci`vhci_bind_transport+0x9ac
              scsi_vhci`vhci_scsi_start+0x350
              sd`sd_start_cmds+0x3a4
              sd`sd_core_iostart+0x228
              sd`sd_mapblockaddr_iostart+0x210
              sd`xbuf_iostart+0x20c
              zfs`vdev_disk_strategy+0x30
              zfs`vdev_disk_io_start+0x26c
              zfs`zio_execute+0xf4
              zfs`vdev_raidz_io_start+0x26c
               16

              genunix`vmem_xalloc+0x670
              genunix`vmem_alloc+0x21c
              px`px_dvma_pool_default_dvma_alloc+0x180
              px`px_atu_dvma_alloc+0x94
              px`px_dvma_map+0x54
              px`px_dma_bindhdl+0xbc
              genunix`ddi_dma_buf_bind_handle+0x54
              scsi`scsi_cache_bind+0x24
              scsi`scsi_cache_init_pkt+0x2d4
              scsi`scsi_init_pkt+0x4c
              scsi_vhci`vhci_bind_transport+0x9ac
              scsi_vhci`vhci_scsi_start+0x350
              sd`sd_start_cmds+0x3a4
              sd`sd_core_iostart+0x228
              sd`sd_mapblockaddr_iostart+0x210
              sd`xbuf_iostart+0x20c
              zfs`vdev_disk_strategy+0x30
              zfs`vdev_disk_io_start+0x26c
              zfs`zio_execute+0xf4
              zfs`vdev_queue_io_done+0x94
              110
root@server:~# dtrace -n 'fbt::vmem_alloc:entry /args[0]->vm_name == "kmem_oversize"/ { @[stack()] = count(); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C


              genunix`kmem_alloc+0x160
              genunix`kmem_zalloc+0x120
              autofs`auto_calldaemon+0x1d0
              autofs`auto_null_request+0x24
              autofs`unmount_tree+0x60
              autofs`unmount_zone_tree+0xc
              unix`thread_start+0x4
                1
root@server:~#