[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"$f2z86JmP3kPWiYlbAIdGFEf3vLg4UwaP_Qe5Ey-JNbas":3},{"item":4,"related":44},{"id":5,"type":6,"title":7,"slug":8,"summary":9,"body":10,"coverUrl":11,"productScreenshots":12,"productLinks":13,"authorName":14,"authorUrl":15,"authorSubject":16,"category":17,"tags":22,"sourceLabel":35,"sourceName":36,"sourceUrl":37,"status":38,"seoTitle":7,"seoDescription":9,"canonicalUrl":35,"isFeatured":39,"sno":40,"sortOrder":41,"publishedAt":42,"updatedAt":43,"createdAt":43},"3158b1ab-fc59-4a63-bf04-fe51a57b5029","article","GQA：大模型怎样在注意力质量和 KV 缓存之间折中？","grouped-query-attention-gqa-kv-cache","GQA 将查询头分组，让组内共享 Key\u002FValue 头，在多头注意力与多查询注意力之间寻找折中。本文说明它如何影响 KV cache，并提醒论文中的质量与速度表现依赖模型和训练条件。","大语言模型生成每个新 token 时，需要反复参考此前 token 的 Key 和 Value。为了避免每一步都重算，推理系统通常把这些中间状态放进 KV cache。上下文越长、并发用户越多，缓存占用就越明显；而多头注意力（MHA）为每个查询头保存自己的 Key\u002FValue，成本可能很高。\n\nGrouped Query Attention（GQA，分组查询注意力）在 MHA 与 Multi-Query Attention（MQA）之间取了一个折中。MHA 通常让每个查询头拥有一组独立的 Key\u002FValue 头；MQA 则让许多查询头共享同一组 Key\u002FValue；GQA 把查询头分成若干组，每组共享一个或少数几个 Key\u002FValue 头。组数不同，缓存规模和模型表达能力也随之改变。\n\n一个简化例子：如果有 16 个查询头，MHA 可能对应 16 组 Key\u002FValue；MQA 可能让 16 个查询头共同使用 1 组；GQA 则可能分成 4 组，每组查询头共享一组 Key\u002FValue。实际模型的头数和维度各不相同，这只是解释结构的示意。由于 KV cache 与 Key\u002FValue 头数相关，GQA 通常能比 MHA 少存储一部分缓存数据，并减少推理时的内存带宽压力。\n\n为什么不一律使用 MQA？共享得越多，模型需要在有限的 Key\u002FValue 表示中服务更多查询头，可能影响质量。GQA 通过保留多个分组，让模型在缓存效率与表达能力之间寻找平衡。原始研究展示了在特定模型规模、训练和 uptraining 设置下，GQA 能兼顾接近 MHA 的质量与更好的推理速度；这并不等于所有模型、所有任务都必然得到同样收益。\n\n可以把注意力头想成一组记者：每位记者提出不同问题（Query），但需要查阅采访记录（Key\u002FValue）。MHA 给每位记者单独建档，档案精细却占空间；MQA 让所有人共用一本档案，最省空间却可能不够丰富；GQA 则让若干记者共享一套档案，形成中间方案。\n\nGQA 改的是注意力内部 Key\u002FValue 头的组织方式，不是把注意力机制替换成检索，也不意味着缓存消失。实际性能还取决于模型是否按这种结构训练、推理引擎是否优化对应内核，以及硬件瓶颈位于计算还是内存带宽。阅读模型说明时，关注查询头与 KV 头数量的比例，往往就能快速看出它在缓存方面做了怎样的取舍。\n\n来源：[Ainslie 等：GQA，EMNLP 2023](https:\u002F\u002Faclanthology.org\u002F2023.emnlp-main.298.pdf)","\u002Fuploads\u002F2026-10-13\u002F8f839d66-84e3-4571-a370-c2133c75bddd.jpg",[],[],"Foundit","https:\u002F\u002Ffoundit.cn","foundit-ai-editorial",{"id":18,"name":19,"slug":20,"description":21},"6179d3b6-dc34-4483-9ded-3cd9f1b37a47","科普","abbreviation","介绍各领域新兴概念",[23,27,31],{"id":24,"name":25,"slug":26},"7c76bfc2-f80f-4ee0-a95d-27bd8708b434","技术","slug",{"id":28,"name":29,"slug":30},"88d2bc27-0e0f-468a-b907-2991cb97b87b","人工智能","ai",{"id":32,"name":33,"slug":34},"4c2bbea6-eab7-40a8-8447-1de478ff7749","分析","analyse",null,"EMNLP 2023 \u002F ACL Anthology","https:\u002F\u002Faclanthology.org\u002F2023.emnlp-main.298.pdf","published",false,59,0,"2026-10-13T00:00:00.000Z","2026-10-13T02:14:04.159Z",[45,54,62],{"id":46,"type":6,"title":47,"slug":48,"summary":49,"coverUrl":50,"authorName":14,"sno":51,"publishedAt":52,"createdAt":53},"872963cd-242c-4912-a3dc-9497386d4772","Certificate Transparency：TLS 证书为什么要公开留日志？","certificate-transparency-public-audit-logs","证书透明度通过公开、可验证的日志提升 TLS 证书签发的可观察性。本文说明监控者如何发现意外证书，也厘清 CT 不会自动阻止误发。","\u002Fuploads\u002F2026-10-06\u002Fa2b8f9d0-fb90-4f19-ae2b-22cdebd395f2.jpg",43,"2026-10-06T00:00:00.000Z","2026-10-06T07:51:39.208Z",{"id":55,"type":6,"title":56,"slug":57,"summary":58,"coverUrl":59,"authorName":14,"sno":60,"publishedAt":42,"createdAt":61},"3eee805c-a6c6-438b-b41c-bb67359211f6","HNSW：向量数据库怎样在“小世界图”里快速找相似项？","hnsw-approximate-nearest-neighbor-search","HNSW 把向量组织为多层可导航图，从稀疏高层逐步下探到局部候选区域，以较少距离计算近似查找邻居。本文讲解图搜索原理，以及召回率、延迟、构建和内存之间的权衡。","\u002Fuploads\u002F2026-10-13\u002Feb9f9a0c-fcbb-4323-86ea-58d5ca17c9f2.jpg",47,"2026-10-13T02:14:06.187Z",{"id":63,"type":6,"title":64,"slug":65,"summary":66,"coverUrl":67,"authorName":14,"sno":68,"publishedAt":42,"createdAt":69},"0d29aae9-204d-474e-8eaf-e7ba96fc8064","FlashAttention：注意力计算没变，为什么显卡能少搬很多数据？","flashattention-io-aware-transformer-attention","FlashAttention 通过分块计算与在线 softmax 减少 GPU 显存往返，并避免物化完整注意力矩阵。本文解释它为何加速长序列训练，同时指出密集注意力的二次计算量并未消失。","\u002Fuploads\u002F2026-10-13\u002Faf2e6109-e447-48bf-9729-874072603e3a.jpg",50,"2026-10-13T02:14:02.226Z"]