From d993516549304f78936b37a976bcf881c6106d58 Mon Sep 17 00:00:00 2001 From: TokenCare contributors Date: Fri, 23 Oct 2026 21:50:04 -0400 Subject: [PATCH] feat(tokencare): add context compression and token optimizer plugin Standard plugin (no UI surfaces). Unifies billion-context's incremental reversible compression with dsh-token-optimizer's output triage, dedup and monitoring into one layered plugin: automatic ledger via PostToolUse hooks, reversible /digest vault, /compress summaries anchoring built-in compaction, /context-search recovery, /token-status statistics with GLM-aware (CJK) token estimation and flash/balanced/max profiles. v0.2.2, MIT, zero-dependency Node. Includes original generated icon (assets/tokencare) with provenance record. --- assets/icon-sources.json | 8 + assets/tokencare/icon.png | Bin 0 -> 10773 bytes .../tokencare/.zcode-plugin/plugin.json | 17 + external_plugins/tokencare/LICENSE | 21 + external_plugins/tokencare/README.md | 42 ++ external_plugins/tokencare/README_CN.md | 46 ++ .../tokencare/commands/compress.md | 11 + .../tokencare/commands/context-search.md | 17 + external_plugins/tokencare/commands/digest.md | 17 + .../tokencare/commands/token-status.md | 13 + external_plugins/tokencare/hooks/hooks.json | 29 + external_plugins/tokencare/scripts/vault.mjs | 605 ++++++++++++++++++ .../tokencare/skills/tokencare/SKILL.md | 61 ++ marketplace.json | 27 + 14 files changed, 914 insertions(+) create mode 100644 assets/tokencare/icon.png create mode 100644 external_plugins/tokencare/.zcode-plugin/plugin.json create mode 100644 external_plugins/tokencare/LICENSE create mode 100644 external_plugins/tokencare/README.md create mode 100644 external_plugins/tokencare/README_CN.md create mode 100644 external_plugins/tokencare/commands/compress.md create mode 100644 external_plugins/tokencare/commands/context-search.md create mode 100644 external_plugins/tokencare/commands/digest.md create mode 100644 external_plugins/tokencare/commands/token-status.md create mode 100644 external_plugins/tokencare/hooks/hooks.json create mode 100644 external_plugins/tokencare/scripts/vault.mjs create mode 100644 external_plugins/tokencare/skills/tokencare/SKILL.md diff --git a/assets/icon-sources.json b/assets/icon-sources.json index 9a1fd00..c3d11d5 100644 --- a/assets/icon-sources.json +++ b/assets/icon-sources.json @@ -2552,5 +2552,13 @@ "homepage": "https://www.liblib.tv", "mimeType": "image/png", "sha256": "ada9717d540928baec64105056b93e8e8bc6cfbff145f9f3a9eadb6dd9b90689" + }, + { + "name": "tokencare", + "icon": "tokencare/icon.png", + "source": "generated original artwork for TokenCare (Pillow script: layered context lines compressing into one bar on an indigo-cyan gradient)", + "matchedBy": "generated-original", + "mimeType": "image/png", + "sha256": "3d567d102826140450531f70e4c683cb0bcf556d5b17b529a4edeb356b085180" } ] diff --git a/assets/tokencare/icon.png b/assets/tokencare/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..c21d99f4e35568b37cb74a8122e7655962629ef9 GIT binary patch literal 10773 zcmaKSby$?&)AzkJA}x{%k_u8%BDH{YN`q3;B`v)yN{FPibSd2+%>q)AONpR>ba$-8 zF7J)M@9%lO*Yn5w$6jaelXGU~oSFHY3D;CtAihC&0{{SGCB>&&0DuMl#R3Sf3qLy9 zG5~n+OX;b!j`z&YtWRvZzR%vln&?`cvb%5T4O~r(@BF=g~*<=nj z@4m|@p%p1h!7U*%J3g=R>=v%BBCPr1a@c+NC^HY?dQuAjPhu9x0RRF3OdxGI000Mz zQidS?yI6oGSW5tY{?l0fQaH#54}jtVz z0L%#q{!LCN*clE^2Cu}y`8V+d*L|Dd0MJQ62Ed8TfxoN&iwdUNAS|E|ToT+ZbRAsn zpM8>D54xuGUwxtAXYe(pB5xqTbzcHMu;!nwfV_bVx>BZqL}Y_i*Sq*v3lbOS|5oSg z?qdIz2mV&OaDe248Uh7JT~Ac&12XV$#eYk%Az01qf0*=lU;i-tFR26d`0Y_<0I~=I zx7HS(3%Y6-pk@Q~_OqRJTHc}J^2JGie=4QQ%Vi#>#H$S|zt!{ZU6)W%mx(SGSIPTO zX23gq8rSrF>yhLQLCT_MN_R9u)PA4dIoEw^*|sfb`PJSJvoXr0PBe8g&FxSov>CFG z`;=r)mL#gcVtgah#({J%B8y&mG1X&bg6^dm;f9!`P`|c8obPu<(bM5?B_q*uFxTCd z@m&8T2;-lXttj#j=n*omLJm{)H|dT}*B6sT1S8}Xku#RxRcH?_o$6dy60OjQ2H zW+*$Cbdey$SSg(K>B&mW#BS$S57x~u#_Ukn84 z_5_l=b!Z$7{IYC{dOzql7pIYdlcazpsA|Thv~oG#NqXsMZrx$n!;J+svRf%g>MzbU zIU3F#TgHAk#4zA<_DN*b)JI;Y*!L?^4yYXto%Ek>-HI~ax^N2EV1zDRY_;I%mHYa2 z_>okx0!FKTv>es0b?C;a4jDngIA}dWC1l#zH%1v*u}1l4cUz zvavpJs{9e}Pl{?Zk>CRLTFWujkYUSX5f* z%YE$Bf;`-7bmlop!^2W9#!9sthqlA$A(fX}yj$nqkZ~E$`s5VuswJFn!s}pGhETg%A-_y9(_uztOSmxgY2>?S( zG7u>#mf$i%jT|Hj&Z1Ycf8dmj)9&V#9}*O;$~KvbB)}2e8Ed7M*K+WtH28sIatg%0 zUD|N}T2~_+Hmr>gs9Z#7Cu&XZ`A9~~cxI}Zal%(%BCQM;8GgkwvRKGqfNc8Wg};k< z6qfy~CU{aB2*{IXQ_ z{+ZIpI6y^k+U;fJP4#>!|9q`JyMD_Ez&3pEQ}X?Zv|WSy`V!AO+Q-zcbe9>OvrG%U z9Sq#q0HXqecOLUOGg7~VoprQ&Ss-+!u<0J-7~QM+nEvjTLoUiDxRJtQe!+mh`|}N| zf~QhPhpKUIyyD2dZbk=7OHTBnmvzmDoU^bS*;HHO_a~$-PPf}+mCV>y;7r-l?tvEI zp_f8MecZrRI3k50oAz)XCd}>ydTtiUhS$YCX)A(a>%mqBFinWGnG?sdSc`*#563oF zW+?^8AATiEsg^S6{gv(ko__9O(*)`+xJ609>;nB4RGa3-xbu*>RMM?#c2Cu!{3?+* z(sa!Rw76O-m(*=SM@Y<{x6Zt$)M~BTeT;37BKZKATMWYVVQLh)1FO=?s@NK$%`+$I zkBlBIdK%|d4A;xZ#B!)}K;`lN_@q~haNx9DW6)i{e4FW-5ZucyqIuG({!hj8RRxD5 zVz)EJHMy2mh4Toxq*S217KFCn3|L&5SBkgxzN}?X82hWQrVp71*6;LdVFA3lWoH|> z^?RB-Z5)gED1FlCf}lkIP`N;?>bzn=hq=w@oeX5x~E4h~HHI zJbv`*^K$ypv|^rSH&KaX=UwRYdc032rONyjoT-mLrn|Ff>$Xj{s|4$nPRgk3PWZU0 zUW(p)wesT5HZ{>3zS#TAIoMJ@lz-+6E-#e}0X-KQ0Jyec3)h$bKeqSURMFKEs#Z|_ z*INCzrTR_uo8QJ}KsIPUnI3|M_g@yfGXS(9<-s_B3ma>yTDbc|ejY(K0BQ8L?HM`w z8k>^OT?3=<@A&^(2pk|uBj8sIrD>556>;e?RiqZK*@K~kUow+oYXUjeGD4h1OGRXW z?YY*7|71jR6#BGNTVO4V!O6f%!-&#=u9-=t0VDAEWAtEtb5;za{}Y&R{%wvnE{9Tb zg`2fYiib%fBnP)<%JF!6=X}mK1OyO&Hp_|B!~ z=S|j1af(_GJjd;toJl9YfwRcN&onB~SW3h5ici(WW|Ekr*7Rfemz0XfynC(rTe;c- zM7kp~pIm4d)Gd{TjNK3ipEFNN95;iLTadSBZ+u@DN`4)gpiC1H4%5TbGZwUyzObaOdYMx&%MLkScIhz=l0^hH>C7XIo8 z{!;fzDw|Tr7Q5Ru0Z_L@bCNj7&}SJpz!K}^C5Eycv6c)-XHS@w`q$nHC*?=XTekIr zu3xlhmgpbI&fp;~JK*w5?pQTk)^oJSS|$3~3mn;I3>}GN_kfTC zk259Ue8`-lTBD zqmx%rtZrL9$eOGA1j6IJFC)n>d-eEctg)fXJx=aNR3lXKIDmR30kQyZiN1cAmTHbj zKB8p)dHqiK;Z;E4M25vVFE@g4aK_C+rRDh6pqkeR;&6FzYowoD9ZQS8%&5|c<*UHA zHH1jUKw$|lMxYba4;(+vM>()v)6ba<#klem*1ERe8WGPvRy2(G``P<;gh2A7AaFcx zoDjWLuH8J5C@KD1`0`r4Fu>^5icR?<47EiWBW`N7mkdDPMr^&vKcey)l2Y~G`w%CT zt9;Y!%whZ->KrBhny+osx%aLF8ETn0FR;0iqjhCTJmjiTCg6 zO)6?nW0*8dtzM#fY*S+Im)QodzkU@bbG7D)m*lgyv&^&Bx32db+;Z5Xt*&0@Ci`Q^ zG;+{4L|9xGRjL^8&k(8iw_pah8`E+p<(~KLZ+#srp1Ab*Cc%Ge@0UJ$R3H&M-p-7T z(XPLa*NGOmiU(|`#&1U{b0@lOrzh{GZw#l#>*F)fK@9sTVI=Bo+C;CKmeWyDPvb~Aq~L-IQZ5}Xp9-u>>rHemBD_AG;7v(_P4H{cU{DQcg>EqG2lNXs~+l(c;l%{~Tjc2cU@lE+e zxXrO?Bfpsr1;?6GWZPwC&~AFsCzG>MLcrspgJsjQvyn=F&d_n67Wr zTe(txYnv%-Ol_o@jQ$)k7%S9j@=AGAgP1FY!CfVMt9CgKtLuThbNsVYR$6hO;K$6J zr0R6eo3=fjpVu|0Mh<_2QLU|DS8%tTFK^3C9&Jzp>y@N)eJ{FOuY|gzTBSD$;KN%N z6oiqOU3$MCc;_W!%#p+Gtd3Xt)EqXox=lMA~rmt5@M#F{`drd2Qnt`-s=h} z?w|1j!!-w78U8QVe~xS@Gt5I4Bmu>>7Fx@wwvE9SYS>u@8V2tx(daMTAz9@bHX`}3 zs;t=8Z+cbqsk*H9Nk3tQonmi!uG=uG@$ZxA@;L)+%ZJA)%4TXG8I7q zXFP)#a9#AgTwnv*Zkjx`yfTQ&%K=5y?<`sUf(jstoqg+~Oe1l`nA&G#(RuCro8rU1oIhtp z`C@s;ATJw<{OXw4wvHUrRn}|4x6dlPMs_5T#_fP28oTu7eBmuv(l8^ za^fy@fY@kbf*WPSy0gwO2ai2P1{x&^XS~w&M8?GF;tOPb?_F_&2gq|JV4uK%tPl5z zsV4%X!7@MFYNo5h=GoQ$-u}FJ@4$d1hD0xL#es#Cv9!FORu~4QCSyH`Bdkk0-*To! z?1l9A&LhP%D^DB?5nTEGQ~7ATfCDYYg|hcTagVPiFRZUtGpCI?*IC(x327 zLu-P3%@;|}V)SLOc%6ugMTkx;OHQy28}`JCU{Mlu-A)N)es%TsU=RmEf+MbTHvvP>hN>a|cSkbqEO z_9uhH)cB3}TSF;CB9bTDOkdDi;E~_Wi5TA&2u4;x#r8O%jDl3C!7U4Xyc~8D?V51nf!~dr~|1 z2P|-xXzUWpbwxbph@#8YeX1Mv{@q~juEuU^iyg|*tH=8hunhqQ#86rH8%-rDE{>c{ zoOCGGqKcnYMHyl~Q14(z9BI89T$^t>|0VQMW1jzeJbx*!GzgX^IqXX5pH$4mo_!TL zE>YfO7gFr{QizwDgE-WvD;Lv!moM}3kC(5YZ#rGHK8JmlnNMt3Tv))+Y4Yc z9avA?&|BxGl4ObYbN3l>L7u11ZdYip9Jf~7Z*%lac+4p1z^X^S$40#w#~-Lm2u&@yV4aNieafh;et+dcMD(^qb8NY zd7Nq{VMWxClB3hhx@RACnaBWWJgHwHD<-n7Fol1Z_bZkvT4L#F%9?U&`y3mjx2kE- zVGqJ06-?g&x2n8!etFg%cyt*JBD%c!L~Nq*3jYqte@hnx5IALL6DNHA{g0w5 z#=i=I&9bwZvzJc{`c$$X^ zN;l`nl8%c5o{oVof#bKJ_F?-uGZ;g&v4Ettt>#}f&I8AcH~@0{M!SmTapOlLwu9mU zK8U_9xz9`HFU51(0CdKklE!vD;g@Z^b+@rR$>3CGX1M8GmQsK=$C61`v5wZ&xoKq13jPT9FOe6! zLq9$~wv(y_gM!Q`jPje|tb|Fxg%|1Nk`;XuOWsUmeV&eIqmgSm^;E*n ze3weL8+y2^LPAm?w#E?~4nPN#T90jZBxF|{>ZAO3s)k2^w{l6uJmujH(#Ge~>3d0M!0T znM|$=JnLIF6&I(Xu^N1mlgOCoMVchlk{#)ZNjp=vT-9?syAyk7Q7Zc0C70#5J3%@c%f2sbNF>&ZWpCE@@*OcISeJeFi5pJ1o=f-mg+|r~thm=RK(} z_KPy|uGTghk+z*TQF3_*T_1Z+%*u08TzRaoM5ast6PHt%_S5Ph*kSqTEahZ7l7`c* zVI@%b_0%uXN6(Z5lHw($0kaTwPuaZ5iK{X0u*~%LPQ3^Q&8vb6R*Zg|)So`@PaGC@ z7eRt2_X#o~scDeuc2%7*(s|LtjC$wo@W`XFk3S)>svQMM_mLl`9b+>u$izm}c)=}~ zZjUeeK3ZLq3%2vZm%!QHb{5x zpcl8kv#XbGl)`GyAwKio_NN2Q9QcRA2bnN&O$hYK<|>>Xr5?CCSC#!Wx=A%1M;>u> z{7a#p6lKjlfB~f(?jVhAc(Gy}FCr!6p!;MQndUoiRUW>{;T64%Z3Xg}oiAS-73^1Y z`t@h1p4E=k^CrE^A;?M67534$W@iZlrGW38t(74C9&U`5l7fhu0Pm@%LVR34P!AIw z%O|h(o$t_9WL&>kUN%Ut1Kz40tOL>m0XS^lCZp>b_y?{^wOw63&C<$6Wl*QSMU;Wb@Md+f6N>`|mfep~~Q&v%6HxJk~VNzWxjYZ;Z^(8sPnTm~}U{l84bB z({Yf78p4zR>02OpLQyFGOA*H!|jowQ8_17u;R@*ub;m)A3TsxN(ux`LHje} zY~~M>;r8ckKS`Y~y5r9ueTkxz-I8s6)@a>}Jk1%K5m(DHqoj$o_$kT#^mS!I3+l8Z zZN90hS?v|cVcnr`=ZT-!Hn9p%!u*}^Wb)f%``-7cRIer%%D z%o6YY3}y&MG`2c;1}sH=&CXJPGU}ZqD|pA5X*Db4tzUYr?BJt@dL%w}yJwzRflnz| zbdM*jqoVc4=3Crd*k9`F1CnpW;J+O{xmB}*2YK!K(!?={_)e-^Q!*f_pCGZOuFkXLcw#z)N0Az-!3(YwHKA{V>pf6x=&aDMb%R#PjYAwCqJ@vd=Tb z`VvNNrxu#{jkxe=+6B!LfRDe53U_Fx((0zR8HVvzjSFt2oxU+>y;u+505^N5@JUu( zs_%jq4k*T0_Se4)6(x}ld$e;xhQz7@>QR!Qv~!w6**Wca_|lJVVgUx3J;**=p_9GK zjv$WD7R_BE_!C!%fg;3V^jp#>{pi|DY*5+*Zg{dhe3XgDvR=e$z3Ns)NlRO6;`E7I zyNt8KI-3@tu>cf_90^dC+t?N>C?|h_2?c=N zRb6&DayO$`y?b{|&A8~OI7&RSV9x3|3SASp-~OYxRE_5H9_X`-J>ez1z?7oLEGR=X zr7ntt@Pu(E6|SC>44MW^nR!{ZB9E@MHCFE@sX-d)D{sd`dJU27iS_dbS25~(Qf??0 zecMR4etPhH8&&a7_-))6oD{=z7Ug54k?6QtC=zhjK(V}Gr(JcMIj{6mwt@Vn?etAP zPU9Td@@~#vo9EFPx;M}r8c>>5{25V&MtTcvEaYpr+`?hvMv&d2%XmM|Re!hT;!yki z(RUxw@9xZ-2%@~hjtp6`P!$K;iy-igzE%nE7M7Tf^4!tax7w3B(FZGV7VfL82Iu8@ zlN5ThV|>u-tJT%k)Ud24PL^zNUw`gAx7@;j4w5+0#vuOu=py!G7he9_8*hr%Ge>C~W`pO4_F00?%-SK2%kQ<8;F=`A$Ly+*~ z_H|>HPoz5x{kT1TzN}CH*hz0?=ah}j8q3mQ8l1@jVO zPo?fNQ>K)7kvKgFPA(ndFu_89U@8z<)x14SRUUtc6Iz7LikgVA69wzwC;UZ^H}@o|k8XH#O)38-~+&wti@J zFFf``kxrq!9ftAco&DW5LnSCnYLckiK_py`4bMo;3%J5ePjfNbLMW28hul}|Pn%X(Iec!cw@d|hhX)ta zmI%}(xMxb^0*d$2CKrbD9kQzg- z+m^A|64kcJxhkFtix0icDrh{0AY3UiG3E@C{b;X&tr&0>9 z`f=IMtew=`6~ohlgr2T)@4j3t`IefrYV+o9=KcEB{#l}R@^WI`&Ru&;t}7)$6@((K zOio?x%p4z8ZXE1lu$dR*Gyv0F>BPH~rDp;%B7(xkUWa+JXnbl$F)#$m38&OrdjsYipudsQ#^>9$ z>(n;R>h%;fS&!d+_VMEnOUEb)bC6j|c?d7z{siIl<5>Q1RMKdmAxX76k3o&AMwQo( zPo=mZ05`sx-BU5$(2FOu_mB%SF!R6?Us(LE6*U7R4gnxouwygX?%vpG+0?KSvBK5e zQe`e)i6P9y1xZDZ1=Nz08Ur6!_$i`f=Q4?eOx+ z8!&P-e=zL3wIOBe*uQ}(Lyi8()WzSx2SxEI)f&%SXkKU`eTDNx@8NLuQRA{rF)IJ&Ov5{8WBu9mwm`n&DS}B4vbIpKSbPIZ2e}rrgLZQynYuyVF~*U7-#fo z@U{L?eLd%A^=7)5Emi*6w;!qQw;J=(D>m)$v98miz(Yfj)*fpi6=2Y?sWP(^tSQL>ilQSd zx{*mufcYU${UNu6u+pDM@Gu<|K*Q>cl8)G)pMU9NEuDPiFAru0A%CMH)AdS>yLZX= zLWGa>F5d~3O&(rry%yve5AKV{o6s+jGr@6KAnnsF!QJC#Bm-pydt=DsS@3}CW|`lu zE}0_z3HMt#VCs{gZ=|a$w<*ubt@n9zVIHZq20()mx4x0Vr+)L(5ghN|upJLB9({fxtgjKV8yXns7Tv0u_8HD_F2iIArhhcHP~B|ZZv<&k zIg0xrVqsD6sH=8{9I73kR+9K8H31)M23lol^Jc~p1~Ys8+E`;mGcMSXpuY1#*`&Se zOSg!s@OOt1cxmkho#YCFckYTxBxl`l9P*PDs@&d^`uf#|)y`5aYR8m71!}Mv8qfOe zCUIt2>Z9|4Y90)Zw5Q|cMB^F6fffxuo+u5FeEW5|nUmu_xR@&-J5n8qIp|tWs;Y`F zi)vkC0l?HMD|lVqw<0xJSsVJa!Y6pypUXSLL(M(i?2nMhfk;;zzGd&9(4ZxT&^Jt( z=`-tcQ)K~TGpuUZr@?7shkH7+KmI&mVX>i^xkfFMHsXLfb0>RW*-u6Wh~04iS5MTi z_lWr`pUNVNaim|Atb{6zq7tz`Mtv?2Q5`Bz%i0rRCw7cdqXsptnBjS%^A ziy?sAd4krHE`N@TGVo!?Gcy08NA_!Bj;xozK9 zoM4e42zW99DCn*H#U!txP5`uLKrkB^mVyWAy731N-e2JIUjQYDH3I*W><4D-{Y?mL zAjA7l{^1W0>HIHFdY#C44L1F^1Sf=ppE*SD!Tj>N?LUpR8CRro8>0Dq TEWH$9EubW){ 30k chars) get a one-line nudge toward `/digest`. After ZCode compacts the session, a `SessionStart(compact)` hook points back at the vault so details are recovered, never guessed. | dsh `monitor` + billion-context session stats & growth nudges, merged into one ledger | +| **L1 — Lossless spill & read-dedup** | `/digest` stores the original verbatim in the vault and emits a structure-aware digest: errors → 300-char excerpt, JSON → shape summary, CSV → header + row count + first/last row, text → head/tail. Duplicate unchanged file reads are detected from the ledger and called out with wasted-token estimates. | dsh `outputLadder` + `fileDiff`, made reversible à la billion-context | +| **L2 — Model-driven compression** | `/compress` writes a structured summary of the settled conversation into the vault, registers it in the index, then anchors ZCode's built-in `/compact` on it. Suggested at ~45% context pressure instead of waiting for an emergency. | billion-context `compress`/`decompress` + dsh `compactionDriver`, unified | +| **Recovery** | `/context-search` greps vault originals and summaries; originals come back whole. | billion-context `search_context` | +| **Statistics** | `/token-status` reports per-tool token estimates, largest outputs, duplicate reads, recoverable savings, vault totals, and MCP-server trimming advice. | dsh `/token-status` + `monitor` + billion-context `acp_status`, unified | + +## Commands + +- `/token-status` — token statistics and savings opportunities +- `/digest ` — spill a large output, keep only its digest +- `/compress [scope]` — compress settled history into a reversible summary +- `/context-search ` — recover details from the vault + +## Install / requirements + +- ZCode with plugins enabled. Install from a marketplace that lists this plugin. +- **Node.js ≥ 16 on PATH** for the hooks and vault CLI (zero npm dependencies, stdlib only). The skill and commands degrade gracefully without it — only automatic ledger recording and the CLI stop working; no session is ever blocked by a failed hook. + +All data lives in `/.zcode/tokencare/` (ledger + vault) and is self-gitignored. + +## Design decisions + +- **Tuned for GLM-5.3 Max and Flash Max.** Token estimation is GLM-aware (CJK ≈ 1 token/char, ASCII ≈ 4 chars/token) instead of the usual chars÷4, which undercounts Chinese-heavy sessions. A profile system shifts strategy per tier — `TOKENCARE_PROFILE=flash|balanced|max` (auto-detected from `TOKENCARE_MODEL` / `ZCODE_MODEL` / `ANTHROPIC_MODEL` or the hook payload): Flash-class nudges at 20k chars and compresses at ~35–40% pressure; balanced (default) 30k / ~45%; Max-class nudges at 40k chars, defers compression to ~55% and focuses on duplicate reads and giant outputs. `/token-status` shows the active profile and its advice. + +- **No proxy.** billion-context works as a stream-rewriting proxy between agent and model API. A marketplace plugin cannot (and should not) intercept model traffic, so TokenCare achieves the same savings with hooks + skills + commands, and deliberately cooperates with ZCode's built-in compaction instead of replacing it. +- **One status surface, one ledger.** Both source projects ship overlapping compaction-drivers and monitors; here they are a single ledger and a single `/token-status`. +- **text2img (dsh) is intentionally not ported.** Rendering text to images to save tokens needs a vision model, an API key, and per-platform text rendering, while `/digest` achieves the same token saving losslessly with no dependencies. +- **toolTrim is advisory.** A plugin cannot remove MCP tool schemas from requests, so `/token-status` surfaces MCP usage and advises disabling unused servers in Settings. + +## Credits & license + +MIT. Design inspired by and indebted to [billion-context](https://github.com/ranxianglei/billion-context) (MIT + attribution) and [dsh-token-optimizer](https://github.com/Zoria-Lind/dsh-token-optimizer) (MIT). No code was copied; both projects are worth a look in their own right. diff --git a/external_plugins/tokencare/README_CN.md b/external_plugins/tokencare/README_CN.md new file mode 100644 index 0000000..b6cb5c8 --- /dev/null +++ b/external_plugins/tokencare/README_CN.md @@ -0,0 +1,46 @@ +# TokenCare + +面向 [ZCode](https://z.ai) 的分层上下文压缩与省 token 优化插件。将 [billion-context](https://github.com/ranxianglei/billion-context)（增量、可逆的上下文压缩）与 [dsh-token-optimizer](https://github.com/Zoria-Lind/dsh-token-optimizer)（工具输出分级、重复读取消除、用量监控）的理念统一为一个插件——重叠部分合并，不做重复造轮子。 + +长会话会积累巨大的工具输出、反复读取同一文件、撑爆上下文窗口。TokenCare 用三层策略削减浪费，并通过磁盘上的"保险库（vault）"保证**一切皆可恢复**。 + +## 工作原理 + +| 层 | 作用 | 来源 | +|---|---|---| +| **L0 — 观测**（自动） | PostToolUse 钩子把每次工具结果的大小、哈希、文件路径记入会话账本；超过 3 万字符的输出会收到一行提示，引导使用 `/digest`；ZCode 压缩会话后，SessionStart(compact) 钩子提示通过保险库找回细节，绝不凭空猜测。 | dsh `monitor` + billion-context 会话统计与增长提示，合并为单一账本 | +| **L1 — 无损外存与去重** | `/digest` 将原文逐字存入保险库并生成结构感知摘要：错误 → 300 字符摘录，JSON → 结构概要，CSV → 表头 + 行数 + 首末行，文本 → 头尾截取。账本自动检测"同一文件未变更却重复读取"并按 token 估算量化浪费。 | dsh `outputLadder` + `fileDiff`，按 billion-context 思路做成可逆 | +| **L2 — 模型驱动压缩** | `/compress` 把会话中已稳定的部分写成结构化摘要存入保险库并登记索引，再以它为锚点配合 ZCode 内置 `/compact`。建议在约 45% 上下文压力时执行，而非等到 80% 以上才应急。 | billion-context `compress`/`decompress` + dsh `compactionDriver`，统一 | +| **恢复** | `/context-search` 在保险库的原文与摘要中检索；原文可完整取回。 | billion-context `search_context` | +| **统计** | `/token-status` 输出按工具的 token 估算、最大输出、重复读取、可回收空间、保险库总量，以及 MCP 服务裁剪建议。 | dsh `/token-status` + `monitor` + billion-context `acp_status`，统一 | + +## 命令 + +- `/token-status` — token 统计与节省机会 +- `/digest <文件 | "last output">` — 大输出外存，仅保留摘要 +- `/compress [范围]` — 将已稳定的会话历史压缩为可逆摘要 +- `/context-search <关键词>` — 从保险库找回细节 + +## GLM-5.3 Max / Flash Max 调优 + +token 估算针对 GLM 系分词器做了中英双语感知（中日韩 ≈ 每字符 1 token，ASCII ≈ 4 字符/token），中文密集会话不再被低估。策略按模型档位自动切换：`TOKENCARE_PROFILE=flash|balanced|max`（也可通过 `TOKENCARE_MODEL` / `ZCODE_MODEL` / `ANTHROPIC_MODEL` 或钩子载荷自动识别）：Flash 档 2 万字符即提示、约 35–40% 压力即压缩；balanced（默认）3 万 / 约 45%；Max 档 4 万才提示、约 55% 再压缩，优先消除重复读取与超大输出。`/token-status` 会显示当前档位与建议。 + +## 安装与依赖 + +- 启用插件功能的 ZCode。从收录本插件的插件市场安装。 +- 钩子与保险库 CLI 需要 **PATH 中有 Node.js ≥ 16**（零 npm 依赖，仅标准库）。缺少 Node 时优雅降级：仅自动记账与 CLI 停止工作，任何会话都不会因此中断。 + +数据保存在 `<项目>/.zcode/tokencare/`（账本 + 保险库），目录自动自 gitignore。 + +权限与副作用说明：插件只读写项目内的 `.zcode/tokencare/` 目录；钩子在每次工具调用后本地执行一次 Node 脚本（无网络请求、无外部依赖）；不会删除或改写保险库中的任何原文。 + +## 设计决策 + +- **不做代理。** billion-context 以流改写代理方式工作；市场插件无法也不应拦截模型流量，TokenCare 改用钩子 + 技能 + 命令达成同样节省，并与 ZCode 内置压缩协作而非取代。 +- **一个状态出口、一份账本。** 两个上游项目各有重叠的压缩驱动与监控，这里合并为单一账本与单一 `/token-status`。 +- **未移植 text2img（dsh）。** 文本转图片省 token 需要视觉模型、API 密钥与各平台文字渲染，而 `/digest` 以零依赖、无损方式达到同样节省。 +- **toolTrim 仅作建议。** 插件无法从请求中移除 MCP 工具 schema，`/token-status` 呈现 MCP 使用情况并建议在设置中停用闲置服务。 + +## 致谢与许可 + +MIT。设计受 [billion-context](https://github.com/ranxianglei/billion-context)（MIT + 署名条款）与 [dsh-token-optimizer](https://github.com/Zoria-Lind/dsh-token-optimizer)（MIT）启发并致谢；未复制任何代码，两个项目本身值得一读。 diff --git a/external_plugins/tokencare/commands/compress.md b/external_plugins/tokencare/commands/compress.md new file mode 100644 index 0000000..911521d --- /dev/null +++ b/external_plugins/tokencare/commands/compress.md @@ -0,0 +1,11 @@ +--- +description: Compress the settled part of this conversation into a reversible vault summary. +argument-hint: "[scope hint, e.g. 'everything before the last hour']" +skills: tokencare +--- + +Use the `tokencare` skill (L2) for this request. + +Scope: $ARGUMENTS (default: the older, settled part of the conversation — recent turns stay intact). + +Follow the skill's L2 protocol exactly: write a structured summary (`summary-.md`) into `.zcode/tokencare/vault/` covering goals/state, decisions and reasons, files touched, key commands and results, open threads, and spill-id pointers; register it in `vault/index.json`; then tell the user the summary is the compaction anchor and suggest running ZCode's built-in `/compact` now. Emphasize that nothing is lost — `/context-search` recovers any detail from the vault afterwards. diff --git a/external_plugins/tokencare/commands/context-search.md b/external_plugins/tokencare/commands/context-search.md new file mode 100644 index 0000000..956f25c --- /dev/null +++ b/external_plugins/tokencare/commands/context-search.md @@ -0,0 +1,17 @@ +--- +description: Search the context vault for details lost to compaction or spilled from context. +argument-hint: "" +skills: tokencare +--- + +Use the `tokencare` skill (recovery) for this request. + +Query: $ARGUMENTS + +Run: + +``` +node "/../../scripts/vault.mjs" search "$ARGUMENTS" +``` + +Report the matches with their vault ids. If a match looks relevant but truncated, fetch the full original with `vault.mjs get ` and answer from it. If there are no matches, say so plainly and fall back to normal tools (re-reading the file, re-running the command) — do not guess at vault contents. diff --git a/external_plugins/tokencare/commands/digest.md b/external_plugins/tokencare/commands/digest.md new file mode 100644 index 0000000..3f34a02 --- /dev/null +++ b/external_plugins/tokencare/commands/digest.md @@ -0,0 +1,17 @@ +--- +description: Spill a large tool output or file to the vault and replace it with a compact digest. +argument-hint: "" +skills: tokencare +--- + +Use the `tokencare` skill (L1) for this request. + +Target: $ARGUMENTS + +If the argument is a file path, spill it directly: + +``` +node "/../../scripts/vault.mjs" spill "" +``` + +If it is "last output" or similar, the most recent large tool output is still in the conversation: save it to a temporary file (or pipe via `--stdin`), then spill that. Use the printed digest block in place of the original from now on, and mention the spill id so the user can recover the original later with `/context-search`. diff --git a/external_plugins/tokencare/commands/token-status.md b/external_plugins/tokencare/commands/token-status.md new file mode 100644 index 0000000..c41bd94 --- /dev/null +++ b/external_plugins/tokencare/commands/token-status.md @@ -0,0 +1,13 @@ +--- +description: Show token usage statistics and savings opportunities for this project's sessions. +argument-hint: "[--json]" +skills: tokencare +--- + +Use the `tokencare` skill for this request. Run the vault statistics tool: + +``` +node "/../../scripts/vault.mjs" stats $ARGUMENTS +``` + +Present the report to the user in a readable form (per-tool estimated tokens, largest outputs, duplicate unchanged re-reads, oversized outputs, vault totals). Then act on what it flags: offer `/digest` for oversized outputs, point out duplicate reads with the exact wasted-token estimates, and remind about unused MCP servers if MCP tools appear. Token figures are chars÷4 estimates — label them as such. diff --git a/external_plugins/tokencare/hooks/hooks.json b/external_plugins/tokencare/hooks/hooks.json new file mode 100644 index 0000000..c593f8b --- /dev/null +++ b/external_plugins/tokencare/hooks/hooks.json @@ -0,0 +1,29 @@ +{ + "hooks": { + "PostToolUse": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/vault.mjs\" record posttooluse", + "timeoutMs": 10000, + "statusMessage": "TokenCare: recording tool output stats" + } + ] + } + ], + "SessionStart": [ + { + "matcher": "compact", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/vault.mjs\" record sessionstart", + "timeoutMs": 10000, + "statusMessage": "TokenCare: checking context vault" + } + ] + } + ] + } +} diff --git a/external_plugins/tokencare/scripts/vault.mjs b/external_plugins/tokencare/scripts/vault.mjs new file mode 100644 index 0000000..c2c606b --- /dev/null +++ b/external_plugins/tokencare/scripts/vault.mjs @@ -0,0 +1,605 @@ +#!/usr/bin/env node +// TokenCare vault: ledger + reversible spill store for tool outputs. +// Subcommands: +// record posttooluse read a PostToolUse hook payload from stdin, append a ledger line, +// emit a nudge when the output is very large +// record sessionstart read a SessionStart(compact) payload from stdin, point at the vault +// spill [FILE|--stdin] [--label L] [--chars N] [--kind auto|error|json|csv|text] +// store the original verbatim in the vault, print a compact digest +// stats [--json] aggregate token statistics from the ledger +// search QUERY [-n N] case-insensitive search over vault originals and labels +// index list vault entries +// get ID print the full original for a vault entry +// Data lives in /.zcode/tokencare/ (ledger/ and vault/). + +import fs from "node:fs"; +import path from "node:path"; +import crypto from "node:crypto"; + +// Token estimation: GLM-family tokenizers encode CJK at roughly 1 token per +// character while ASCII averages ~4 chars/token, so a plain chars/4 estimate +// badly undercounts Chinese-heavy sessions. All figures remain estimates. +const CJK_RE = /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]/g; + +function estimateTokensFromText(text) { + if (!text) return 0; + // count via replace-length, not match(): a match() array on a large CJK + // document would allocate millions of string entries on this hot path + const asciiOnly = text.replace(/[^\x00-\x7F]/g, ""); + const withoutCjk = text.replace(CJK_RE, ""); + const nonAscii = text.length - asciiOnly.length; // CJK + other non-ASCII + const other = withoutCjk.length - asciiOnly.length; // non-ASCII, non-CJK + const cjk = nonAscii - other; + return cjk + Math.ceil(other / 2) + Math.ceil(asciiOnly.length / 4); +} + +// Profiles: a Flash-class model (compact, low-latency window) rewards early +// hygiene; a Max-class model tolerates far more live context. Select with +// TOKENCARE_PROFILE=flash|balanced|max, or a model id in TOKENCARE_MODEL / +// ZCODE_MODEL / ANTHROPIC_MODEL (also read from the hook payload when present). +const PROFILES = { + flash: { + nudgeChars: 20000, + label: "flash (compact window)", + advice: "spill early (nudge at 20k chars), compress at ~35-40% pressure", + }, + balanced: { + nudgeChars: 30000, + label: "balanced", + advice: "nudge at 30k chars, compress at ~45% pressure", + }, + max: { + nudgeChars: 40000, + label: "max (large window)", + advice: "large window tolerates live context (nudge at 40k chars); defer compression to ~55% and spend effort on duplicate reads and giant outputs", + }, +}; + +function activeProfile(hint) { + const explicit = (process.env.TOKENCARE_PROFILE || "").toLowerCase(); + if (PROFILES[explicit]) return explicit; + const m = + `${hint || ""} ${process.env.TOKENCARE_MODEL || ""} ${process.env.ZCODE_MODEL || ""} ${process.env.ANTHROPIC_MODEL || ""}` + .toLowerCase(); + if (m.includes("flash")) return "flash"; + if (m.includes("max")) return "max"; + return "balanced"; +} + +function nudgeThreshold(profile) { + if (Number(process.env.TOKENCARE_NUDGE_CHARS) > 0) return Number(process.env.TOKENCARE_NUDGE_CHARS); + return PROFILES[profile || activeProfile()].nudgeChars; +} + +function estTokens(chars) { + return Math.ceil(chars / 4); +} + +const SPILL_DEFAULT_CHARS = 1000; // below this, spilling saves nothing meaningful + +function projectDir() { + return ( + process.env.ZCODE_PROJECT_DIR || + process.env.CLAUDE_PROJECT_DIR || + process.cwd() + ); +} + +function dataDir() { + return path.join(projectDir(), ".zcode", "tokencare"); +} + +function ensureDataDirs() { + const root = dataDir(); + // fast path for the hook (runs on every tool call): one stat instead of + // two mkdirs plus a gitignore check once the tree exists + if (fs.existsSync(path.join(root, "ledger")) && fs.existsSync(path.join(root, "vault"))) return root; + fs.mkdirSync(path.join(root, "ledger"), { recursive: true }); + fs.mkdirSync(path.join(root, "vault"), { recursive: true }); + const gi = path.join(root, ".gitignore"); + if (!fs.existsSync(gi)) fs.writeFileSync(gi, "*\n!.gitignore\n"); + return root; +} + +function readStdin() { + return new Promise((resolve) => { + let data = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (c) => (data += c)); + process.stdin.on("end", () => resolve(data)); + process.stdin.on("error", () => resolve(data)); + setTimeout(() => resolve(data), 8000).unref(); + }); +} + +function responseText(r) { + if (r == null) return ""; + if (typeof r === "string") return r; + if (Array.isArray(r)) return r.map(responseText).join("\n"); + if (typeof r === "object") { + if (typeof r.stdout === "string" || typeof r.stderr === "string") { + return [r.stdout || "", r.stderr || ""].filter(Boolean).join("\n"); + } + for (const k of ["content", "output", "text", "result"]) { + if (r[k] != null) return responseText(r[k]); + } + try { + return JSON.stringify(r); + } catch { + return String(r); + } + } + return String(r); +} + +function sha1(s) { + if (s.length > 10 * 1024 * 1024) return "(too-large)"; + try { + return crypto.createHash("sha1").update(s, "utf8").digest("hex").slice(0, 16); + } catch { + return "(unhashable)"; + } +} + +function ledgerFile(sessionId) { + const safe = String(sessionId || "default").replace(/[^\w.-]/g, "_"); + return path.join(dataDir(), "ledger", `${safe}.jsonl`); +} + +function appendLedger(entry) { + ensureDataDirs(); + const file = ledgerFile(entry.session); + fs.appendFileSync(file, JSON.stringify(entry) + "\n", "utf8"); +} + +function readLedger() { + const dir = path.join(dataDir(), "ledger"); + if (!fs.existsSync(dir)) return []; + const lines = []; + for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".jsonl"))) { + for (const line of fs.readFileSync(path.join(dir, f), "utf8").split("\n")) { + const t = line.trim(); + if (!t) continue; + try { + lines.push(JSON.parse(t)); + } catch { + /* skip malformed line */ + } + } + } + return lines; +} + +// ---- vault ---- + +function vaultIndexFile() { + return path.join(dataDir(), "vault", "index.json"); +} + +function readVaultIndex() { + try { + return JSON.parse(fs.readFileSync(vaultIndexFile(), "utf8")); + } catch { + return []; + } +} + +function writeVaultIndex(entries) { + ensureDataDirs(); + fs.writeFileSync(vaultIndexFile(), JSON.stringify(entries, null, 2), "utf8"); +} + +function looksLikeError(text) { + const head = text.slice(0, 2000); + return /\b(error|failed|exception|traceback|fatal|panic)\b/i.test(head); +} + +function digestOf(text, kind, vaultRel) { + const lines = text.split("\n"); + if (kind === "error") { + const first = text.slice(0, 300).trim(); + const last = text.slice(-300).trim(); + return `[error output — first 300 chars]\n${first}\n[…]\n[last 300 chars]\n${last}\n[full original: ${vaultRel}]`; + } + if (kind === "json") { + let shape = "(unparseable)"; + try { + const v = JSON.parse(text); + if (Array.isArray(v)) { + shape = + `array of ${v.length} item(s)` + + (v.length ? `; first item: ${JSON.stringify(v[0]).slice(0, 400)}` : ""); + } else if (v && typeof v === "object") { + const keys = Object.keys(v) + .slice(0, 30) + .map((k) => `${k}: ${Array.isArray(v[k]) ? `array(${v[k].length})` : typeof v[k]}`) + .join(", "); + shape = `object keys — ${keys}`; + } else { + shape = `${typeof v}: ${String(v).slice(0, 200)}`; + } + } catch { + return digestOf(text, "text", vaultRel); + } + return `[JSON — ${shape}]\n[full original: ${vaultRel}]`; + } + if (kind === "csv") { + const header = lines[0] || ""; + const first = lines[1] || ""; + const last = lines[lines.length - 1] || ""; + return `[CSV — ${lines.length - 1} data row(s), ${header.split(",").length} column(s)]\nheader: ${header}\nfirst: ${first}\nlast: ${last}\n[full original: ${vaultRel}]`; + } + // text: head/tail, clamped so they never overlap or drop lines on short inputs + const head = lines.slice(0, 15); + const tail = lines.length > 25 ? lines.slice(-10) : []; + const omitted = lines.length - head.length - tail.length; + const parts = [head.join("\n")]; + if (omitted > 0) { + parts.push(`[… ${omitted} line(s) omitted — full original: ${vaultRel} …]`); + } + parts.push(tail.join("\n")); + return parts.filter(Boolean).join("\n"); +} + +function detectKind(text, requested) { + if (requested && requested !== "auto") return requested; + if (looksLikeError(text)) return "error"; + const t = text.trim(); + if ((t.startsWith("{") && t.endsWith("}")) || (t.startsWith("[") && t.endsWith("]"))) { + try { + JSON.parse(t); + return "json"; + } catch { + /* fall through */ + } + } + const lines = t.split("\n").filter(Boolean); + if ( + lines.length >= 3 && + (lines[0].match(/,/g) || []).length >= 2 && + lines.slice(0, 10).every((l) => (l.match(/,/g) || []).length === (lines[0].match(/,/g) || []).length) + ) { + return "csv"; + } + return "text"; +} + +function parseArgs(argv, flagDefs) { + // flagDefs: { "--flag": "value" | true } — returns { flags: {...}, positionals: [...] } + const flags = {}; + const positionals = []; + for (let i = 0; i < argv.length; i++) { + const def = flagDefs[argv[i]]; + if (def === true) flags[argv[i]] = true; + else if (def === "value") flags[argv[i]] = argv[++i]; + else positionals.push(argv[i]); + } + return { flags, positionals }; +} + +async function cmdSpill(args) { + const { flags, positionals } = parseArgs(args, { + "--stdin": true, + "--chars": "value", + "--kind": "value", + "--label": "value", + }); + const useStdin = "--stdin" in flags; + const threshold = Number(flags["--chars"]) > 0 ? Number(flags["--chars"]) : SPILL_DEFAULT_CHARS; + const requestedKind = flags["--kind"] || "auto"; + const label = flags["--label"] || ""; + const file = positionals[0]; + if (!useStdin && !file) { + console.error("usage: vault.mjs spill [FILE|--stdin] [--label L] [--chars N] [--kind auto|error|json|csv|text]"); + process.exit(1); + } + + const text = useStdin ? await readStdin() : fs.readFileSync(file, "utf8"); + const kind = detectKind(text, requestedKind); + const now = new Date(); + const digits = now.toISOString().replace(/\D/g, "").slice(0, 14); // yyyymmddhhmmss + const id = `spill-${digits.slice(0, 8)}-${digits.slice(8)}-${crypto.randomBytes(2).toString("hex")}`; + ensureDataDirs(); + const vaultFile = path.join(dataDir(), "vault", `${id}.txt`); + fs.writeFileSync(vaultFile, text, "utf8"); + const vaultRel = path.join(".zcode", "tokencare", "vault", `${id}.txt`).replace(/\\/g, "/"); + + const entries = readVaultIndex(); + entries.push({ + id, + label: label || (file ? path.basename(file) : "stdin"), + source: file || "stdin", + kind, + chars: text.length, + est_tokens: estimateTokensFromText(text), + created: now.toISOString(), + }); + writeVaultIndex(entries); + appendLedger({ + ts: now.toISOString(), + session: process.env.CLAUDE_SESSION_ID || "default", + tool: "(vault)", + event: "spill", + chars: text.length, + est_tokens: estimateTokensFromText(text), + vault_id: id, + }); + + const digest = digestOf(text, kind, vaultRel); + const saved = Math.max(0, text.length - digest.length); + console.log(`[digest id=${id} label=${label || (file ? path.basename(file) : "stdin")} kind=${kind} chars=${text.length}]`); + console.log(digest); + if (text.length < threshold) { + console.log(`(note: only ${text.length} chars — below the ${threshold}-char spilling threshold; no meaningful saving)`); + } else { + console.log(`(saved ~${estTokens(saved)} tokens vs. keeping the original in context; recover with: /context-search or vault get ${id})`); + } +} + +// ---- record (hook mode) ---- + +async function cmdRecordPostToolUse() { + const raw = await readStdin(); + let payload = {}; + try { + payload = JSON.parse(raw); + } catch { + process.exit(0); // never break the session over a malformed payload + } + const session = payload.session_id || process.env.CLAUDE_SESSION_ID || "default"; + const tool = payload.tool_name || "(unknown)"; + const text = responseText(payload.tool_response ?? payload.tool_result); + const input = payload.tool_input || {}; + const filePath = + input.file_path || input.path || input.notebook_path || input.pathname || ""; + const profile = activeProfile(payload.model || payload.modelInfo?.displayName); + const est = estimateTokensFromText(text); + + appendLedger({ + ts: new Date().toISOString(), + session, + tool, + event: "posttooluse", + chars: text.length, + est_tokens: est, + file_path: typeof filePath === "string" ? filePath.replace(/\\/g, "/") : "", + hash: sha1(text), + profile, + }); + + if (text.length > nudgeThreshold(profile)) { + // exitCode (not process.exit) so the piped stdout write always flushes + process.stdout.write( + JSON.stringify({ + hookSpecificOutput: { + hookEventName: "PostToolUse", + additionalContext: `[tokencare] ${tool} returned ${text.length.toLocaleString()} chars (~${est.toLocaleString()} tokens). Consider /digest to spill it to the vault and keep only a summary in context.`, + }, + }) + ); + } + process.exitCode = 0; +} + +async function cmdRecordSessionStart() { + const raw = await readStdin(); + try { + JSON.parse(raw); + } catch { + process.exit(0); + } + const entries = readVaultIndex(); + if (!entries.length) { + process.exitCode = 0; + return; + } + const totalChars = entries.reduce((a, e) => a + (e.chars || 0), 0); + // exitCode (not process.exit) so the piped stdout write always flushes + process.stdout.write( + JSON.stringify({ + additionalContext: `[tokencare] Context was just compacted. ${entries.length} spilled original(s) (~${estTokens(totalChars).toLocaleString()} tokens worth) are preserved in .zcode/tokencare/vault/. Use /context-search to recover exact details instead of re-reading files or guessing.`, + }) + ); + process.exitCode = 0; +} + +// ---- stats ---- + +function fmt(n) { + return Number(n).toLocaleString("en-US"); +} + +function cmdStats(asJson) { + const ledger = readLedger(); + const entries = readVaultIndex(); + const perTool = new Map(); + const outputs = []; + const readGroups = new Map(); // path+hash -> count, chars, tokens + let totalChars = 0; + let totalTokens = 0; + + for (const e of ledger) { + if (e.event !== "posttooluse") continue; + totalChars += e.chars || 0; + totalTokens += e.est_tokens || estTokens(e.chars || 0); + const t = perTool.get(e.tool) || { calls: 0, chars: 0, tokens: 0, max_chars: 0 }; + t.calls += 1; + t.chars += e.chars || 0; + t.tokens += e.est_tokens || estTokens(e.chars || 0); + t.max_chars = Math.max(t.max_chars, e.chars || 0); + perTool.set(e.tool, t); + outputs.push(e); + if (e.tool === "Read" && e.file_path && e.hash) { + const k = `${e.file_path}|${e.hash}`; + const g = readGroups.get(k) || { path: e.file_path, count: 0, chars: 0, tokens: 0 }; + g.count += 1; + g.chars += e.chars || 0; + g.tokens += e.est_tokens || estTokens(e.chars || 0); + readGroups.set(k, g); + } + } + + const dups = [...readGroups.values()].filter((g) => g.count > 1); + const oversized = outputs.filter((e) => e.chars > SPILL_DEFAULT_CHARS); + const spillChars = entries.reduce((a, e) => a + (e.chars || 0), 0); + + const sessions = new Set(ledger.map((e) => e.session)); + const profile = activeProfile(); + const result = { + project: projectDir(), + profile, + profile_advice: PROFILES[profile].advice, + sessions: sessions.size, + tool_calls: outputs.length, + total_output_chars: totalChars, + est_output_tokens: totalTokens, + per_tool: Object.fromEntries( + [...perTool.entries()].sort((a, b) => b[1].chars - a[1].chars).map(([k, v]) => [k, { calls: v.calls, chars: v.chars, est_tokens: v.tokens, max_chars: v.max_chars }]) + ), + largest_outputs: outputs + .slice() + .sort((a, b) => b.chars - a.chars) + .slice(0, 5) + .map((e) => ({ tool: e.tool, file_path: e.file_path || "", chars: e.chars, est_tokens: e.est_tokens || estTokens(e.chars) })), + duplicate_reads: dups + .sort((a, b) => b.chars * b.count - a.chars * a.count) + .slice(0, 10) + .map((g) => ({ path: g.path, unchanged_rereads: g.count - 1, chars_each: g.chars, wasted_est_tokens: Math.ceil((g.tokens / g.count) * (g.count - 1)) })), + duplicate_waste_est_tokens: dups.reduce((a, g) => a + Math.ceil((g.tokens / g.count) * (g.count - 1)), 0), + oversized_outputs: oversized.length, + spill_candidates_est_tokens: oversized.reduce((a, e) => a + Math.max(0, (e.est_tokens || estTokens(e.chars)) - 250), 0), + vault_entries: entries.length, + vault_chars: spillChars, + vault_est_tokens: entries.reduce((a, e) => a + (e.est_tokens || estTokens(e.chars || 0)), 0), + }; + + if (asJson) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + console.log(`TokenCare — token statistics for ${result.project}`); + console.log(`Sessions tracked: ${result.sessions} | tool calls: ${fmt(result.tool_calls)} | est. tool-output tokens: ~${fmt(result.est_output_tokens)}`); + console.log(`Profile: ${PROFILES[profile].label} — ${PROFILES[profile].advice}`); + console.log(`\nPer tool (est. tokens of output):`); + for (const [tool, t] of Object.entries(result.per_tool)) { + console.log(` ${tool.padEnd(24)} ${String(t.calls).padStart(4)} calls ~${fmt(t.est_tokens).padStart(9)} tok (largest ~${fmt(estTokens(t.max_chars))} tok)`); + } + console.log(`\nLargest outputs:`); + result.largest_outputs.forEach((o, i) => { + console.log(` ${i + 1}. ${o.tool} ${o.file_path || "(no file)"} — ${fmt(o.chars)} chars (~${fmt(o.est_tokens)} tok)`); + }); + if (result.duplicate_reads.length) { + console.log(`\nDuplicate reads (same file, unchanged content):`); + result.duplicate_reads.forEach((d) => { + console.log(` ${d.path} re-read unchanged x${d.unchanged_rereads} — ~${fmt(d.wasted_est_tokens)} wasted tokens`); + }); + console.log(` → Have the model reference earlier reads instead of re-reading, or use /digest markers.`); + } else { + console.log(`\nDuplicate reads: none detected.`); + } + console.log(`\nOversized outputs (>${SPILL_DEFAULT_CHARS} chars): ${result.oversized_outputs} — ~${fmt(result.spill_candidates_est_tokens)} tokens recoverable via /digest`); + console.log(`Vault: ${result.vault_entries} original(s), ~${fmt(result.vault_est_tokens)} tokens preserved on disk, recoverable via /context-search`); + const mcpCalls = Object.keys(result.per_tool).filter((k) => k.startsWith("mcp__")); + if (mcpCalls.length) { + console.log(`\nMCP tools used: ${mcpCalls.join(", ")}`); + console.log(` → Every enabled MCP server re-sends its tool schemas each request; disable unused servers in Settings.`); + } +} + +// ---- search / index / get ---- + +function cmdSearch(args) { + const query = args.find((a) => !a.startsWith("-")); + const nIdx = args.indexOf("-n"); + const limit = nIdx !== -1 ? Number(args[nIdx + 1]) || 30 : 30; + if (!query) { + console.error("usage: vault.mjs search QUERY [-n LIMIT]"); + process.exit(1); + } + const q = query.toLowerCase(); + let matches = 0; + // labels first: tiny, high-signal, and immune to being crowded out of the + // limit by long content matches + for (const e of readVaultIndex()) { + if (matches >= limit) break; + if (String(e.label || "").toLowerCase().includes(q)) { + matches += 1; + console.log(`index: ${e.id} label=${e.label} kind=${e.kind} chars=${e.chars}`); + } + } + const vdir = path.join(dataDir(), "vault"); + if (fs.existsSync(vdir) && matches < limit) { + for (const f of fs.readdirSync(vdir).filter((f) => f.endsWith(".txt"))) { + const lines = fs.readFileSync(path.join(vdir, f), "utf8").split("\n"); + lines.forEach((line, i) => { + if (matches >= limit) return; + if (line.toLowerCase().includes(q)) { + matches += 1; + console.log(`${f}:${i + 1}: ${line.slice(0, 200)}`); + } + }); + if (matches >= limit) break; + } + } + if (!matches) console.log(`No matches for "${query}" in the vault.`); + else console.log(`\n(${matches} match(es); open with: vault.mjs get )`); +} + +function cmdIndex() { + const entries = readVaultIndex(); + if (!entries.length) { + console.log("Vault is empty. Use /digest to spill a large tool output into it."); + return; + } + for (const e of entries) { + console.log(`${e.id} ${String(e.chars).padStart(8)} chars ${e.kind.padEnd(5)} ${e.created} ${e.label}`); + } +} + +function cmdGet(id) { + if (!id || !/^[\w.-]+$/.test(id)) { + console.error("usage: vault.mjs get "); + process.exit(1); + } + const f = path.join(dataDir(), "vault", `${id}.txt`); + if (!fs.existsSync(f)) { + console.error(`No vault entry "${id}". Use: vault.mjs index`); + process.exit(1); + } + process.stdout.write(fs.readFileSync(f, "utf8")); +} + +// ---- main ---- + +const [cmd, sub, ...rest] = process.argv.slice(2); +const tail = process.argv.slice(3); // everything after " " — flags and positionals +try { + if (cmd === "record" && sub === "posttooluse") await cmdRecordPostToolUse(); + else if (cmd === "record" && sub === "sessionstart") await cmdRecordSessionStart(); + else if (cmd === "spill") await cmdSpill(tail); + else if (cmd === "stats") cmdStats(tail.includes("--json")); + else if (cmd === "search") cmdSearch(tail); + else if (cmd === "index") cmdIndex(); + else if (cmd === "get") cmdGet(sub); + else { + console.error( + [ + "TokenCare vault", + "usage:", + " vault.mjs record posttooluse | record sessionstart (hook mode, reads stdin)", + " vault.mjs spill [FILE|--stdin] [--label L] [--chars N] [--kind auto|error|json|csv|text]", + " vault.mjs stats [--json]", + " vault.mjs search QUERY [-n LIMIT]", + " vault.mjs index", + " vault.mjs get ", + ].join("\n") + ); + process.exit(cmd ? 1 : 0); + } +} catch (err) { + // Hook mode must never surface errors into the session; CLI mode may. + if (cmd === "record") process.exit(0); + console.error(`tokencare: ${err && err.message ? err.message : err}`); + process.exit(1); +} diff --git a/external_plugins/tokencare/skills/tokencare/SKILL.md b/external_plugins/tokencare/skills/tokencare/SKILL.md new file mode 100644 index 0000000..a8dc27b --- /dev/null +++ b/external_plugins/tokencare/skills/tokencare/SKILL.md @@ -0,0 +1,61 @@ +--- +name: tokencare +description: Layered context compression and token saving for long ZCode sessions. Use when the user wants to reduce token usage or context size, compress or summarize conversation history, spill or digest large tool outputs, stop re-reading unchanged files, check token statistics (/token-status), or recover details lost after context compaction (/context-search). Maintains a reversible vault of original outputs under .zcode/tokencare/. +--- + +# TokenCare — layered context compression + +Long sessions die from token bloat: huge tool outputs, repeated reads of unchanged files, and context that only grows. TokenCare applies three layers, cheapest first. **Nothing is ever lost** — every compression is reversible through the vault on disk. + +The vault CLI is `node "/../../scripts/vault.mjs"` (below: `vault.mjs`). All data lives in `/.zcode/tokencare/` (ledger + vault), which is self-gitignored. + +## L0 — Observe (automatic, hooks) + +A PostToolUse hook records every tool result's size, content hash, and file path to a per-session ledger. You do nothing for L0, except: + +- When a nudge appears (`[tokencare] Bash returned 48,231 chars …`), act on it: offer `/digest` on that output instead of carrying it. +- After ZCode compacts the conversation, a reminder points at the vault — recover details with `/context-search`, never by re-reading files blindly. + +## L1 — Lossless spill and read-dedup (`/digest`) + +For any tool output or file larger than ~1,000 chars that will not be needed verbatim in future turns: + +1. Save the original to the vault and print a structure-aware digest: + `node vault.mjs spill ` (or pipe content via `--stdin`, optional `--label`, `--kind error|json|csv|text`). + Errors collapse to a 300-char excerpt, JSON to a shape summary (keys, array counts, first item), CSV to header + row count + first/last row, text to head/tail. +2. Use the digest block in your reasoning and replies. The original stays on disk, recoverable via `/context-search` or `vault.mjs get `. + +Read-dedup (fileDiff): before reading a file again, check whether an earlier read of the same path is already in the conversation. If the file is unchanged (same hash in `/token-status` duplicate report), reference the earlier read instead of re-reading; if only a small part is needed, grep the vault. Only re-read when the ledger shows the file changed or the earlier content is no longer in context. + +## L2 — Model-driven compression (`/compress`) + +Trigger: the user asks for it, context feels heavy (many large tool outputs accumulated), or `/token-status` shows large totals. Rough guidance: compress at ~35–40% of the usable context on a Flash-class model, ~45% balanced, ~55% on a Max-class model (see Model profiles) — never wait for an emergency at 80%+. + +1. Write a structured summary of the **older, settled** part of the conversation to the vault as `vault/summary-.md` (create the file with the Write tool), covering: goals and current state, decisions and their reasons, files touched and their outcome, key commands and results, open threads and next steps, and pointers to relevant spill ids. +2. Register it: append an entry to `vault/index.json` (`{"id": "summary-", "kind": "summary", "label": "", "created": ""}`). +3. Tell the user the summary is the anchor and suggest ZCode's built-in compaction (`/compact`) now — TokenCare deliberately does not fight the host's compaction; it makes it lossless by guaranteeing recovery through the vault. + +## Recovery (`/context-search`) + +`node vault.mjs search ""` greps vault originals and summaries; `vault.mjs get ` prints a full original; `vault.mjs index` lists entries. After any compaction, prefer searching the vault over re-running tools — it is free. + +## Statistics (`/token-status`) + +`node vault.mjs stats` reports per-tool output tokens, largest outputs, duplicate unchanged re-reads, oversized-output count, recoverable savings, and vault totals. Present the human report, act on what it flags. If MCP tools appear, remind the user that every enabled MCP server re-sends its tool schemas on each request — disable unused servers in Settings (TokenCare can only advise; it cannot trim schemas). + +## Model profiles (GLM-5.3 Max / Flash Max) + +Token pressure behaves differently per tier: a Flash-class window (e.g. GLM-5.3 Flash Max) rewards early hygiene, while a Max-class window (e.g. GLM-5.3 Max) tolerates far more live context. Select with `TOKENCARE_PROFILE=flash|balanced|max` or a model id in `TOKENCARE_MODEL` / `ZCODE_MODEL` / `ANTHROPIC_MODEL` (the hook also reads the model from the payload when provided; `/token-status` prints the active profile and its advice). + +- **flash** — nudge at 20k chars, compress at ~35–40% pressure. +- **balanced** (default) — nudge at 30k chars, compress at ~45%. +- **max** — nudge at 40k chars, defer compression to ~55%; prioritize duplicate-read elimination and digesting giant outputs over compressing early. + +Token figures use GLM-aware estimation (CJK ≈ 1 token/char, ASCII ≈ 4 chars/token) — still estimates; label them as such. + +## Rules + +- Never delete or rewrite vault originals; only add. +- Do not spill outputs under ~1,000 chars — the digest costs more than it saves. +- Ledger and hooks must stay silent: if a vault command fails, say so and continue; never block work. +- Token estimates are chars÷4 approximations — label them as estimates. diff --git a/marketplace.json b/marketplace.json index 884bfc8..de6ef73 100644 --- a/marketplace.json +++ b/marketplace.json @@ -1409,6 +1409,33 @@ "mcp", "verification" ] + }, + { + "name": "tokencare", + "displayName": "TokenCare", + "displayName_i18n": { + "en": "TokenCare", + "zh-CN": "TokenCare 上下文压缩" + }, + "version": "0.2.2", + "source": "./external_plugins/tokencare", + "description": "Layered context compression and token optimization: a reversible vault for large tool outputs, duplicate-read detection, model-driven conversation compression, and per-session token statistics.", + "description_i18n": { + "en": "Layered context compression and token optimization: a reversible vault for large tool outputs, duplicate-read detection, model-driven conversation compression, and per-session token statistics.", + "zh-CN": "分层上下文压缩与省 token 优化：可逆的工具输出保险库、重复读取检测、模型驱动的对话压缩与会话 token 统计。" + }, + "category": "developer-tools", + "author": { + "name": "TokenCare contributors" + }, + "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/tokencare/icon.png", + "keywords": [ + "context", + "compression", + "token-optimization", + "hooks", + "glm" + ] } ] } \ No newline at end of file