From 0a89407e4d8ad644fde5d403a006f77ad2c17026 Mon Sep 17 00:00:00 2001
From: erew123 <35898566+erew123@users.noreply.github.com>
Date: Sat, 16 Mar 2024 11:15:01 +0000
Subject: [PATCH] Custom tokenizer for Japanese

Replaces the standard TTS tokenizer within Finetuning.
---
 .../__pycache__/tokenizer.cpython-311.pyc     | Bin 0 -> 39933 bytes
 templates/tokenizer/tokenizer.py              | 869 ++++++++++++++++++
 2 files changed, 869 insertions(+)
 create mode 100644 templates/tokenizer/__pycache__/tokenizer.cpython-311.pyc
 create mode 100644 templates/tokenizer/tokenizer.py

diff --git a/templates/tokenizer/__pycache__/tokenizer.cpython-311.pyc b/templates/tokenizer/__pycache__/tokenizer.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2110cb49e414c7ab1183fb898b9881b8d7d9510f
GIT binary patch
literal 39933
zcmdsg3v^UPx^6#{PIu_elV^Yp36DT%AiNP!Av_gS5?>J})9JlQI_Y%B-Q9t-%}jWx
z0plS!c#I>?#2H6};uuk9&|y3tBX@Zp>#lBApG(tgv6kmV#>2?08?Vl~F7LYM-0!db
z=-!=9Wahfo8Qp~H+V%fcyY^py{q@&hRsVi@!UU@h*L#jX3tv5=(+$&&auv#vFYdJH
zbkFE`9k2K5>ZM<Oy`J40>J9icc#UnQdXrv)nd{9AmQ$aDU!ym-%~Ee+Fq1d0&023|
zakJOfW~;Zcc#b!}Z9@G77SHwC+Z^={7Poi{+6wCnSv=2M)K*+y%;HvWNn2@sDT~{@
z6Wb=$Ph#<W@8q^A^-~d_a8y@6jX%LVj_T{n3_3yIGF^EszThRjPWLMQOIQ62kB%?o
zi;n8};u_scNPJZR>3*W;OZZYiCT4+5;wJ+#B@1LKKMjzwERgB^3_xaPfpGjRKxSuw
zl=BsU%*g_o%g+O3eile2zhDf=LcR)+MOpGV`Ne=N$pTr*R|B#v3uHOJ0+5wiAglP*
zfUL;^S<Bx7NVZz9<JSYSVRWAQnS3vQE83D{ZFL*4ZXb<>w%Q2DrYw+2d<`I*vp}Zu
zTL9Ub1u~uA2FUg-5RTsg$Q@ZA<@}w1?92k0%kKi@t}KvB{%%0-$pTr(?*^nc3&hFS
z0dj8^$Ws13K<>{1S<XKI$b(rRtN40A9?Ak)%kKeXZx+Zp{s(|GjE2<DLR<Y1Z8bZi
z-?_B?E(6*)F4R3Plphx=j0@d2F0^S}X!E#G&$!T*aiOi_LcQZc+s1|Z#)bN|P`>@N
zqrN<&r5@Je)K_G{MJ+r78W<NE)I#~tX=DAIj1oGuIQ4Tg;QKS+^D^KEGT`(1PK*|n
zd>G*Z@a`kv-G%%~{vdv<aQ`FxF2e7R@$1BI7k(G>hk(6=Ka6lGe*|GQe-z;|{!xU>
z`Nt5h;2%f0QqZ-mQU;X%)k-Y=SA(%89iqfF7;BZ(^k3l??OSQ@`0kes7}xM$y6V>%
zbX#@&G4$Z$8Ibk-6R7nDz6as0{F4Z86K?1Ku9f8#InZ*0c8t{Pkala9wA(V%czrE&
zh+19cQ`e~)yDIfbhuh_D7WjsC(cdnJ!A>Mu>qM6?;PnIr5qB1s+r<lQ9yj87zEE5B
ze!s{EaFcVp=xSuixto1WUQeJIiMh4yE}s;&>~JA21O!Un)a;R?d3F9)!RHA}h1UrO
zg1h`;n_JZeP%nB@BmRH!9)f3d5!6|JdwVbqPy2SYglACkD8GDV!859OW(7-^p?R?K
zOe$XP*m%Lbv0;_wT}eZ@IcaEdB@HeyX%GUHX3>TWBI$i16)RGH5h|YUj6P}dx_nIm
zdQ0x>#Wid9xV+w=%iG!z3<mZDg|>FDD<~kQ&@dHqws$7&O+v6C(C%_~Hc&C*L=;ns
z|A7Y)bm<bgw(j8JNLS>lW8$&Sp3bOcaw2a+Z^`kp?y{?nNyoxH;izRwT2kq;13d@2
z9cd{_zCY*a3MR>Ae1XDs*S|*KPNTYxUWm2god}=NHR%o-BZgl6p=0T77}06_NyN~~
zTBKJmh9buPwAQAFNEdI==F#hdX{{L1YoP~C5mSG9y_9008EY}AA4N==H9BaHm|ILr
zrW#$Zey{mpj^-VVylIMpknuk0ctL7*`HJLdpRK(M0}GU3Z!d0$p!@@d1BTQyc=LE=
zQSR^rY7Mtx9hIc1Uw@8V1<jAPk5CTE*|Yzzt3T?_C??(+bfg~BLL9-a@oFD2@Hu=g
zZ#kRy5?b|DwPkB`dn*p+j>Qw25_7fG)ZXcJJ#%`7Z>pSe+ei;i=*dl|Pb7n%c#BL0
z2k2={Ydu9u87Y0e&Q}`I-At=A>CuCXklwcNl)FU%65A~V0$i|J;5@$eP>>6fFiq-`
zwt(OZ3O+Xp+3Wv~>uWt%x8*vqQ>7_s-tXaq%}ECjiQlll8DgF!hL9|16a`mngWuQC
z+}YkN_yWn?psO)Jg^)n5%u5=*f-jllYHt^OJd-UL9C!>Jo^~N=re_9{#z0U^=6Df$
z+LIQl(0<X?o-|QdX-S$WMJz#a$wG=kSYe!LK)Fq*k}Z{ll6>Cwq>=Z#lNR}P1PDhO
zMTm_!`4UOICGAo%3Z7Vv#B%%(oWVPH={_zh>AEA4J2{qH@mldf-I*z8r$kq7y;Qk1
zR=G7{DZXr(ddV`iuR3O#5x2~UT4r3em-W@f?6c$c+1<v3#dae9cz*BRn58^!DUVvp
z6ZXPxcW=cpPmc%S$<t2fjpz#Vsz0Zod)rk<QSU8%;;CC>j`Fyp{8#SR$_JXyEIPX=
zHg|P=?&_Fhb#&d%OOBmU$Ig))U13?@#$HFS<CLR+8S3`HndxVz$I4g63RY6;Nk?M#
zqJhR27hj&e`qJ#xvDs_mv)A_8`Zk^f@M>Yn)3tq*PwqXnH&$34FD#GFU3IB&RkU!`
zwX%xSJFk{Z>a(2Od1_~3>cT|D{KT9EiL&V<ww&ojpX(5ONx_INx3HQ8y)`3tT`70D
zWbUPsxxWewEPtW%#m-pC(s;?zDEk|65NZ)Be7RuurGnYNTGoHyh4nA4j}<J97c7ky
zEJfs+&EEY;%r-4<o7U&LWSbea%}h9^_ATxCQGd}TM|spyo+z2wukYRU)yI>jJ@d#@
zkDNSs>R_+G+xD?-V$?KIoP;I|8}|ejVwm)4Gy)nX*;wL3gvOGHZg2HLeR}U}QO62i
z7t!<jRDTOY;sudUxmr{$OsjbzSDK97XE<r-Syrn%V+`l)X`B<-<7C}HoPlD_7?TE3
zNanizZS5YfkhE^!y6djpn>XEFyZMYEX>cw{>JJ3yDICWb^lR%n=tn_9-yp6>`W^Tm
z*oL4>H=>(p%Si<v=TG>vQD@EXncBOfzp#JlizTs=IlmHLJQ&Mc5zkvO`0)9{w}Mg2
zx)Gx$8>3xiQKcbiY!d^?oHkDYvH`b-HuOi_L7Ahf@wh{u3i{byAc%hJ^7w*;==Mu5
z<nnbAW(zP|LOzdQ1kl*#YDt<ANOw(;1GxwU(d8v5k4NT8#nLm4qR^4FAP_`I3M3f3
z!KBse5kinkF7`BIv)`A@L*R!J=@lqbpeaaQU2GC06qDQ49z>N8qKbqe1RhT2A_#dz
zmctw5leu0`8$=jA#pw2X=@q<wpFq_z><cDM`}|^1ECQwfR`^L=fy#(0DOigjolO{P
zu%Ho;FBJaBC;m`^o97QD2g@t*hlSBubcLut1%#nDKz$-qKn#&qAf<>K5hX-kbSbUs
zI%5#GAeq?37xY&eM5<hfR9Iqa*U%e7uern>xCLSZAzE0wxOk|&`^4`Q?fs~}N_!j8
zUI}iV_DT+xSE79`k+iK%KyL(9+Q7i#ZH!QDeqR7x1KmvAL<y`T;a=Q~JDJkSJVxdw
zpGRO+Mlw-3L_;AX^w1drom==$5gO+16ruBo&`NOggjRB}yb__!AP%)5`aU@Qj7(KX
zy;ev)z*IhNh1g?!oHXzAh;0)8CJpVu?-Ugu#jA?@D4^l95?x$uqPNe5Hq^}ex7bYg
zInq7AjLSfekVFp;!<KGcVk;FQlSN`WU$X~7LZl&!$&HqwcWG1*+bEyG^WCD>Q>e7~
zFa?ZSIZPmWnm{d26UHLyl}aZJf;>VNdY}_ZtODG0Q5S@$YxjPqc;{zCh)$In=s8I}
zlyx%UP_!*+L{y}Hid#cqKa)2XpWE;G@Y!G|6GS4dQGKW0c>onw#k-lLjuPCw)KPM<
zyo{61=tP?7ey1q$YgAZ#ngZ$zPyhqpsp`LlN~;ZNp@vj~n{P-Z2g@so1f#<5Mvt#4
zM5Z~9@8WxcZ|L_lc$^?|kr2TIAoRCFE_I61n3|$|lZN?wdXb+}zyzVO(!hj!^vy-l
z-$VyPc0i(UM&FAaivC0NJxT%s*<uLc_n`tU_^s&q=<gMTw-&{^LivR77IMBDJB(7g
zk_O**OEmr`yg-_0G|@o8H;dt1Uf`Mq7_SXUQ=8BsCUYtDhxRKY2U9c{+}P4ACM_*~
zUn^xu8iL;~Jvd4f&~Qjn-<UKux;lv#jh(HGVH$@nbPio;Wt}EzX#Sh&6E`)@x{~pV
za@QBnAf3&K8i!tR2QU_f$eiiBc(Q}NGew5Zpha8#e=8#3re617=yg?rFH$GYUjGLG
zk*zHA{t_#T*3z2Ao-)5yCz8^E&a#RsL#uJ)OoRUw;6M!mGLvZLvsp!aEN^Z+Z|;D8
zVAfev)Us&AsL3J;I_SqCvLyXs@C0k&;X_XkAN~mvbC7uL#aBOh_uUYg3x*FJNBZ!g
zSB4M$dic<};X?xo&XYgA_RCk<z%YF1_2ENL3?F(<DYO3-Ciwo)|H`!sCqB9G(@$S~
zPXV1iCV@^LyZ&si3X&fB^0n7qV$k73{lkZj4j<|nK6FBP+@sx}p8B~AeU4S>IpD~}
zUh92Mev1<Txyj_DTvRl@=oeCBrW?Q_X~r>_)u-)2zJrE?#vtL#SIS5hF+_}56anm2
z6@F0HQx?&|IHuoKX-wK1T#YaucX(Vu4<@s$X<^R^HJ&kv3sIhA@HLA<lW-tugXP)_
zdvhDvi>X_YC08=z&Jqql-{iHWHME7iK@SYSO(B<8JdX@t;eUXvC|$a%GpYva;xk~w
z)SH*EAZD4`ZSH>Ls-v){qgUvAXrLnISR8jO?lyIs5;jNI9gGb$Y)@l6qZEznGTYaT
z&Gs6jqe^kD$x+)8Mj0SgI)0-8ax*hP8s-F2BRhzy_MK#o5P*S95Z9pU<8-8d2l$J-
z5l~0c7$?#?(&pzkp5ET?8qmMUKYQ2dyJ95^U)z0lZ!GVYc-}4N?>YbQg}SI^<A_m{
zUFuY@ZG5et8#Z%jJ?;$Rw}nldE67puoNyk83cAtU?7lpl!!^4C%?z(1Y~gsn*Xt4i
z`!f;qgI|VoxpsdD1t}?mgd{YHg1~*3+D&|y+T?c>jBV({_B)-(bTOuefsi6CdWVf%
zCkQ0IPjL(95=DQb3j`7WkrMJjAdz!>KvF*?en5$N92V!vAWQi^q?BADEhvloi_*Q<
z(<riHNV`F~O`?nU2flw)#b(58RQ=k-3B-gFS@S?vP&FW^bhQ!Vfv^&*GP1H6FUNa^
z27o6Hfo#aGQnN#3nTeWgKwxY-bmn_V(eg1UYUZe?ivJ)HhH)plyXX(6(GZLXLtJna
z{o40Qy<BA6n8Ud7`^SrmGbo4NNIm>(BL{+fltwq=K8g8Ko!ZK|T_W%Cp;v?Ns8`Dj
zIrJv=Z1KNH`56IoiGZQ+A@ALiK|zsbCQ2{P=iJR6zuWDBs7vq2LMACEfRsMB;PnbH
z>?zcQ)RcxFmcshAm7}#X=o=|P$tcTvzJI*AHG{GssyzC5h(hnH{MP2Wc(~m~BT1@@
zYs4h21aYlmxbxzL&Q?zR&r%C9g644O=S=J}Hq8Tl{q0!D`2LZz7UcY5H(q_zbOkH@
z6YR>m?#=00<}8yJMLl5CQSM5&wviI=TseB>n9>PCWSQlz9J+FxOS}xgqluRiX9!5z
zf+R%}Z(KQi<xxswl7(zGT;laB-B*q#UQWD0>C}L1m5)oDz4F+V<5wQ1R5sqR7EipM
zcq4HZ<$muJvma8|_#b$ERl{XngW15iYftqM9o{GEm=SQ-P7hE5b&imY5<Y$Qy=&c%
zee%u|l*}kD;hp#`C8PxTr|<pUwMUOo8Y8uZ^8V`+Z(M)+B&9JzOBvsN>HA0Hw(K;9
z^2=oJ`ydA95t1M#(uS?1T9DjPM@Wg<trwpidJRTeS@NV0i;`Xz2!`5yA3pc}BVjl@
z2_*p$Jx@$TWMuYS4#wQ~qHjb8qwhus3BuU51Yj=u53w$oX&;Ne9sNBSY@_E0&iIx|
zh3M~*b|Lz9tc%ha`;vv2V<z6;MSm9^lpu_WnPi9^jUAz~2%oXBg#RX<GZ=j<b|iM_
z`_~wcXKxJfu;3#)P)lHv+~{fH#5m$IiJ>1i3#}~~`f(1_K{g4N)MII~hH}1llJ*o)
zEz1<x#C3oqX)_CH(1Nu&3>$okcD$!k2y0ai(lMA0r*I#$Ac%tdd#H5uXD6RDF40_6
zVZ2<fHPq^I(>?3>tYZhlLob9~-k~$C3Psu44%Z4T`QphyD|T=N!c2K%ot{i(Ok*B;
zfh;!COc)BDKTW|i<F@tt$EIhqcW!9_`r~iUfA-SZKfc+oP+KDZjc0mQax=ByPfuRB
zarDBUjx)iWQp7$x*ZrqHmX^{CK6~QqXFory5LtT6pL%+3{PeBQesY?UQ_8_-CytIm
z?+DsUwrH;Twk?`k8)p_{|5_OP$p%(wOc^Z23rHBp9`+8X9JmuWf0>~&mRA+es~T_(
z>d*2~%d)YIlxZp)4EkzYu_18y@Gpjs92q{+J$&S6!$&R*AAVhJI;KevA34dAe>Z&O
zO<)m_H6R<;hL0R)$-O`!NU9l!j~p95@+zgVv5|4X@R8>kHq+HvGn(+m+_iUK%5K4@
zEep?Rwy<Vc@R$BI7JO_vSMAVvfO_%R2CKD8!`fdBY^vm_>m{VTs-|Ldx^Y+70E7N?
zq*a=er4513wno1<pf%$&1NK`$N#@GS0oYkcd&b4Lk(jY8Fil<-$Xe1c@INA^xd#zi
z5Lh-?6Q8kWL}xJ1W5L)90trX)iOBItUrm2`%rQ6am@BUloLGB&?Xh(|>!PMfO!*I&
z&VOjhqBVOLR?ZIuBR)PN@{wS3WvC2KR3RU{8~Nash#*G%5m&??h`0h{A&5dmhzt#2
z2ViJ!dI@P|N2JpBIOpHGeor7$QCS(H?aUbuOMBAPOz0rhh7ddvJZD-a_?1Z7(=sZG
z`QKv^O!Xd%Bwmdq-iTZ|Rv9i*9-XREXkL1ui=yvD7R63P7DeC2|Mw`2^{_ZL5v1jx
z&kJ}oC{P)yNYB5=8S&u2LWKT@eC|+W=z`BZbRn|GSs7ZAf#~8rtr39&zZx6*4MkZf
zL_Dn-uLSve<K^=a{6uaH4nz>I49!X}_0y+ciClZ`XOZDUFGfCn?$?n|Upp50<o8cj
zRyxFGz+)ST*``8qKi&SAg2yO$k^+(sVjl&kDfl@BFH>-qf?rcWYpvlNiH*Vw=g;3$
zLGeA#Jym-b$jlXCG)_<2Lk+(WIp@#+;FmXn|G_VTznI}?V*ktGvz3wg^YN1jsl;VW
z8t<*GNgB6qygO;!eDCgX*`CHd{3A=NmK~g5g~+mll?&#hX8gj+TlX}kLy;y!fhR!~
zUU|q_wO4*Mjm}D9gapqQJOVo1*N@-P7oT~K3t}mv@6d@8BYM(#Mqj)EC%Oy}NBA#Y
z2lWv;?P%J?R?oiHSBcb7&KQ#BCT!Vg7yp^vi6`;H7An&>V`EiQ6x#e9LWA4i*5(R`
zH-L2m5B{PW5gewo<)2t}d{yt#V{3cXcGV<ut=)XrBVCUuQe{(rQOr>hcT|kI?)V=_
z2qz<ic%5GSGX$C!&sScY_Xh*w2*6+9wZ%UXGObc+1}U;mu-5EH<mOxRrd!Yuq$kK%
zkoK+1S0v*UAQTW&D;v;6@QF{)o6qKGxdEJzyNjLQ5NTt9_&+J2Zi}sNVX+vIN|X3U
zN+iAza|mMI=k>dS&|nykkfMMCP;FSA3<l^~>e6*-gFHxK54NYx<!xy9<5-3GXW)Dd
z;sv%K0-l**KXL!@`+M&>R^L;9dBV&~6K3`s`)gwpD&rF>yS7}ll{|fK-%2{sGgxxI
z<XqWc*+9ua$@|Nrwrw%nwzzFu)U=H?f%pl^)10<+Ac3_*J01sR0ikkbU)HB#eoPf0
z<0M=5p`SPLW-W!an*_89BDNRUlp3)!i8<I<pyzWnZAPd5&@Eo2uEDN@Cf>^1_<Vi>
zZ^u7t#y`htJI-esZ|3xzxDe++)A&bu0_tK;SEn*+ofFC73*<UlBPN_J%6T;Hln|XR
z1z#2Jf^h%A9^ScE>R2R!QcRY#xUrodN18gNQR06A3Ylxb2IK!th{RT5Llp<xfZOlm
z#Q#Q#|DA#uf=c_%j6oU04;zv)#vsyHDlH-%*Oe4<Qou49-s%Z7fOD}`Lrj{*eIA_O
z$xSi8RNpU=FKOEs@_HMjzL0d#whaLXb2?l&%ZGD-7OyK1Y(Q#&NRDnom$b{HK<O$f
zQT`1m|4zN+pAmr$H=*e1ji<Kst?6$ZnDzR+SLO}QI<w&Hg3AlmU0Sg2eC_+?u?1V=
z3%0~cw#G}g#wKi!PuPyWQ#7q_OMm$MwrK7B(T5r?)iy+H8$Me8(ekL^i59iQidy1D
zEnRoO5br2X*h>@k5*8wkzoM*vBhwv|u&We7*IgsCbr$>O+{u^d6yel(?$i;T(Y!QK
zJmYe4#iink{)WN2_?)$|;#=ayw{%+)w&Gs#_=fHciK%n?J^cszJ@Khay6;Fk=BAOM
zL?YIAI}jPW@1nP=BO4IaD|4jOaxjgWyiwE3OlQs7o(c}mktAbX8iyK^xj5=ii;s*$
z&ln|<h<&j^#x-XQDaOzYAP%L4mF#d#q=OS6U0^vXjRuCCtL-cN`J|st>MwkD>glN`
z8%{M0@b8y=RPsUD`(@`L`=}Ld55#N_#BC2mO%IG7cT}1MQRKHxGn&k5C}uos(n>l|
z%q(dYDT1*fMAYM5MI&)DHI~Bkc7-I^q|qpdVh#{EqR7&*DKfbH{PJ__2G<QNA6Wi=
zUDUQCX4?_B?TDIoj3$RV3_pfKAJ;#lJBA~Wvh#u|2F-Br5}x=f{!7=W<L)@t?BkHz
z099*N_Msz4L!*2JX~{yqM(+~yabuJ~He&e-<|{FX;!*p|@^oXAK=W?-;*B&rlCEz3
zJ}3f5Eu%}iiE_*t<>dU8<>Y3RW64&IFSlH$*FC8784nmA)a}=U=6N#H%^0JuG>ecg
z_MX;k<&KfJNXe-KZ)DWZ_7{{>k}8MQAwOGQ+8bCoVHZ0S0n-PYL$SHhHRRUJ28eM<
z_4;Bif~Uc_>6%M~vGFKbW!NQBIcM}DjT~v6txGy>Ygy-YwKejt4dF$yRwng2tr5bx
z&g*x(ynzjfNa%~ucfKe^kzaM`{%Otjk7j*T7p;9Dw&ub3ng_ci-B(h)(0SGCuY7?|
zP=J`Hl@(I;MYw0v{e?AmqUIvoOGXO1_y+{v29>FtZ?BSzwUs38L{mvp`WP!zAd-z0
zGDWa>4v3X{re%-fg1?50%TU7rRe=~__0Er6(R=QVt-de5`o1oC>XY8`e|3FnIQ&-X
zyPWDvsvhe5_O0(PeV>mlyCc5rj&HTLG*o>nwOx@`+t+rTUwWR8Rc(q_ZIWwSX-npZ
zn;TRYLn$3KSs=~935LNF(^EDX#|#}a)U-sJswVRr6ulC7$)W~jLZ;?T+8SiNgb=K$
z{Lp6>q6DVwNQ3teC5=@vKxUI7;uMev4!CrITM<D!s7`hgj_FTEj{T_TM}55H3ZmO|
z%=EF%(Yp?Q5%71lxb)P@f10`gq^OA%-ySc%y}PEn<{G6w?|iN5yeU?^K3=@OyJo~-
z&MRct&sX(_;!_tz7eAQ#d#E8+{KI(h54&r=9hB~xuSN{|yuw6&0is`hg-z4F%a5<?
zUMJVLD(bv1_4mM@Sn=L?@m@Tgc3eN*+Fuc$I4@dxhx+$XUCgmN?%3UJk}z7Lvp1=K
z?*q6a?${yQNCFc;|9_l!d)1~A-B5{P(`2JYK!D1Up1c5&v26`60h&^9um~ZEiKN-<
z-w!Vcddbv-*3!u{k~{`*V=JgQ9m#8`HEF0NBab7~8my@i34f-!uSP^WcBTAL1W8cH
z7q8D$OfXPlJY(i^d7b1f;DWbPu33oT%TO`NEtJHQ#qe|Bb+reCG(3?5(t4^rIesEN
zdqU63r=9)f{lS>6DsHQanyN;tj#NAL4nIWhv8kGES%A$fs8w|0#xd=9qpSo6$F}*w
z^N&`@kArLB^D=SMRUgn9@00Q}dQj^n3O+(c<)%4>d_}UsY4tM=e%3mwgmeqXsOK}K
z$hPZi{cOodKm%ZisQ2MuQtx5A--rHQ`uUCYb6WbjAQ`7}B)b%`D3dzuFoZaa0$9U~
zlL637I!e=zPBgf^aI7JS0dYR?ZYM5z5m_Z3Uw3?6pRsRu%*MrS92qlZM`4OFGi`|^
zm4uTO^Wu(q=)3AdikgHeUT@v8TYGNps<~>L+aDaLi`lB<wraY?k&VISG2806ZM6&?
ztccoH#cZqMwpEleqYwOA5w}&KCfSc{5Wg~>-H3>`-(_ic!j!MkP}VB?@ZB3-0gsy_
zUq<9)2ws(NenFKB0d}%M>2d)s)i=5Qo?tVM(sM!8X&yTT<AmDFT8jGQsNQSGzH^T|
z)g%jmbQd+rOw_YWcP*WI-%JhSOGb_3h`>*<z8}A%FWzu;jLB3)7bHHEuNIYFDZght
zJc7Y$U`n^mWqKaf_BY9<K=yWx#6$^_k~aepd1=~8c6wfynv}rvjt%+s(xI2X2PUX_
z3x3%7+jJgh-crJq*aJ-N1sl>EK}c8d74JI_5RuDJQkO1j*)RR|&gw4gUfNTlCNLSf
zr4A-KEJ}95zfWGXfc+$XUuRo#q7PndaLS?~2rn}Y)c6hiJRJg-I!!Gw`6eyxo$Vf9
zCt^ABuep|$VXnJt*XCuTIH5Fa)x{NHi2JCnn^;{Fwh6S@kYH7Q`oO6l_Xpoz{nm!~
zvaOh)L>~@CCw0Uob;Kui;2mO?{c+3w&y9xsoG*0<XcaVPwAiOcW=yjS5s3XUm}Yc`
z8DGd!cajENLo!G7!)1_%^#e2@DOX8zt1B1?r8i-+utjzhnd%Q~@cOi-9VnMyzu*Lk
zC{xTfJ#L#$GC~zE-y|XG*z2XwD${vg%d^CX*==l@WKdqxC=x@9qzAy=nJfa7ch!rU
z_bh3JLkX8p%cSCZylNx8>MZuES1m<PuZ~&D;+C?m+i8sni_yKj<Wwtan#S5OJkjY~
z%)5fF#i}1jXG_5EbE5?HLXqrZ$G)D?2|98Z4rk$3hyi`QQ84kwqZuwQ;Wpd^ccVGH
z8DTD;gV4g~BFy702(5e`LK|}+YD*U0=l8gUjqQT!&vTc?sHyVTc~pa(mq&CBa+P4`
z(I1+U&PQpd4KtSg(sRM}H0)Av2@YX|(9Gu`gsVV=dAt=N777qzsQ{saFF**l;s}fR
z5`?AvM1*h}j&KS;72!0#4B>Q!KZBq76DVg5RBv|iWW(TgNydxGNMdWh*Z(iY{0Ka7
zi!QAUUuP>xk0I^)KM|Cchp+!viqZ1$^^Yk=>oeDPQ;g<o*GWfZmc#3`Lx!yzU;hoo
zXz}>^|DhPE2iO0FVx)^+-%2rcLHRn-63+bcR1OfyjF!yXhdu|11fgYfH@n0J<{gNf
za+v~xcVE)l0h>o7L(uT1r2$rdc;agaPaV~+P6bBx!~iV=LTQ*Xz1!4dPuNPMwpmxF
z&y7!CGB`CheO-L|y6&2uT_c9VGAm{fz2zfD#6D&*Gh(z6z7VOo6t_t6Jc?VTxQ*iZ
zQhWl%?NZ!9@dAp^9x0@Fk(6Fc@e(OsO7V$Od=kYcOYtcbpDM+tQM`=evqz>=e1?=h
zlj59|J`3?~Q`|Bc{!#HKiB7b&dQsORHeoBlG~V($ua3!%bSxD;b3Q`)J{5sR$#I8*
zIFORe*~ES!IPM{)x)(uu^N6dF5S}!;WtcEMOwB@rCp;ZmO1me}zmFZICk`Gzc<jeL
zKZZj?=@nGY1}dq#>=iyj1k&HokKfT(=F&pCgS>-SK1@w=#t@sMm*kC_McmBZZj@bS
zPEL2K%bOy`=XAU|VnCSlyjd%>>B9qXYfQa07s~Y_Nok-C8(#O&ti9a*F42e6!)v)Z
zILagslbqykk`oTN1%a0wXi{HsxiQFdtiy2reO%aqjf}X5Bi)dzN#N#%E9O-}FZ2Ys
zHcwM?5a)?PI20HZv0nvFd3nwi;yr#RGyyYnM=9QoMoSvul2<H8Ws>H0kv#30SQ96W
zLB$cKS?Z!Rj$n;Uo{|>9DYLs3eG+~>W%rojiqY*q8mpU_p$BE5k3uaE9}oAH6BqP7
z9Cyrus$NhMFW@d0R9q^k=&u`C7AsgBFIe1dz6#fkfn%$CR`&{*Y}2B)X;<?LM+~~c
zyA2<OU{=Ae6k;_k{Z@)Y*O*|3t}($5T|*NXb@^o|Y&8F=6K6Y!MEu9o){a}!hv|qO
zOZ<2n&2|oh>f+gvK7NdH`is0+^rBt{(VM=7U^J$0GjJ70&l9!W2@a=rVOM5de3i&&
zmY-3eS=`Rx>VtY;aga<K^2KB68=Rc`e8w{SVQIUHW!DB8Ez-6Xu}w$Jh+aeh8hxap
zfW{nYgAt7z(rzRgeWZ;@G=i{!M+$yTw*?5`r^6Gr-kD<T;&~jn+UE~)flxb6A%dHo
zSoWofBGW{fk|L5)#w3wQ)0JfLE`M-498$BB`~trjXOu;(5i$X`hfryyK)XxA(XP6!
zZJ?eQo;8}KGf;YopPxrbY@!-1Zj0LJ?`qM!0V5`rMb+`5>h4Wo_1@s|NO$Dw#DxQ!
zViT9dCobtVrll?bt4~}UpSZZ!_*DYZzwY>+?mbtF%KJABl$@O!D_R~eTHd`01u3v=
zMO=SztY}WWXbz=cE1DCXd*|SC1kzuuXlJ}=XEc9j!co+19nIjxHcY_I04e@@0o#w<
zqp#6jLr*pp#_pNeqj51(r8);#--r8yU}Z_XM)c3=o;R`4EPcRGizS6Cvn!-`lYZ*j
zqpr+u*OH^4Ky2sj5+5Ljq53mc4V%Nwi`>4i^$&fm-^*-cU*|qld-v^|Htz*nK1jGf
zpnztUl2K$AVsM7wbNhKAY4q>`u@Na6?h~7l5}rJo<D^_fy}(aUgE0iBCI7Nz<|PYl
z*r|(I7Q`(JqLu~NH*ncL`;vWj|FQv7%)TgYUj&!HQ)l#7{IsgCs(XL;{(%|@UHgpQ
zil}`C{`!igUtrk_PqZFy?JMu=jM*#W_DUo<ihF)^*->%HQGrcCj><UwErKT#iM`UB
zS7k36finys`t%7-O0(8Bnm89zovaP4Rag4yNuw?;4wyyPqBeW>%{Z9g>(6G$^X%Jk
zNW0h86`F=&3|#Z5Eq`O<=KKZa=4QhMt6Fw#!RHK$1FfER_^!rYFJ=kA2Mq#@O(nx#
z6ERxOL$!5x-Mu$#e(2s^ckG4lYwWD?_yR#Xgq$>E>yNuRnNx?1ei5E)+e1k+mKWMP
zMTwKKY5`7Wyp7FgBvau~MR-^w!K&rzhv{KixLPXV3~@CXP{<lw+UtIP#p!i1+x)m~
zKD?#mRV0c^pPBd6ypxruDxX>W)Z$oCdAz6`QqW%1<GXAxzhsAiJb!Q0ULLb=irY6`
zw%>8de#b|fqW9Fs>~(Q_U5bq>!LbS(%bY#Ge`Cx(FK(Zg!kX7#9kb7k+vgIqt8x=>
zpkBn}Ol~em1aec?qJpx^LM<yQ^KyekNA^JkY05*6%whW{Ql{I{a~nv#!n|k7Xu%+Z
z&JhhIi@MxHt1daJVh(5A;fz|GQs+#62eoq^VDF$&YQ9lr*1pfpbWyE3%<IE=Dp(b>
z46?Cdk_hk32PAsr(FY<JJ(4-tsudF8`8R_gg@ht6ma)lYAY4A0G^3E-CL%4y^T25E
zfWG|HhRekZFBLD06)%bxFS=a3{8I7qSn<ku@yeKERot=a{H&PcmZ;?xMm@=3l%K>b
z)CbqN4Gmb9YiRTHAuq-A8yX&l?;i;w(jjY+I7CuKQsV%LcaeI7*g?S~6cFc2PoQOe
zk(TIKKvGfGD!Nc(`Zr=IHCsn?+Q6yHDIGBw%`<7c*UZn2xINk+C0KLJP<Pd!z-}%c
z(WwEKYj(o1x)K!S5>yR}^ybCntw;`X^yYPRn;A$or%Jm~d-YQ1ZtX}S6DqI&g`O5l
z;LFLm2c$WuXWpEUPr0B37&E0WZC%8N8mPlz5qSqbhY>MBD@7VYBW77hPc5Sz*7q!C
zb1d_&%8ixAaKUyTx3t>1f(t-WVVj>bJRK>){el;#_SoiEj_jf;n3++KZxUu<@S1{!
zIDuQSgbVn+JbW4>`_weRKF_{jX9}j|Evlo*7yew_(g1z#AP@|fq|)HTS58$jl-7|I
zf4|4;<r)R9q8iWhh|Vx(f@#U9vky;`{6GX4hMYe{n<+H7)l0C0UMgg|1|Sy#x6OV{
z31t7)AXYDNE?uD(Ql){V6;cK2?F$NY-jnuIrk5Emq)hVy+|ngWs+KHS;#}bj&y$m2
zljfS?RYk;fl$Isnt(#A44W~kw!p+orRZ8m#p#MxH!3l6!;5s~F6M7yT4EY|G+YbnA
z)`13`xZMliM6M2(C#W<aEV|VEsdUO2f}?OWKiZeTO74<P7&AFE9Q?*`OEKMMjW{_y
z6%&9qumpoB*~&HvUKiJa7vlE8#}R8v#c#GVJU0W{f$4rwZBP|YE-XY+F(Zf6t}14@
zA_Fd!NUe*>zwEDDd8g2<Y~@u+t*j!4SB+C7yb@vWtPixO*7mdsqn{Ek+C-fR?~Z17
zqS>p^?3|Lud~vja&$X?mR54u0dMnzyR%iu#2tH?MCTfk;@cyCSc=vI4&>07AA5Puu
zOW`nsCz`x?bAW4dHBxievi79>WN?zxS+=mJDFsz>mehFgE?9iw1P|KV$<<Mxouo-c
z?+2q0K*?2H;{pNjU8*2!=xx|9=?u?EC5HFAeDt^)$#WQk%Xvd2=Tn$cyGjL=rIM8b
zHdA-iWGAyx3a@&J+>8u|9j!?LHHDBTBZ&+`vk%RJ^RVgVhSNG+6L9_1VOWQALt5fA
zfuDMt8?r-?!R*LUr7=Xik+!)S;a*?uZInAif<;cJoFP2XCG|D}%ekrLKwC6HdT*PQ
zmI#{^oQC<O&PK)sJd#d}Z4UThCc}1gD6Hbp3zF<)9}IA^gI}5O4zw?x#52D{c%Jkr
z0i_-x+TM_vMV6&<p%qy$<U*xxmr){@1z&A088@^h3zDW#+HOxh53S%HIzwF#9bEe)
zd8AWirN<ebg?_{1>=`5($x8A(#3Y_F+IgJpa2WNRP4m!YeSWp=g2+BqNmRt7+;D)&
z`4|`91MLiaF*Lv$?}I_sMr=sZK&S=f_InJki}oWPu7kHCw4ERh^~>WPb8qep&CURJ
zkid~;r3b@|gr?Mk4PR+RDdV497|D?ICrO3LgP^t~;(&D2R9nf943Wa0j*(WFz#54W
zoFttPz&AfK>xwScri^KF9}00O5~)1gk#QYEuhFo^no85r$X<q~D%$3P1534`SQnil
zrAdve?O;-_5ZSm`cN#jwh4B0j-et`s$l?)QLISXU=8-#^z*dORQY1sS2hi2rAx%Fc
z$Nf%rKvE0EFr@8d3{feK`mGjU>0(lOG)`&ZQh7@M(iBF6q!u~VV(h~LPS#*rl=SkX
zHlrbJ=oPhNr9UF<xL;{68k0Jm>^SWG91rR4y7(O28^YU580vx#eqGw(a{j}|(KRJh
zX6Xb0N!oAhym**(u!|S`7Z1}HGQuc}_*{ZWS@|$VBF~492Dx-Jr5x5P{9NOQ&k`Qj
z>W9+<|9%&qD3Opn;tS4@6NT1gMMI@2Cso_Uu?`?*Q64a7%$zD&<!984i8<;SRg5e_
zg<8}#hTh!IX`f7+9a3JPx+Ib(UWHF6_;X5}afVi8Ai($2t5+UNyv!xuNxTI&P+S$4
z_%+-|y-5dyxwM?jK#fM=sDL9>Aa~_)%6a7obtCc^1#eQvuRM`>*%_Lj1u4DQbY!Jq
zS}CoE!b6H`M^L!<EH+j++2fGPces4;&WvRk!HWT^{ln*iDI~VEs`7^y^3l~sb?Wf9
z4872L@pUH~<TiH7kam3GGBET43Q*um0n|gO#Y1p85(pBC$`IfOCjT^0)XEKQPT>H-
zsT4%^1EnB!4uF<IDGSm-Y5CE*uAuUYzTm~zsbNzE(zBy4lS>{r?fET;gt-_T{Jag1
z>u|gS7duZlLtC{t@YZvVJp3>ZJdeR=&*4}P7yT`6{~>k)Zhr=2C&&{JqDapui>)Zg
zjUD|3vRvSzZ^WK}>!0_Cu*|j4@9+Q=>SQOjGm(^nf%A6k5IsbjNqGQu#pr9~goqw+
zgrOe>)#YguvOXE>a)$29ipJiM{0pgX_jdFgxi+ejUo7M0H6D-TqLqrQ%=!c%k$<G~
zj7AbKr1{Bbj_{qK9p9XF%$jneijympQsSj4H<a-33}_<3{Vo?Y_*SODcX(R%K}yn;
z9B{Uq0ApL}W|7<P!U;rmc*BRdpopl+lr0F2TAHq<<(yJOGiB_T#(c`<ljdFc?w0_C
zyg~~K?#JMSBBXgV2lvi|CD(v;H!Z$II~grJTMJYQWBn+jC|XE^zcpg`PhWoO+GCHh
zQ2;5|o;>u)yFH&Ce&>^SPe~BAm;k1=Bg-F7A6v2HTJMu^-hJ)Sx0#P`BmouK)MO<C
zt&)MOUE~0V>wpW+58rUIEbt%c(I&Hf8lbUtV~8d=Y(DEiywQW@UTBDvAmfBc2ry~Z
zHuOTk&y!nZ&V?_!2Bo1T2ot4G^1Q<dP4L|l=7xR)I|q_^Zd4Yw0*D7YP*Lmy@cU?j
z>_t&DWk5dYPnvWE-@ZA_xx#W?!$IaJP)ZWKfr}@p!r`Ed%$xx+(Q@OhZtlj@-SBAR
zWZL|VCtkbp{9D|O{$B30Ctkd9^mQ%`L;N+W0J@VBf!clc%A+@)@8dpu^W=@gXa6|Z
z#ohQB3hG8aX`H^%_acfL4^1hMhEY8BkLQlucpgI;CpmWD{`6?ijpHxEt<H^~J^I-b
zR4|*Qq%d!sdv$DFRyf;Y`?Y@F*E#dJjsQ0gn-8bJfOf{5v{DN;(C+X+Qtx8l24G8@
zNh7v>258xXmhI(rd)D0AvwqrBN8)2B^&JCDZUN*Zv*R9p5%%b%!mb@7VS{ecEZVoX
zhy};2-TLmO-TYO1<-jcLm9?*m*;mEwtC$hJ*LD0rH~vMZRQ1&&z~6vg`bEe-;0x%8
zSt(31SL500^&f6eECm-AY(v-bb2~*RSBnq&H9Nz_Qi}BLLOD^fUlr8=ZwIVh%pBMZ
zW1=%$Dt&-3BU#B&u#Ki^waB*BB|=G)x(Q2^=6g!IGIgGDr;Do*T~2PNOAJB>4bwEb
zEz}8J03R{LBoJ`viLNnFl#+@!`h6|rSejn0CPZ%CoZ-nzqGBOZAd1PWVrQFxhtrn9
z5HpzJTdXwq!8Q{rjGxyW!d!B$^f9?_!c$CNq%>@SmtedZ${xA^gOSM0GXbGl<XXL<
zCaFg<9O;W`W1uPa76(eaixe;-ZLP=K0m_mcfu%{G9FvokX$sBs8DTe(aiI!A3o`_I
zLlnY$0wB-OYaN33R@Q~(&ysQ9fGd|Bp0qLK_ch_`WIlft9^2&PcKZVXnscWE<XNr^
zOP}l3kPd$o?b;}IR&jSbVHewvBZ93SC-e5B1(TkgLQpjQa=hM7C$|e3TdO#CxDy7$
zrVpRx!uaYR<`eCmSON41x3bwCO87=ZrCi$NjSB7ZcR+(>K3S#QbgL+tv;~k__C9}0
z@Wbca%^_+Xxfzu-88>XJX$IyFeCb{Ah3!yp=_57ry`neRcjDVNPIxuv?h`S63Kz)V
zx=~Y<;k^pai9Q3pi@ZCYV>;3~=x*;oS9+U^{sf;4;-Y;}J_ng2$#c%os&O-<K0L$}
zDs5gxKifz}oWjS3xabS;O?e*ACNGueNTZc>%$_PP_Bd15ps8}G&pCXjh(Jv5MQvh-
zoT2ShOxl-<{=Z67>PAEGPC;yTHM+QsPVT`E-|)7%!cJx`P}1=s9TlJ`2sGoN2E4uV
z;?D(08u;f&e<CA;+=?22l8c^3xoX2O6u!xMf|$s-V4@{;L(Z20l6DVKKCuhqk$4CJ
zM!fTsa2WB-VQnTG12tdy&^q+$My4_>Jq2h@ibl0^8q|~kL)s4g7(?3PLH&7C)V@Au
zUmv%x|Go@qMHteGw&Oz=SOQzRd=A$LB~K7L!&B5W^`jUvNExV#>R_u9{M>EWd5A?I
z$VC?x&7?H>I}j>BF%lDl?3m~B;8Cy>qPy<H2Mh3V0%j$a0qVC2Qeb6NAj1(%^4xC0
z-Rukbny^?F6o+2K8p;R1#LSDSaoORL3;*gR%4jePCEW$xA0t+q%i|^E7mPV`Tl_Bn
zZB3Nogfk^J;T?^|2^`@mAG=V|71=&h`UXrB#5=|b@&pWj_gJ+E<C`-Ax2Zh5V0?sh
z|L^HCf2Y621+}dW+PEu-9>)d!+_II<l^+a-F|1~XDL%{<;Y{;|okK6c04TM!0CCCe
zuySI`Z!ub%fQrN>(${_X+{G7$UTed=jhr0^1*}!xhRMDd>|~=(HaPGYf>I7yd7{7M
z$VV@{?w(-vpqYKFD|Q0XZIGF+v<TUp{-$S9WEn}3=<D)iJaz<CW8(4x3Z>>w2Qe#v
z;$T?8Ql_uP+gmZlG%6$_GPH!Js)-7T<gVgrQ3jz>ppa1x4svfp0YR()qEmz?rIN{^
zt_o0wBo1|hBXby_*b`L~SYjaYTXJts#xJ&_qauxRmYy|_Vw@u3`G@Y4pZ@gdr>EY&
z{`iSc&c90!Vl6g0J>?Kwo??a_cjKc*ZQPut%b{*~Lryk}mC|Gvw+ciN^Q1W4j=5WF
zYX}GXV2ko&uRj;W`W^lUgMpzl*riNWAT#AnAaW_^ugY|f8uYWl!9N`xggaAA(@uT%
z^J6s4L#gW2N-|hw$Kq56rz!N*^hz_<4C7H*$C@c@tjX%^@i<1Atu)H`k(oxBVj5+X
z0He%;F$S4$Yu-dVR3OH%UkIZmt+U`~8U4sUE<U07#-zb7eFRIa#68YrV260n4=0$?
zzLlkT3z2-ECmqeS2iMy`=cIg?<+H;?$&!pzr-;>fcEm_>z%wDx?s9iJy~yjth(Ob~
zWO|z56UixyHZ36LWu~>bL~CYRxLKq#teEurv8j!!6p#*<aCLRHPB0lB+e!O14p3Zk
zT%YVRVj;a7Ner=?g4Gnvq_j6F#`KGiDaKlVA_3U9dP*o(L_skHr4-yCyb%iiOu@fV
zz&t195R@k8X$LIXR%P0jEbUibLoqs8MIW;bHj7Irn#+4c+C)U3rgwQMotl%*%3zaG
zQO)Kpx8HkbUBi8MZMuD9!<OxLZkDzS(R^Jb^+qJ_6C)J-J=KLahbaC=q`gWT#M6M{
z4@V3^*%6~&uTSXmy4b(8h@*@BOX%iAwSOP$R8W2w`<KwojcWfAy2_~bFQJ<m)&3>m
znO*yr(9Mc!|FWgbj_Ndj*>EyT{#ds;s{I?uvFWGaGh<3nHcLNgM5hh5>2*bAy+x;{
z_HBt3md6XryX+%5t%f2)!eklIO{A~6QeY}BvW}P$`<!mSWNFyeUt}0rqBG~EeG|}>
zM~L`j#^>~%z%SXy0+%vGD4v9=eznYGAxzwUPHA5<z#VXvgfbf?RP~$UrW``W?dSAm
z@-Nxv$=wnvwwQh6^Y=@rcN@OB0{37a7Anvzp^jStkAxbLP}MIHn+#M}+<vY@@FfK!
z2PM>PMpl7wQMXG}t&>pI&*GZQR4Q&ir;qD?$-c0AuY|hqCQzFs)B_UgxUbbIq26g?
zbse_?J8@V8#QMQapf*UT?Goy^6nI!djYz2K$5*m3)<K2}j@@H6>I<+nP!r^DHBOP(
zcwC+{5*m)BeIs*DV~Ew?jJS*;-W<O(RGHt$t)^SgQ1P9KZ>;De66)@o;P+YsD;3)?
zHN7Z@`ev@S7q?2NH4+c0?UD_3vqaM$+yrWager69xTSVVsJk;5BU=S(P3*P266&~g
zZIe)I%->Ac&1P2W9tm~ay6%-wJ4Tmk&y;!l8ERBlI#QQo(oY`IX@dftt+ab~&w^;_
zqL|egw>lHn66t<<%(^0OT`}hUhM4u%xb;>QzBFd7j$5nM`|6l=S=_ozy<ZlyE{|K6
ztM{4ZFNj$e#;prg_^mPPZE@>u>U}1BUCg>ZZe6d!x5cd6<JRr!eP;P<V%D{B>)Mf-
z1r8i?L(pY|w@3Tr?%h50eR;77v*QzHcUeCMIjVcsMkm(9Y@6e@%{cj*{T`lLx5sVU
zM{-RCC=NkaJ`S{&%;>E?wKh6qeXMvxym&)YSCA;2M&n<!?3P&Jx_IHbE_<SMX7BD(
z^-*p`taN3(bY)amG=ipqfKdax8e?g8gMPanO9Yvrbt;Xx+F-3-{~&Njg%-X34wjw~
zT8ik&+F*g67)1?`6$=z<V6jmSH7Ludt5J_ZT@CUxgr-4XMlT`<R^7zOr|y4d&r^H)
zXTLb_<;A~PJUIKUd6%m<UaH>s{_0ruo$=~BW3zY0XYY*O)f6jjj+ZtwYTc>VV-t1u
z@N<^_Wjgc+h99!pr-fUL^XaA3;6D9ix>tjhI#Ygk_>yr-)Ho$!s)(8@Vy3xq)7+?O
rZo*U^HI>IqbK<5sQPZ4+X?oN&J!YC2H_ePHe<Qg%(^O<-+T;HN$>E);

literal 0
HcmV?d00001

diff --git a/templates/tokenizer/tokenizer.py b/templates/tokenizer/tokenizer.py
new file mode 100644
index 0000000..1611d5a
--- /dev/null
+++ b/templates/tokenizer/tokenizer.py
@@ -0,0 +1,869 @@
+import os
+import re
+import textwrap
+from functools import cached_property
+
+import pypinyin
+import torch
+from hangul_romanize import Transliter
+from hangul_romanize.rule import academic
+from num2words import num2words
+from spacy.lang.ar import Arabic
+from spacy.lang.en import English
+from spacy.lang.es import Spanish
+from spacy.lang.ja import Japanese
+from spacy.lang.zh import Chinese
+from tokenizers import Tokenizer
+
+from TTS.tts.layers.xtts.zh_num2words import TextNorm as zh_num2words
+
+
+def get_spacy_lang(lang):
+    if lang == "zh":
+        return Chinese()
+    elif lang == "ja":
+        return Japanese()
+    elif lang == "ar":
+        return Arabic()
+    elif lang == "es":
+        return Spanish()
+    else:
+        # For most languages, Enlish does the job
+        return English()
+
+
+def split_sentence(text, lang, text_split_length=250):
+    """Preprocess the input text"""
+    text_splits = []
+    if text_split_length is not None and len(text) >= text_split_length:
+        text_splits.append("")
+        nlp = get_spacy_lang(lang)
+        nlp.add_pipe("sentencizer")
+        doc = nlp(text)
+        for sentence in doc.sents:
+            if len(text_splits[-1]) + len(str(sentence)) <= text_split_length:
+                # if the last sentence + the current sentence is less than the text_split_length
+                # then add the current sentence to the last sentence
+                text_splits[-1] += " " + str(sentence)
+                text_splits[-1] = text_splits[-1].lstrip()
+            elif len(str(sentence)) > text_split_length:
+                # if the current sentence is greater than the text_split_length
+                for line in textwrap.wrap(
+                    str(sentence),
+                    width=text_split_length,
+                    drop_whitespace=True,
+                    break_on_hyphens=False,
+                    tabsize=1,
+                ):
+                    text_splits.append(str(line))
+            else:
+                text_splits.append(str(sentence))
+
+        if len(text_splits) > 1:
+            if text_splits[0] == "":
+                del text_splits[0]
+    else:
+        text_splits = [text.lstrip()]
+
+    return text_splits
+
+
+_whitespace_re = re.compile(r"\s+")
+
+# List of (regular expression, replacement) pairs for abbreviations:
+_abbreviations = {
+    "en": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("mrs", "misess"),
+            ("mr", "mister"),
+            ("dr", "doctor"),
+            ("st", "saint"),
+            ("co", "company"),
+            ("jr", "junior"),
+            ("maj", "major"),
+            ("gen", "general"),
+            ("drs", "doctors"),
+            ("rev", "reverend"),
+            ("lt", "lieutenant"),
+            ("hon", "honorable"),
+            ("sgt", "sergeant"),
+            ("capt", "captain"),
+            ("esq", "esquire"),
+            ("ltd", "limited"),
+            ("col", "colonel"),
+            ("ft", "fort"),
+        ]
+    ],
+    "es": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("sra", "señora"),
+            ("sr", "señor"),
+            ("dr", "doctor"),
+            ("dra", "doctora"),
+            ("st", "santo"),
+            ("co", "compañía"),
+            ("jr", "junior"),
+            ("ltd", "limitada"),
+        ]
+    ],
+    "fr": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("mme", "madame"),
+            ("mr", "monsieur"),
+            ("dr", "docteur"),
+            ("st", "saint"),
+            ("co", "compagnie"),
+            ("jr", "junior"),
+            ("ltd", "limitée"),
+        ]
+    ],
+    "de": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("fr", "frau"),
+            ("dr", "doktor"),
+            ("st", "sankt"),
+            ("co", "firma"),
+            ("jr", "junior"),
+        ]
+    ],
+    "pt": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("sra", "senhora"),
+            ("sr", "senhor"),
+            ("dr", "doutor"),
+            ("dra", "doutora"),
+            ("st", "santo"),
+            ("co", "companhia"),
+            ("jr", "júnior"),
+            ("ltd", "limitada"),
+        ]
+    ],
+    "it": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            # ("sig.ra", "signora"),
+            ("sig", "signore"),
+            ("dr", "dottore"),
+            ("st", "santo"),
+            ("co", "compagnia"),
+            ("jr", "junior"),
+            ("ltd", "limitata"),
+        ]
+    ],
+    "pl": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("p", "pani"),
+            ("m", "pan"),
+            ("dr", "doktor"),
+            ("sw", "święty"),
+            ("jr", "junior"),
+        ]
+    ],
+    "ar": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            # There are not many common abbreviations in Arabic as in English.
+        ]
+    ],
+    "zh": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            # Chinese doesn't typically use abbreviations in the same way as Latin-based scripts.
+        ]
+    ],
+    "cs": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("dr", "doktor"),  # doctor
+            ("ing", "inženýr"),  # engineer
+            ("p", "pan"),  # Could also map to pani for woman but no easy way to do it
+            # Other abbreviations would be specialized and not as common.
+        ]
+    ],
+    "ru": [
+        (re.compile("\\b%s\\b" % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("г-жа", "госпожа"),  # Mrs.
+            ("г-н", "господин"),  # Mr.
+            ("д-р", "доктор"),  # doctor
+            # Other abbreviations are less common or specialized.
+        ]
+    ],
+    "nl": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("dhr", "de heer"),  # Mr.
+            ("mevr", "mevrouw"),  # Mrs.
+            ("dr", "dokter"),  # doctor
+            ("jhr", "jonkheer"),  # young lord or nobleman
+            # Dutch uses more abbreviations, but these are the most common ones.
+        ]
+    ],
+    "tr": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("b", "bay"),  # Mr.
+            ("byk", "büyük"),  # büyük
+            ("dr", "doktor"),  # doctor
+            # Add other Turkish abbreviations here if needed.
+        ]
+    ],
+    "hu": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            ("dr", "doktor"),  # doctor
+            ("b", "bácsi"),  # Mr.
+            ("nőv", "nővér"),  # nurse
+            # Add other Hungarian abbreviations here if needed.
+        ]
+    ],
+    "ko": [
+        (re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
+        for x in [
+            # Korean doesn't typically use abbreviations in the same way as Latin-based scripts.
+        ]
+    ],
+    "ja": [
+        (re.compile("\\b%s\\b" % x[0]), x[1])
+        for x in [
+            ("氏", "さん"),  # Mr.
+            ("夫人", "おんなのひと"),  # Mrs.
+            ("博士", "はかせ"),  # Doctor or PhD
+            ("株", "株式会社"),  # Corporation
+            ("有", "有限会社"),  # Limited company
+            ("大学", "だいがく"),   # University
+            ("先生", "せんせい"),   # Teacher/Professor/Master
+            ("君", "くん")   # Used at the end of boys' names to express familiarity or affection.
+        ]
+    ],
+}
+
+
+def expand_abbreviations_multilingual(text, lang="en"):
+    for regex, replacement in _abbreviations[lang]:
+        text = re.sub(regex, replacement, text)
+    return text
+
+
+_symbols_multilingual = {
+    "en": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " and "),
+            ("@", " at "),
+            ("%", " percent "),
+            ("#", " hash "),
+            ("$", " dollar "),
+            ("£", " pound "),
+            ("°", " degree "),
+        ]
+    ],
+    "es": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " y "),
+            ("@", " arroba "),
+            ("%", " por ciento "),
+            ("#", " numeral "),
+            ("$", " dolar "),
+            ("£", " libra "),
+            ("°", " grados "),
+        ]
+    ],
+    "fr": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " et "),
+            ("@", " arobase "),
+            ("%", " pour cent "),
+            ("#", " dièse "),
+            ("$", " dollar "),
+            ("£", " livre "),
+            ("°", " degrés "),
+        ]
+    ],
+    "de": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " und "),
+            ("@", " at "),
+            ("%", " prozent "),
+            ("#", " raute "),
+            ("$", " dollar "),
+            ("£", " pfund "),
+            ("°", " grad "),
+        ]
+    ],
+    "pt": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " e "),
+            ("@", " arroba "),
+            ("%", " por cento "),
+            ("#", " cardinal "),
+            ("$", " dólar "),
+            ("£", " libra "),
+            ("°", " graus "),
+        ]
+    ],
+    "it": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " e "),
+            ("@", " chiocciola "),
+            ("%", " per cento "),
+            ("#", " cancelletto "),
+            ("$", " dollaro "),
+            ("£", " sterlina "),
+            ("°", " gradi "),
+        ]
+    ],
+    "pl": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " i "),
+            ("@", " małpa "),
+            ("%", " procent "),
+            ("#", " krzyżyk "),
+            ("$", " dolar "),
+            ("£", " funt "),
+            ("°", " stopnie "),
+        ]
+    ],
+    "ar": [
+        # Arabic
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " و "),
+            ("@", " على "),
+            ("%", " في المئة "),
+            ("#", " رقم "),
+            ("$", " دولار "),
+            ("£", " جنيه "),
+            ("°", " درجة "),
+        ]
+    ],
+    "zh": [
+        # Chinese
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " 和 "),
+            ("@", " 在 "),
+            ("%", " 百分之 "),
+            ("#", " 号 "),
+            ("$", " 美元 "),
+            ("£", " 英镑 "),
+            ("°", " 度 "),
+        ]
+    ],
+    "cs": [
+        # Czech
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " a "),
+            ("@", " na "),
+            ("%", " procento "),
+            ("#", " křížek "),
+            ("$", " dolar "),
+            ("£", " libra "),
+            ("°", " stupně "),
+        ]
+    ],
+    "ru": [
+        # Russian
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " и "),
+            ("@", " собака "),
+            ("%", " процентов "),
+            ("#", " номер "),
+            ("$", " доллар "),
+            ("£", " фунт "),
+            ("°", " градус "),
+        ]
+    ],
+    "nl": [
+        # Dutch
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " en "),
+            ("@", " bij "),
+            ("%", " procent "),
+            ("#", " hekje "),
+            ("$", " dollar "),
+            ("£", " pond "),
+            ("°", " graden "),
+        ]
+    ],
+    "tr": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " ve "),
+            ("@", " at "),
+            ("%", " yüzde "),
+            ("#", " diyez "),
+            ("$", " dolar "),
+            ("£", " sterlin "),
+            ("°", " derece "),
+        ]
+    ],
+    "hu": [
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " és "),
+            ("@", " kukac "),
+            ("%", " százalék "),
+            ("#", " kettőskereszt "),
+            ("$", " dollár "),
+            ("£", " font "),
+            ("°", " fok "),
+        ]
+    ],
+    "ko": [
+        # Korean
+        (re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
+        for x in [
+            ("&", " 그리고 "),
+            ("@", " 에 "),
+            ("%", " 퍼센트 "),
+            ("#", " 번호 "),
+            ("$", " 달러 "),
+            ("£", " 파운드 "),
+            ("°", " 도 "),
+        ]
+    ],
+    "ja": [
+        (re.compile(r"%s" % re.escape(x[0])), x[1])
+        for x in [
+            ("&", " と "),
+            ("@", " アットマーク "),
+            ("%", " パーセント "),
+            ("#", " ナンバー "),
+            ("$", " ドル "),
+            ("£", " ポンド "),
+            ("°", " 度"),
+            ]
+    ],
+}
+
+
+def expand_symbols_multilingual(text, lang="en"):
+    for regex, replacement in _symbols_multilingual[lang]:
+        text = re.sub(regex, replacement, text)
+        text = text.replace("  ", " ")  # Ensure there are no double spaces
+    return text.strip()
+
+
+_ordinal_re = {
+    "en": re.compile(r"([0-9]+)(st|nd|rd|th)"),
+    "es": re.compile(r"([0-9]+)(º|ª|er|o|a|os|as)"),
+    "fr": re.compile(r"([0-9]+)(º|ª|er|re|e|ème)"),
+    "de": re.compile(r"([0-9]+)(st|nd|rd|th|º|ª|\.(?=\s|$))"),
+    "pt": re.compile(r"([0-9]+)(º|ª|o|a|os|as)"),
+    "it": re.compile(r"([0-9]+)(º|°|ª|o|a|i|e)"),
+    "pl": re.compile(r"([0-9]+)(º|ª|st|nd|rd|th)"),
+    "ar": re.compile(r"([0-9]+)(ون|ين|ث|ر|ى)"),
+    "cs": re.compile(r"([0-9]+)\.(?=\s|$)"),  # In Czech, a dot is often used after the number to indicate ordinals.
+    "ru": re.compile(r"([0-9]+)(-й|-я|-е|-ое|-ье|-го)"),
+    "nl": re.compile(r"([0-9]+)(de|ste|e)"),
+    "tr": re.compile(r"([0-9]+)(\.|inci|nci|uncu|üncü|\.)"),
+    "hu": re.compile(r"([0-9]+)(\.|adik|edik|odik|edik|ödik|ödike|ik)"),
+    "ko": re.compile(r"([0-9]+)(번째|번|차|째)"),
+    "ja": re.compile(r"([0-9]+)(番|回|つ|目|等|位)")
+}
+_number_re = re.compile(r"[0-9]+")
+_currency_re = {
+    "USD": re.compile(r"((\$[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+\$))"),
+    "GBP": re.compile(r"((£[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+£))"),
+    "EUR": re.compile(r"(([0-9\.\,]*[0-9]+€)|((€[0-9\.\,]*[0-9]+)))"),
+}
+
+_comma_number_re = re.compile(r"\b\d{1,3}(,\d{3})*(\.\d+)?\b")
+_dot_number_re = re.compile(r"\b\d{1,3}(.\d{3})*(\,\d+)?\b")
+_decimal_number_re = re.compile(r"([0-9]+[.,][0-9]+)")
+
+
+def _remove_commas(m):
+    text = m.group(0)
+    if "," in text:
+        text = text.replace(",", "")
+    return text
+
+
+def _remove_dots(m):
+    text = m.group(0)
+    if "." in text:
+        text = text.replace(".", "")
+    return text
+
+
+def _expand_decimal_point(m, lang="en"):
+    amount = m.group(1).replace(",", ".")
+    return num2words(float(amount), lang=lang if lang != "cs" else "cz")
+
+
+def _expand_currency(m, lang="en", currency="USD"):
+    amount = float((re.sub(r"[^\d.]", "", m.group(0).replace(",", "."))))
+    full_amount = num2words(amount, to="currency", currency=currency, lang=lang if lang != "cs" else "cz")
+
+    and_equivalents = {
+        "en": ", ",
+        "es": " con ",
+        "fr": " et ",
+        "de": " und ",
+        "pt": " e ",
+        "it": " e ",
+        "pl": ", ",
+        "cs": ", ",
+        "ru": ", ",
+        "nl": ", ",
+        "ar": ", ",
+        "tr": ", ",
+        "hu": ", ",
+        "ko": ", ",
+    }
+
+    if amount.is_integer():
+        last_and = full_amount.rfind(and_equivalents[lang])
+        if last_and != -1:
+            full_amount = full_amount[:last_and]
+
+    return full_amount
+
+
+def _expand_ordinal(m, lang="en"):
+    return num2words(int(m.group(1)), ordinal=True, lang=lang if lang != "cs" else "cz")
+
+
+def _expand_number(m, lang="en"):
+    return num2words(int(m.group(0)), lang=lang if lang != "cs" else "cz")
+
+
+def expand_numbers_multilingual(text, lang="en"):
+    if lang == "zh":
+        text = zh_num2words()(text)
+    else:
+        if lang in ["en", "ru"]:
+            text = re.sub(_comma_number_re, _remove_commas, text)
+        else:
+            text = re.sub(_dot_number_re, _remove_dots, text)
+        try:
+            text = re.sub(_currency_re["GBP"], lambda m: _expand_currency(m, lang, "GBP"), text)
+            text = re.sub(_currency_re["USD"], lambda m: _expand_currency(m, lang, "USD"), text)
+            text = re.sub(_currency_re["EUR"], lambda m: _expand_currency(m, lang, "EUR"), text)
+        except:
+            pass
+        if lang != "tr":
+            text = re.sub(_decimal_number_re, lambda m: _expand_decimal_point(m, lang), text)
+        text = re.sub(_ordinal_re[lang], lambda m: _expand_ordinal(m, lang), text)
+        text = re.sub(_number_re, lambda m: _expand_number(m, lang), text)
+    return text
+
+
+def lowercase(text):
+    return text.lower()
+
+
+def collapse_whitespace(text):
+    return re.sub(_whitespace_re, " ", text)
+
+
+def multilingual_cleaners(text, lang):
+    text = text.replace('"', "")
+    if lang == "tr":
+        text = text.replace("İ", "i")
+        text = text.replace("Ö", "ö")
+        text = text.replace("Ü", "ü")
+    text = lowercase(text)
+    text = expand_numbers_multilingual(text, lang)
+    text = expand_abbreviations_multilingual(text, lang)
+    text = expand_symbols_multilingual(text, lang=lang)
+    text = collapse_whitespace(text)
+    return text
+
+
+def basic_cleaners(text):
+    """Basic pipeline that lowercases and collapses whitespace without transliteration."""
+    text = lowercase(text)
+    text = collapse_whitespace(text)
+    return text
+
+
+def chinese_transliterate(text):
+    return "".join(
+        [p[0] for p in pypinyin.pinyin(text, style=pypinyin.Style.TONE3, heteronym=False, neutral_tone_with_five=True)]
+    )
+
+
+def japanese_cleaners(text, katsu):
+    text = katsu.romaji(text)
+    text = lowercase(text)
+    return text
+
+
+def korean_transliterate(text):
+    r = Transliter(academic)
+    return r.translit(text)
+
+
+DEFAULT_VOCAB_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../data/tokenizer.json")
+
+
+class VoiceBpeTokenizer:
+    def __init__(self, vocab_file=None):
+        self.tokenizer = None
+        if vocab_file is not None:
+            self.tokenizer = Tokenizer.from_file(vocab_file)
+        self.char_limits = {
+            "en": 250,
+            "de": 253,
+            "fr": 273,
+            "es": 239,
+            "it": 213,
+            "pt": 203,
+            "pl": 224,
+            "zh": 82,
+            "ar": 166,
+            "cs": 186,
+            "ru": 182,
+            "nl": 251,
+            "tr": 226,
+            "ja": 71,
+            "hu": 224,
+            "ko": 95,
+        }
+
+    @cached_property
+    def katsu(self):
+        import cutlet
+
+        return cutlet.Cutlet()
+
+    def check_input_length(self, txt, lang):
+        lang = lang.split("-")[0]  # remove the region
+        limit = self.char_limits.get(lang, 250)
+        if len(txt) > limit:
+            print(
+                f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio."
+            )
+
+    def preprocess_text(self, txt, lang):
+        if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "ko"}:
+            txt = multilingual_cleaners(txt, lang)
+            if lang == "zh":
+                txt = chinese_transliterate(txt)
+            if lang == "ko":
+                txt = korean_transliterate(txt)
+        elif lang == "ja":
+            txt = japanese_cleaners(txt, self.katsu)
+        elif lang == "hi":
+            # @manmay will implement this
+            txt = basic_cleaners(txt)
+        else:
+            raise NotImplementedError(f"Language '{lang}' is not supported.")
+        return txt
+
+    def encode(self, txt, lang):
+        lang = lang.split("-")[0]  # remove the region
+        self.check_input_length(txt, lang)
+        txt = self.preprocess_text(txt, lang)
+        lang = "zh-cn" if lang == "zh" else lang
+        txt = f"[{lang}]{txt}"
+        txt = txt.replace(" ", "[SPACE]")
+        return self.tokenizer.encode(txt).ids
+
+    def decode(self, seq):
+        if isinstance(seq, torch.Tensor):
+            seq = seq.cpu().numpy()
+        txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")
+        txt = txt.replace("[SPACE]", " ")
+        txt = txt.replace("[STOP]", "")
+        txt = txt.replace("[UNK]", "")
+        return txt
+
+    def __len__(self):
+        return self.tokenizer.get_vocab_size()
+
+    def get_number_tokens(self):
+        return max(self.tokenizer.get_vocab().values()) + 1
+
+
+def test_expand_numbers_multilingual():
+    test_cases = [
+        # English
+        ("In 12.5 seconds.", "In twelve point five seconds.", "en"),
+        ("There were 50 soldiers.", "There were fifty soldiers.", "en"),
+        ("This is a 1st test", "This is a first test", "en"),
+        ("That will be $20 sir.", "That will be twenty dollars sir.", "en"),
+        ("That will be 20€ sir.", "That will be twenty euro sir.", "en"),
+        ("That will be 20.15€ sir.", "That will be twenty euro, fifteen cents sir.", "en"),
+        ("That's 100,000.5.", "That's one hundred thousand point five.", "en"),
+        # French
+        ("En 12,5 secondes.", "En douze virgule cinq secondes.", "fr"),
+        ("Il y avait 50 soldats.", "Il y avait cinquante soldats.", "fr"),
+        ("Ceci est un 1er test", "Ceci est un premier test", "fr"),
+        ("Cela vous fera $20 monsieur.", "Cela vous fera vingt dollars monsieur.", "fr"),
+        ("Cela vous fera 20€ monsieur.", "Cela vous fera vingt euros monsieur.", "fr"),
+        ("Cela vous fera 20,15€ monsieur.", "Cela vous fera vingt euros et quinze centimes monsieur.", "fr"),
+        ("Ce sera 100.000,5.", "Ce sera cent mille virgule cinq.", "fr"),
+        # German
+        ("In 12,5 Sekunden.", "In zwölf Komma fünf Sekunden.", "de"),
+        ("Es gab 50 Soldaten.", "Es gab fünfzig Soldaten.", "de"),
+        ("Dies ist ein 1. Test", "Dies ist ein erste Test", "de"),  # Issue with gender
+        ("Das macht $20 Herr.", "Das macht zwanzig Dollar Herr.", "de"),
+        ("Das macht 20€ Herr.", "Das macht zwanzig Euro Herr.", "de"),
+        ("Das macht 20,15€ Herr.", "Das macht zwanzig Euro und fünfzehn Cent Herr.", "de"),
+        # Spanish
+        ("En 12,5 segundos.", "En doce punto cinco segundos.", "es"),
+        ("Había 50 soldados.", "Había cincuenta soldados.", "es"),
+        ("Este es un 1er test", "Este es un primero test", "es"),
+        ("Eso le costará $20 señor.", "Eso le costará veinte dólares señor.", "es"),
+        ("Eso le costará 20€ señor.", "Eso le costará veinte euros señor.", "es"),
+        ("Eso le costará 20,15€ señor.", "Eso le costará veinte euros con quince céntimos señor.", "es"),
+        # Italian
+        ("In 12,5 secondi.", "In dodici virgola cinque secondi.", "it"),
+        ("C'erano 50 soldati.", "C'erano cinquanta soldati.", "it"),
+        ("Questo è un 1° test", "Questo è un primo test", "it"),
+        ("Ti costerà $20 signore.", "Ti costerà venti dollari signore.", "it"),
+        ("Ti costerà 20€ signore.", "Ti costerà venti euro signore.", "it"),
+        ("Ti costerà 20,15€ signore.", "Ti costerà venti euro e quindici centesimi signore.", "it"),
+        # Portuguese
+        ("Em 12,5 segundos.", "Em doze vírgula cinco segundos.", "pt"),
+        ("Havia 50 soldados.", "Havia cinquenta soldados.", "pt"),
+        ("Este é um 1º teste", "Este é um primeiro teste", "pt"),
+        ("Isso custará $20 senhor.", "Isso custará vinte dólares senhor.", "pt"),
+        ("Isso custará 20€ senhor.", "Isso custará vinte euros senhor.", "pt"),
+        (
+            "Isso custará 20,15€ senhor.",
+            "Isso custará vinte euros e quinze cêntimos senhor.",
+            "pt",
+        ),  # "cêntimos" should be "centavos" num2words issue
+        # Polish
+        ("W 12,5 sekundy.", "W dwanaście przecinek pięć sekundy.", "pl"),
+        ("Było 50 żołnierzy.", "Było pięćdziesiąt żołnierzy.", "pl"),
+        ("To będzie kosztować 20€ panie.", "To będzie kosztować dwadzieścia euro panie.", "pl"),
+        ("To będzie kosztować 20,15€ panie.", "To będzie kosztować dwadzieścia euro, piętnaście centów panie.", "pl"),
+        # Arabic
+        ("في الـ 12,5 ثانية.", "في الـ اثنا عشر  , خمسون ثانية.", "ar"),
+        ("كان هناك 50 جنديًا.", "كان هناك خمسون جنديًا.", "ar"),
+        # ("ستكون النتيجة $20 يا سيد.", 'ستكون النتيجة عشرون دولار يا سيد.', 'ar'), # $ and € are mising from num2words
+        # ("ستكون النتيجة 20€ يا سيد.", 'ستكون النتيجة عشرون يورو يا سيد.', 'ar'),
+        # Czech
+        ("Za 12,5 vteřiny.", "Za dvanáct celá pět vteřiny.", "cs"),
+        ("Bylo tam 50 vojáků.", "Bylo tam padesát vojáků.", "cs"),
+        ("To bude stát 20€ pane.", "To bude stát dvacet euro pane.", "cs"),
+        ("To bude 20.15€ pane.", "To bude dvacet euro, patnáct centů pane.", "cs"),
+        # Russian
+        ("Через 12.5 секунды.", "Через двенадцать запятая пять секунды.", "ru"),
+        ("Там было 50 солдат.", "Там было пятьдесят солдат.", "ru"),
+        ("Это будет 20.15€ сэр.", "Это будет двадцать евро, пятнадцать центов сэр.", "ru"),
+        ("Это будет стоить 20€ господин.", "Это будет стоить двадцать евро господин.", "ru"),
+        # Dutch
+        ("In 12,5 seconden.", "In twaalf komma vijf seconden.", "nl"),
+        ("Er waren 50 soldaten.", "Er waren vijftig soldaten.", "nl"),
+        ("Dat wordt dan $20 meneer.", "Dat wordt dan twintig dollar meneer.", "nl"),
+        ("Dat wordt dan 20€ meneer.", "Dat wordt dan twintig euro meneer.", "nl"),
+        # Chinese (Simplified)
+        ("在12.5秒内", "在十二点五秒内", "zh"),
+        ("有50名士兵", "有五十名士兵", "zh"),
+        # ("那将是$20先生", '那将是二十美元先生', 'zh'), currency doesn't work
+        # ("那将是20€先生", '那将是二十欧元先生', 'zh'),
+        # Turkish
+        # ("12,5 saniye içinde.", 'On iki virgül beş saniye içinde.', 'tr'), # decimal doesn't work for TR
+        ("50 asker vardı.", "elli asker vardı.", "tr"),
+        ("Bu 1. test", "Bu birinci test", "tr"),
+        # ("Bu 100.000,5.", 'Bu yüz bin virgül beş.', 'tr'),
+        # Hungarian
+        ("12,5 másodperc alatt.", "tizenkettő egész öt tized másodperc alatt.", "hu"),
+        ("50 katona volt.", "ötven katona volt.", "hu"),
+        ("Ez az 1. teszt", "Ez az első teszt", "hu"),
+        # Korean
+        ("12.5 초 안에.", "십이 점 다섯 초 안에.", "ko"),
+        ("50 명의 병사가 있었다.", "오십 명의 병사가 있었다.", "ko"),
+        ("이것은 1 번째 테스트입니다", "이것은 첫 번째 테스트입니다", "ko"),
+    ]
+    for a, b, lang in test_cases:
+        out = expand_numbers_multilingual(a, lang=lang)
+        assert out == b, f"'{out}' vs '{b}'"
+
+
+def test_abbreviations_multilingual():
+    test_cases = [
+        # English
+        ("Hello Mr. Smith.", "Hello mister Smith.", "en"),
+        ("Dr. Jones is here.", "doctor Jones is here.", "en"),
+        # Spanish
+        ("Hola Sr. Garcia.", "Hola señor Garcia.", "es"),
+        ("La Dra. Martinez es muy buena.", "La doctora Martinez es muy buena.", "es"),
+        # French
+        ("Bonjour Mr. Dupond.", "Bonjour monsieur Dupond.", "fr"),
+        ("Mme. Moreau est absente aujourd'hui.", "madame Moreau est absente aujourd'hui.", "fr"),
+        # German
+        ("Frau Dr. Müller ist sehr klug.", "Frau doktor Müller ist sehr klug.", "de"),
+        # Portuguese
+        ("Olá Sr. Silva.", "Olá senhor Silva.", "pt"),
+        ("Dra. Costa, você está disponível?", "doutora Costa, você está disponível?", "pt"),
+        # Italian
+        ("Buongiorno, Sig. Rossi.", "Buongiorno, signore Rossi.", "it"),
+        # ("Sig.ra Bianchi, posso aiutarti?", 'signora Bianchi, posso aiutarti?', 'it'), # Issue with matching that pattern
+        # Polish
+        ("Dzień dobry, P. Kowalski.", "Dzień dobry, pani Kowalski.", "pl"),
+        ("M. Nowak, czy mogę zadać pytanie?", "pan Nowak, czy mogę zadać pytanie?", "pl"),
+        # Czech
+        ("P. Novák", "pan Novák", "cs"),
+        ("Dr. Vojtěch", "doktor Vojtěch", "cs"),
+        # Dutch
+        ("Dhr. Jansen", "de heer Jansen", "nl"),
+        ("Mevr. de Vries", "mevrouw de Vries", "nl"),
+        # Russian
+        ("Здравствуйте Г-н Иванов.", "Здравствуйте господин Иванов.", "ru"),
+        ("Д-р Смирнов здесь, чтобы увидеть вас.", "доктор Смирнов здесь, чтобы увидеть вас.", "ru"),
+        # Turkish
+        ("Merhaba B. Yılmaz.", "Merhaba bay Yılmaz.", "tr"),
+        ("Dr. Ayşe burada.", "doktor Ayşe burada.", "tr"),
+        # Hungarian
+        ("Dr. Szabó itt van.", "doktor Szabó itt van.", "hu"),
+    ]
+
+    for a, b, lang in test_cases:
+        out = expand_abbreviations_multilingual(a, lang=lang)
+        assert out == b, f"'{out}' vs '{b}'"
+
+
+def test_symbols_multilingual():
+    test_cases = [
+        ("I have 14% battery", "I have 14 percent battery", "en"),
+        ("Te veo @ la fiesta", "Te veo arroba la fiesta", "es"),
+        ("J'ai 14° de fièvre", "J'ai 14 degrés de fièvre", "fr"),
+        ("Die Rechnung beträgt £ 20", "Die Rechnung beträgt pfund 20", "de"),
+        ("O meu email é ana&joao@gmail.com", "O meu email é ana e joao arroba gmail.com", "pt"),
+        ("linguaggio di programmazione C#", "linguaggio di programmazione C cancelletto", "it"),
+        ("Moja temperatura to 36.6°", "Moja temperatura to 36.6 stopnie", "pl"),
+        ("Mám 14% baterie", "Mám 14 procento baterie", "cs"),
+        ("Těším se na tebe @ party", "Těším se na tebe na party", "cs"),
+        ("У меня 14% заряда", "У меня 14 процентов заряда", "ru"),
+        ("Я буду @ дома", "Я буду собака дома", "ru"),
+        ("Ik heb 14% batterij", "Ik heb 14 procent batterij", "nl"),
+        ("Ik zie je @ het feest", "Ik zie je bij het feest", "nl"),
+        ("لدي 14% في البطارية", "لدي 14 في المئة في البطارية", "ar"),
+        ("我的电量为 14%", "我的电量为 14 百分之", "zh"),
+        ("Pilim %14 dolu.", "Pilim yüzde 14 dolu.", "tr"),
+        ("Az akkumulátorom töltöttsége 14%", "Az akkumulátorom töltöttsége 14 százalék", "hu"),
+        ("배터리 잔량이 14%입니다.", "배터리 잔량이 14 퍼센트입니다.", "ko"),
+    ]
+
+    for a, b, lang in test_cases:
+        out = expand_symbols_multilingual(a, lang=lang)
+        assert out == b, f"'{out}' vs '{b}'"
+
+
+if __name__ == "__main__":
+    test_expand_numbers_multilingual()
+    test_abbreviations_multilingual()
+    test_symbols_multilingual()