From de703343d0f084c118f66484dd3b33750dd84ba8 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Thu, 24 Mar 2022 13:11:26 -0400 Subject: [PATCH] (core) disentangle some server tests, release to core, add GRIST_PROXY_AUTH_HEADER test Summary: This shuffles some server tests to make them available in grist-core, and adds a test for the `GRIST_PROXY_AUTH_HEADER` feature added in https://github.com/gristlabs/grist-core/pull/165 It includes a fix for a header normalization issue for websocket connections. Test Plan: added test Reviewers: georgegevoian Reviewed By: georgegevoian Differential Revision: https://phab.getgrist.com/D3326 --- app/server/declarations/tmp.d.ts | 8 + app/server/lib/Authorizer.ts | 17 +- package.json | 4 +- test/fixtures/docs/ApiDataRecordsTest.grist | Bin 0 -> 172032 bytes test/fixtures/docs/Favorite_Films.grist | Bin 0 -> 300032 bytes test/gen-server/seed.ts | 640 +++++ test/gen-server/testUtils.ts | 103 + test/server/customUtil.ts | 66 + test/server/docTools.ts | 243 ++ test/server/gristClient.ts | 167 ++ test/server/lib/Authorizer.ts | 305 +++ test/server/lib/DocApi.ts | 2589 +++++++++++++++++++ test/server/testUtils.ts | 9 +- test/tsconfig.json | 2 +- yarn.lock | 5 + 15 files changed, 4151 insertions(+), 7 deletions(-) create mode 100644 app/server/declarations/tmp.d.ts create mode 100644 test/fixtures/docs/ApiDataRecordsTest.grist create mode 100644 test/fixtures/docs/Favorite_Films.grist create mode 100644 test/gen-server/seed.ts create mode 100644 test/gen-server/testUtils.ts create mode 100644 test/server/customUtil.ts create mode 100644 test/server/docTools.ts create mode 100644 test/server/gristClient.ts create mode 100644 test/server/lib/Authorizer.ts create mode 100644 test/server/lib/DocApi.ts diff --git a/app/server/declarations/tmp.d.ts b/app/server/declarations/tmp.d.ts new file mode 100644 index 00000000..039ec371 --- /dev/null +++ b/app/server/declarations/tmp.d.ts @@ -0,0 +1,8 @@ +import {Options, SimpleOptions} from "tmp"; + +// Add declarations of the promisifies methods of tmp. +declare module "tmp" { + function dirAsync(config?: Options): Promise; + function fileAsync(config?: Options): Promise; + function tmpNameAsync(config?: SimpleOptions): Promise; +} diff --git a/app/server/lib/Authorizer.ts b/app/server/lib/Authorizer.ts index 63f1a906..6c735317 100644 --- a/app/server/lib/Authorizer.ts +++ b/app/server/lib/Authorizer.ts @@ -17,6 +17,7 @@ import {IPermitStore, Permit} from 'app/server/lib/Permit'; import {allowHost, optStringParam} from 'app/server/lib/requestUtils'; import * as cookie from 'cookie'; import {NextFunction, Request, RequestHandler, Response} from 'express'; +import {IncomingMessage} from 'http'; import * as onHeaders from 'on-headers'; export interface RequestWithLogin extends Request { @@ -95,12 +96,14 @@ export function isSingleUserMode(): boolean { * header to specify the users' email address. The header to set comes from the * environment variable GRIST_PROXY_AUTH_HEADER. */ -export function getRequestProfile(req: Request): UserProfile|undefined { +export function getRequestProfile(req: Request|IncomingMessage): UserProfile|undefined { const header = process.env.GRIST_PROXY_AUTH_HEADER; let profile: UserProfile|undefined; - if (header && req.headers && req.headers[header]) { - const headerContent = req.headers[header]; + if (header) { + // Careful reading headers. If we have an IncomingMessage, there is no + // get() function, and header names are lowercased. + const headerContent = ('get' in req) ? req.get(header) : req.headers[header.toLowerCase()]; if (headerContent) { const userEmail = headerContent.toString(); const [userName] = userEmail.split("@", 1); @@ -543,7 +546,7 @@ export function getTransitiveHeaders(req: Request): {[key: string]: string} { const XRequestedWith = req.get('X-Requested-With'); const Origin = req.get('Origin'); // Pass along the original Origin since it may // play a role in granular access control. - return { + const result: Record = { ...(Authorization ? { Authorization } : undefined), ...(Cookie ? { Cookie } : undefined), ...(Organization ? { Organization } : undefined), @@ -551,6 +554,12 @@ export function getTransitiveHeaders(req: Request): {[key: string]: string} { ...(XRequestedWith ? { 'X-Requested-With': XRequestedWith } : undefined), ...(Origin ? { Origin } : undefined), }; + const extraHeader = process.env.GRIST_PROXY_AUTH_HEADER; + const extraHeaderValue = extraHeader && req.get(extraHeader); + if (extraHeader && extraHeaderValue) { + result[extraHeader] = extraHeaderValue; + } + return result; } export const signInStatusCookieName = sessionCookieName + '_status'; diff --git a/package.json b/package.json index 81f7e629..5c7c8f43 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "install:python3": "buildtools/prepare_python3.sh", "build:prod": "tsc --build && webpack --config buildtools/webpack.config.js --mode production && webpack --config buildtools/webpack.check.js --mode production && cat app/client/*.css app/client/*/*.css > static/bundle.css", "start:prod": "NODE_PATH=_build:_build/stubs node _build/stubs/app/server/server.js", - "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support NODE_PATH=_build:_build/stubs mocha _build/test/nbrowser/*.js", + "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support NODE_PATH=_build:_build/stubs mocha _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js", + "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js", "test:smoke": "NODE_PATH=_build:_build/stubs mocha _build/test/nbrowser/Smoke.js", "test:docker": "./test/test_under_docker.sh" }, @@ -57,6 +58,7 @@ "@types/tmp": "0.0.33", "@types/uuid": "3.4.4", "@types/which": "2.0.1", + "app-module-path": "2.2.0", "catw": "1.0.1", "chai": "4.2.0", "chai-as-promised": "7.1.1", diff --git a/test/fixtures/docs/ApiDataRecordsTest.grist b/test/fixtures/docs/ApiDataRecordsTest.grist new file mode 100644 index 0000000000000000000000000000000000000000..bf3c0f397a69ae63c966c0ef4b951e67c1abf8da GIT binary patch literal 172032 zcmeI5Z)_arnb>zpic3nSR<@F?WykTDjw5X>*)#ip?K<(2Tv?1Lil$bUb%IOJ&b&)b zw7avOKav^UAtfaz#9B^L(+}($3)9Z%-hazaN zz_lsh=F)!X^Stlu?Ck8!k`(V8Nq!a*x%>Wo-rw`Q&--_WSKeGP9456)tD`znK5-(E zOeS8Jq(mYy1^>Sc|NAfejQ{(uB)m!WKg9f=O1$*dA0L6piS);qZERYG?QAsgy}hBC zZMV~n3QI4p*K3Ws)Tk}2)TQ~TB=gc4Lzk9U8}&Ep>(bi#@}=7Pb?IXL`dMkdr&_G* zEbH@r{8IgV?dnQHlKt2<)1H@BuU@LJFE2{{XoXd)RY(GoVeUr#?S>zB`ZPp1+dUQ) ze%sJDnRB`47-kpB9nEOi=fR-ewmL5@T)w|R^lj*Y)%*aoOAHr`^I+SaaZv2JwvB7TYW z!hBY*uB>Et-knQjPM=Qhelx;Qf>v{UjA@Ls30`cpQhSzxeBLtH?KqnmLdeZGx{+pg ziFMR2*v^O|vODd^QkiTvdG}sA{*&Nj%%e3r9~QUk1;C@o?CQN zwQn-sS4K_G*0!KEZ$VBl>*xkMMu&aF?2dMd?9NliQkjdzg;{dKzrNXSZlkR zMt4bd)D5?5yyvown`}EO`Z15f)z#%UuhyY2FV)|cq7nwmk<4ynk~2MC>#}}Ug4(jX zSB`SObZW0IP4e2FH6zR-S$$Q1QCmwz7oV3>7lYk~wd zV0w{8wQW-~REO!(ZNu5(F~O3uJ8QG4jG`oWFNUa|H`;7q<;b)jax{i$KYmZ1(&B~s z;zd3fye7%nvwpjHR~YfcezbYJ%dF+D?WkRiE$iWgUb^h=*;Fd?#v94I*FsIIY3S=O zz;I?-+Y1(yvNf=Wh{&0khDIdav(29RQ9G+{hgpUuHMs=R@{-@cuG(RuRigj&Gx??? zgqAR!pGoDdT4ybPfBOiFDZ6M8$nv*Jq|Poi~yR-sSE0+Ax#bP+i9q zE?_L64P|Wf?A_Ooq%yC*n*3H~oF-z*nBEWlgK!`2ydI9_CF#t(k8nO4!g@iqx1uxx zSZkcS1pZdL31f|)QOsll`WVt8WCSN;wj_}Z-#J@`?Pv#A1WGZv+T=E`^R){~M z3Ui+ttA>MUz_;*_XhSN1naY@G*Jf5IdPqY$v$b*YCXAETh=zg!w2kh~u?fbq-FY^d z%Dnhua>os^#gWn;m^uGA5kr@Ohlkns5<8!#WJb^#xMFfZd5g-CYfCvx)B0vO)01+SpM1Tl9ssxT=!)X2g zsH&IRMFfZd5g-CYfCvx)B0vO)01+SpM1T{Z^M5h~B0vO)01+SpM1Tko0U|&IhyW2F z0*^icTK_-#I;Iv90U|&IhyW2F0z`la5CI}U1c(3;z-RMKrvDd$cpjh4C*Bg52bjQz z?@r7lcIIZ2-+e_>nNn6uIgM5Gg`8Z@mvbehTxpeAf$4fx(elkoRjHIzxm1$#Wxb+T zDy@>-gwQg}w^|c-T>Juvc%TP9FvoozJ|4DIch)Zx9)n{{z$X_U9Ene?y8*uwJOS(8 zVr{d>>>KzXNg)V4wa(}n@DvOY$a6hkTUn9$I|H8Fb^}W@u#h`E3?CG->c!d)Z;jiv znRCv7hfTTDkWM@ZglBQ_ah2YqLqq_Ig%4HRO#Jq5T>IT0++N^C;nPXn$015>v-e+z zM-B2vsl23_9ofb2%7_O5iN~t=)QSaFEj7z!tx(M?T2amAlvXvL*IBvPEalWzrCCt& za;unA^3`Iiq?L+ItyL^lHC<^|t1X=ii_f=W`VACo&{F=9$o^^26%V#ABoxUnii+D z)jP8Mz0D-Ln8fV&zjj!&j`T2xEi~j|&@_0*eV{$eiJvuwJq$OHNQbf-bOxxXKriM7 zH3MPL<9Ig=P zpnP}0lbvrg8oVz4_}hKfg*MoK_Ak#OntHF2P!0IFxIicKWp$B&rQY7ar*rL5Jjyw+ zu?5xg%Ey5Kj0xEA0ItAWP(M6`kO%BhY(VNYb(8g_#^D4X0@-y`d~W8zZhMgEe(MUD zFDJBNGyWV!uO(P#6DAbQx@PF`U_`V7KJe+RrMhiLgz6X?v>cHRbs?@0Ex{#*^PI23A_*XIdSWC45!VwwU1yM?Tt@C$w7tpo3qg}SX<|_2QJ&7 zPu}5zL3e~oMSRGJgY60w;P;47XW-V)BmHtlskY*zFtR8-305odvw ztF62yD@S-KCCHVpC{?XesN$h$K|xHcR4i1Ql`>PSWwoj3t%_1AGB_huOVu0$sdc5L z7i6Wyrg$lGK?a;kDPJinRggi|iv^IRS_e5iBm~?3;qzhj(1RwzA)Q1HTChysl*FOeAkA**=)mYpqccL|77~l6X~C(|2+Nir<@}-p9l~EB0vO)01+SpM1Tko0U|&Ih`>Wd;Kbzcx_|Qc zWN_6#F*|u`R%{pGw>Y5g-CYfCvzQM~=W$Vk$8= zhwJ|YKE8lnM1Tko0U|&IhyW2F0z`la5CI}U1c<;RO91cx)B69BRWCJ+2oM1xKm>>Y z5g-CYfCvx)B0vO)04E^s|NluM{U-#32oM1xKm>>Y5g-CYfCvx)B0vO)01>Y5g-CYfCvx)B0vO)01+SppJD=EIdUQ~o6Mh_oBeEl249IIo0efa z7g)Q;tc@kZ?zPqJT6g=4i!Uui#GIPI{AtL4GA;5?)F8F;vRnhI$3)cJ{1i|1oJf^~ z)BLH$b=G=Ctkx~2EgqT7O!73Jotv5SN}60~u=kw>B?%F;&z**d=R_49wd<>Y5g-CYfCzkQ2+U3WIB`7j2Z?kh{cq_#`uF*@uUW|u`34Z^q`)^)Bh z$1%E_qt$2ced%~AQ!FOGZ3dAvP)+uRX13i<*8?*YmX5?Tlw=&Fo@#+q*yO%4`>0S^ zA{O?X=B3rEm+I@wi#|z(RjXA<0+K}obGCbtByJn}CUY+L9N;*L0ZKIN^QP5t+v>cu zaQX5|y|(ICiZZqOk)w)jt4-F9qVIMcgDs+@=Pk3dB9cWFZo8e1YHeRJT}#6Z(O7h` zT^JjG%diblTWmb+1gdTA+7^p3BoV*FdSO1RS65cDJMYe=GN(@`cfT27D566P^&cN& z8slt&7aOhAo@F4Pw+wbW&Sr)XazUaYn%yPnP+hRR5k+LdW&JgG4=007^!ofG=N_^< zPaR8TE*6tJHKVJu_eBHkZF^&_?QR;~CDl_#Rz)8n-+>t`jXExUW=DECYEmIJy73CFE&IPBPv@kNOuojEFbueh;< zN1r<ikqq50q%XGc_^X#2;Imd%&Wb8{AqX>=1DY3^xL>Wa z#SnpC!!5P#GHGGu@`CS?PM`LgAVCe7UZhcN+tduzVY+nNaJG0%u%ztH+H5MLD9PQ6 zA*$z%HX96r$h04FG=^wDeovm#;)VL+MLu%BCdt{ee!KWcIWkcC(dO+gvzEKIqjoj6 ztcMeN>9V_LQ>n}wZzS(t3pJ^xp|8IH$1SF{yfBOiFDZ6 zM8$nv*Jq|Poi~yR-sSE0+J?h6RM#Kg;?!Zti^>CEG%x0jL(wp69~6O|FqC~Lo;Pr$hM^=H&IDm@6<>*E!u)1z3hw8} zrue#z3x!Y}ef{oUpG;-WolD+h(F*ZLRAKILjvQEwg*9~$4fqxw5^YEYu#g!O?b^%= zMGxWCnXQeBH({K#Ml=)@plx(-j!iI@?as5wROZDOlRIvREsm7lDz`td+7mHkdD;Vj zHFWI9d`;pFBEHG;i3-|5KP3Cw#B}D(7x(VnLg0Y6=K+(jo}a-rJ)Qp#b~fl85g-CY zfCvx)B0vO)01+SpM1Tkofk&0V+|eH3g(G(+pHBYH z@o&w2<=77s{|}O^CeH6cfbS+8{b90!dxS&VCL6WIl}oJCgmdpLqu0M8FdR5Pw1v{Y zDG*9C=E8v3l8rtk)V7Rv)Zug%+V%ZRKA+s_1U7k4+I430)7yZoevlL=sh{FL&qez? zilH;zp0 zWZNT+flCjL;j|-fQ|PRvx^3rLyb7A8r88?x0@#hk?cNYUsML(VqoHl7meYv2;Q`;I zGF|6NkHIeRXsJSDaj`aBM1p%0z%YFG0k)W>dE6MjVrf@I*NG+VB@+cjTP7xHs1{sB z8J7w@pN`!o)Q;@V>CdGyFT9Z4eSFlu0(NyO-k!$UkvQWB)rn?saN^mW%4db7`Jkk0 z>Sli{e1pG#GE!0=HBQo7;L2jvfJd2KN5#|MXeIPOiul__*_~6L;Wu4&-V2J}P@4;? zfi7H~ZM?9TH~ zq%tqR92(6*pf(d8Pgv<7;InZudKvz1q+ENPpG)q3l{eH=YW;oi>0PzG5g+)ONNI;N z-oM0nVgr`}OTMc}Dj`Jkg8uIl7^?!QDm`I1nZT0V>1G9?-i-vSkyA$=}0!j(494@_~ulnq_n~ zqo=m}y=N%wxkxTUNyagW^PJf7dPg;)uMI+Eq>`b1o0jSJ;vxTy>TFn!WYp~@_gQFD=UwZBsHeO|gVK#C~=qqES1<-g^Yx{RJM^z88 zFs8zN)##!Jnb%ayW}|~En(9D4{Bm@Z3>NT-_s>uUsin{0i2+ z;*cU7N7c6Qs;+N_gOD#ot7DL0oJL^c1GlH3;1N+kSd5IbCX9=lYhm0K;~UV=qO=H? zH8W^8W7bbOv*+}zOU+Q(2hsM(K@bjEl5y8a<>Y5g-CYfCvzQM}PpX|C8xo zB;X&thyW2F0z`la5CI}U1c(3;AOb{y2oQlsj=)rMDlt1N*8hK#NdL_v*Fb6z5g-CY zfCvx)B0vO)01+SpM1Tko0U~fDc_cAA>Y5g-DeUII@ipMZ=0r{>Y5g-CYfCvx)BJl7Kcx>`$Vm6tanS-_dV!< z`af~v;b|C2N(6`i5g-CYfCvx)B0vO)01+SpM1Tk!oB*x=56+Bo5CI}U1c(3;AOb{y z2oM1xKm>>Y5qQ`L(E9&j(-_H11c(3;AOb{y2oM1xKm>>Y5g-CY;NS#s{hv(#Gy(tU zMFfZd5g-CYfCvx)B0vO)01+SpM1Tl9+ypYoWAHqHlXEk3bE&zn-ca88=KmB866^n8 zB+|clxEn~)69FPX1c(3;AOb{y2oM1xKm>>Y5g-Cl1g4Tx3HWmW5H*$l*NOE1NdKqw ze}FgiA_7E!2oM1xKm>>Y5g-CYfCvx)B0vOw$plVLr4pIBxu@qgEz|A2Ww6`KvQJJ1 z1M9lsn3ny-1P?qVQr7g2(Y2pQ2E#7j?m}kz{{JsorBpo;AOb{y2oM1xKm>>Y5g-CY zfCvzQPk;cw|DU>+fPeHN0z`la5CI}U1P({wAAf0PCb2V@O#bsFxssPzlNEKX+SKHh z#?*YZT5Xo{TC-TqX>y^Z$~nEPH}#65t6DBsQcH?fEayuFRw@)*shKpCKk z=X$=jvLf?$Mz>|&n1I9v7IKG&>CDzvy;$4ft#P|HbI!q`gyT*_y0+21xvcXnZVEn| z&K9#E0N?YBj?rP8iQhi<`G56;+Y3CzwVCBU4)JQ6z5n_qb2cr*c3#rVj_l$eH14Su z$XFYumeX=Ns6cBq%bJ|iN_v$w0eDfZwknlkQ!X+k-)!catpZbX`9eW$<~2nvROF&o z()3)83v0sx3)63)+=G_hKzQzB@Z)MvR~@#_G}F>~`i9zUGsO@D@#i9cFNZUuMryjg zXtv!>H;~gk2@$+?7g)Q;tc@kZ?zPqJI~-NZv^s8E;&RVYsb&g=fx-OZz(b^tWAXmER3l6Gc zA4QF~q8N*CvEyAZL{D_Px+#qGAf|Z(&BR54*8?W#87z;(hOV=Yd5euUG)68_=pf^M zY%Qik9ZZ8Ds^>q3dVmjPzY6pFj-T_s^s_3v$50}0UBWR57s?$0cUrYxuf2U06xWaD zPUdix_Kjiybib_*n`yKh2h#|4{Xxf#tikhx`!SLS-XU>hpOj;d)(@Ic+z=$=`v2(k ze@~=q$G9!kYwOFGYU|gfi}mYgrTJTM zBkv01OHNoR%=@X9>gQ`$R~nKGX<+pf8`t8}%u9{>+YPCo>GbLB&flL;WnO$SxoZby zZ&=3W=0N!V$84nZ{n&Bh^TK0=W4B;E-GF(t9TIT$>dFc<%&^y)s^jV>wg|C@5zdbd zy!KowbLv#`?kty~!B@=U=b7Q=h`@2J=%`&+ZC^1hXI@&pda1s?yy&x380@`(thIS* zVde6IAMQ0_A=I+dry=cPv>qcClXss9NxNnlUD)pJwg`YDwg$#-g0;P^V}VLuo#xoZ4XI^AD@kt4r9kjW_fLnFcc6D zHljL)ZNr`<6w-HxeV>JnuIHP~2xo{@wYIlK6%#8q7ov+EWL{G(n~f@RSiS~(AACVb z*O0^l0Vt&Ty0 zaT6uWhbMd>2TFPmxnlI$!a=x4^DdkG5%nD4` ztBRIyR;o&+q{^j|oGdz~-ThL3}A$69$=jy$Fq+PjY3zD2$5ugu{M+mZc7 z+~gfh%A?P|uysRXjg*2%5;d_?j&JMy?C)2YWiA=F44j74(Mbo6K9Iov_Pj&rAYVcr znq9~{cyoZynbh=vKW9A6~wmBS@P9668AzxR>#`234|)*rwZJdp7^BevI@ zrE#D(cTsm-42MNbN42)`tCqq034BTghu{M*HR5P7_k!EL32NHoB#Uc}*J)tCH&DX{ z${9G_63$@Qlk!vA8vnV#e=Y_O-$f26CEz$!@&EOW=L5&xu>F zV>sXrAqNcK+`jPbz@mnG<5Lh9RCvER8!dyib$(-k%Qonfcer3;|5n6@j5ye?KmmS_ z2z3VD&v~R@&WPM2^P*uxf*l+WKZXxGD=axM9YVxn`h-ly1rF}+0;U~vPb0cckr=O9 zAy`r1{d&5XiUa1&O^&xUChf40{T>#RI8gAQ-^8RHYxg2gB!q(-m@w5ts2p?wDo6c} zTruI^1sTD3RPXehZLWKM{y#bK`-!6;P5&GCMK2=oz!Uf|c|=^U{mwV?tvp*of(GPp~fB8g`-_qo*#X9%};{CSKn-d>+t0L5-J&GADalDxZZ-E&1@a^a|(dYrl zZ{Gg)Q$j+9E(*#YCZ;?e@cv=VZgMKp2OPH8kPqO^#~%;04|w;tZXC`B$ORd`-k_B7 zm7-DwN380_0=VO5L6a+LQBjqmtd&|7Ij>b)C8bn`z3Wz~3HJt+QUx4xo=-*(=L2vF z)xQ*vwhb$Zy{FZC^#A)R^FDeGw-UU!eISnzlnxqv@Xbrn+4fsZbi3m2HuEMO)ZYe0 z@pQ=hlfP}zkmX-rm=rGHd%t>EtDBgNbODDgHsk`ZA>8Lb&@SL7|L6MQT)?4^kI=sM z_>RrEA#x$`MGG`OJhwfB13<0qx5Pcf!X>zG!RPmuDYBz2L~n5HzlVti{VmCY^Qtjl zIN1XTs)tXjP$vG}qyu`3kaG4YCYtAJU;?#=yhG3$fqeU$(?pPuEhJi2q5FeOuwu&b z^v2i!6O%tp9DM?Q(2EEVfk%^bh{! zzdPhysaY%*byY5?uo0p(Rk>U#XmE>1QT3c$f$xInSxYMyRkd0wl`G9EE}L38JzvQ) zrpkIVxLm?rro(KR_=D#m_eyZ79{3V1cBjFSSwtlm#P>b+b%ek-(*n&5Af>c;p)O)%L%$uaR>_;$-iW_&WYU%#Uxu z#hma&m$S#l24)(Xfy@8!=Cl|p{^_cANbhuHI&!EuY_XxC0z2x1KUDnWKOb^;{m^F; z$jMt)#|;(xTgITx2*u*XJGg-aPM2R=8ClVOQe7xCVtLi>aTv`wL#~1iP2LJk(_)6?q_$z-9 zKKpn$V&uEB7mN!Aej^}(UtgdX5g-CYfCvx)B0vO)01+SpM1Tko0V43o5}@_}Bdc0! d7!e=>M1Tko0U|&IhyW2F0z`la5CNaS{|EDBD6{|o literal 0 HcmV?d00001 diff --git a/test/fixtures/docs/Favorite_Films.grist b/test/fixtures/docs/Favorite_Films.grist new file mode 100644 index 0000000000000000000000000000000000000000..f5c4feccccc28143c3de704a288e770660d573a0 GIT binary patch literal 300032 zcmeFa37i~NdM6l>DXAoZKnJ3@NeYAz2#EU-O}B0&kc5!L)dGqa5ig`fRh30nr9h|! zDo}~5yWQ^b__nug@6LFq?OA-ZXLiO%`yAWj@wSg?AG01a_PBTbvpsHmy}oC6{@;tp z%FL+9s>*y-DL@$%p{h8(c=6)>zxTcGJo?yy@+2?CmFg&)ESbtOMOBsUrIMm3%N0f4 zgn#SXYW%XS{s;b5@&7>6-+8|-SMK}7?|_i9y7;Tut2Y<_q4?LuUoHMd@h^&hTKvC? ze^C6r;&12OxG2Y7-@a^M&~?=dw@$KPgb$ynmM11B&P)vN2q(*xvEhkHKAwAU`K}{- zb{yMNI<{lyfjy<6+-rtPca@{k{zJ$1JhbOX>F|;L2X`EKy7b7Nr?-}d##xn*P416| zN;?lcy0f%<&)yx6A2?RhaBA|*IL{m&8p820Hkx}}w0ssNaZ6Yk*&j{h5vx{SYIEi^ z8>#W!dtR8RjLo98<@IZe1B3V8tG+Ru--VejT{d^e&DCd2e3L`9(M%&mrL92=k6tCQtI}tiB{lxi637BjvGEvu+rxjf`x0RUH_-=N@&+%=F9N@(7=3 z3>fiep`}+cC)*FBa)d!tI=1JjW2MIr?SJg?JzGl=n`B~4ZF#V;V(_thRHZx?@s}rF z907`l8BnukC17^@D*U#ogk9-I>~6RgyN&CyTYnYK-f|^=TeSkafn{s3Z$UNv-QW5> z(C>lEq6cVF@8kcn_|*NE?DxQ|9_Y{iv$~{zpx*;8JKT?W2ia%Yr3%_hkI^ZjVTehgLKi;I=9Nt|C_m9PuM15KRYekD{Gu4#W zPtK+8jX*7{9~a8oan7r`3hIgQBp+o@;4F0a8TBsda051RiqBTMZ{ED6ZwF|$LWo}O z+W}^czK@U{pf*_%c7frHtzlTt94He0i^X44ihosnAOG~Xeh>6};KSSlYl_R28`YJ2 zt7SeGO)OtuSgveRS03h7Vd5VP`9yKGx?CAtwGyVu(TO$1Ym|a|x1!!%*rwds?!}AO z6s{SxZS~DP%>uzOGOAAu!wg*;9ZSW6mSd}Pg@cw$<`NO+u%aiY4p*{*g3f*4hb$mW z;~5!2%zN$3?=O|36MXW~aqN=c9abvOIPIT_Wtzbx#HeRZ~x{7#Q5VS$%ypXG{pUfZ^mzubg8P%=P&N*;%u6 zA|qDIqpW)7i86VUan z%_`emxpHOk6JJlxiZ8y8kBsx`@b2=&_y{|*W9-Zwd)etqm3$M3+uV1oJUPO1u6+3F zI?>ySXkk90bI~eYMYJ|-T(fe`25R7TI7HC4;{-p(U!KH2>erMF>cHk(!Bd`$_^s4q z^|#!0l2zeJ!>ja!nLj`gZ@iJ-7)?VzM$ych@h9 z*~kRnI>biGC&q~J;kFSTPY#_+A6KG_Cm=7c8o1}ivn$u!Pc2GcW@n8ikKxhs$Ougz z@L{55cYeEg2-Y<+0rEOp()_q;`NT;)FCkXCt4?jp?FD-1Oh4DVSzw7?Sp4FsbkWQ} zT5vCu{NK0#*So9#Z@&jFuO8^*|ML3Y{kQG+Kywce{tL=EMg3Fdocbxq`@h_Lc8Py~ zePpFrQcrJaUTEzf9f#bT`cMz2e_fHgh)hq;yNZN7o3{Xj#06=WlOUcsEmoNkA1RMw z?UL_fUa3L*@nWqqnZ1`5pm0EE<2-~go<~^R)$ZiPsq%PvtUOtUlsL|?u9zJ|faOLd z!m2!TN;LW;o7l%$G~0%vZMm>+VBm7s;!KddUmIbWuA^^83bNT}M5m6hfRAKu5hq#{ z4ZAAYYlv)ZY_iO8YJar1S{Xf%Ly4}6k|lFKiPPnYG6q}z$ts6MFMRSOhfS;=jZ#YaM$0zyYlPu{B5aEHUK!GG&!IuwsDzEDS7Pfqz#PS1cE2moHyNe->60 z#o6Mr0xn*;OvQc`;eT22Q3crlwc@|UFa53G1N|Q8wg+AqSfOkfoZ7QvWQ327Po6nD z6rL=PL@*EyZ9AU%16GW9Xq&E`+X}b5Yvisu(SZ^lk#|Zxeg}+rW^Z#P;i^a3Wqs6=MXMgMW zz(=|V?pw1KGv)QBx@rx!b!5LBn1D)Ugf}Cn6!A$0{2qUH%a!8#Esg7o16WyCs#w(v zPn%Nm7cRZ?inW-Fuiw8M-$Pi0juh=oXhZPA zUaz7>tFCXlbLJG`zn~ma)GuKty}ce?H88kihkD^?vqF#DG$w{8c#?D@F7uHn^%lxG zvmsZ}mviA&LFB z-@9!6rkP{)dyDJum^oIzx3K>9`mxpZdsnX?nmJa#ch&maW{%bGUAg{-`murfy;rTj ze&$&H-Zks5n>kj$_saF_>c^JV?_Irq?aZ;2LHx8DPoRS&Ewd)yJ7L0wlb+}JG7@0grq;Yo6mtQ+Z@jtu4snN1hWC1OI#;`=Gm(hBiH0Mzv-5_USJNx`&b*-jCN}g^4&j0E6_Kv zaEHmf-qOWffv(s%RE#`8x|~tCB+sJesR~VzTs5ZFGG5Bj?4@`>y{h<#rX< z-8!(WuKqh#Ia4}{(9AR2Kc3vF{qRX;P4Qmff2C4fS^O`>pDuoT@pSR&ly3i@eh*wW zJ#agM-3JEMb(`vFIZ^@23x?5*F7(FMfsN|Ap*ll2h$I6=-9N74OpSJj@&BFTk{efm zAg&$ETr1!_v^+Vvsm?L>oMvMM({6s)re(=B_taU%PJ|aPH%($fP0Zz%A}+gLU3Yz* zUmQIXjqx+f3_W2jHy3bz!>V<+Zb#E-J60Jr>=|CLruYE#f4`y>9#;e9SBlTW0{BD8 zb&WkP5xfZheqr$b`_-v(vu@=W=~)sxb5qs_ap2}wMM$cQY1}ZEQbar=-&A@U+MJv- zTc$Q_6d5U|T68~n+)h?)JR5)9)UI7PH&>h5JQ!*gw6;w$k}Xr?*9;79+N8dbC<2>( zJDPvLv9(Rjr{-$Xyt*S1grCv;2;C8>r837!VKkF7AahRKlasn3bD0P#nF==y4BmL7 z`bMHaOg&`Mf39zR(e$X)JGRpb*Ap^`xjI0CEyJK>iNa{&x$H6mC$zrv9kB3rr$Kak# z^WM|^B<)IZ*Wf+dsjdq)1pG~&EFIv;LQ!4*aOD)Tf3!{##ud2Z_Eq=X`iXx`?$})! zD;+6^r;xwtVD-Mz;mRbNtduBlaAsz>e>HW%sx7yEB6SBnRvs;pU+dV;+AFV=4v=Jf zhO_F-4If{H8+7&FP1}c(ha3%Abp-i?N)J~~vat!&7g})$((c4bRytAvL87Nu>LD z3DaQd;p&QG74$qD(yFJLpJvkhd^>f0D%|fniRw@#nusS>9G&E+IT}fVe)CNp&D&el z+wZB*+u|mKYL200BYb(%$S2vEF`oGX{xZ=_^ZNR1ezbf7*@>u$PjYseSC{R^cX$Z7 zTd>5aoj`h$@)%EMaGhrOd(};YGc$bV*5V%eD)^q`Bm9hzFCc`FJag-+!CUL{%*+6O zvQml8%zQLj|1T^4yi)wj;vW`&6TkGgeh>6};KS7eYpy_G&%nU;G=JZk)%0hvt}|Fe zF+u~&>saT7>_9-{=3s_09XO?e~C053Hb&K#CLu4Yl|WMCoTmZO*2; zoT$?ItHoa`{#@~=ivOngrQ&;t5Bg`t?JO^luYRxko$5EMpHtsaFQ~7oFRN8` zM2*$w)laI&)I;h%b*K6<)loI1M@_b$nxgFJKRHEe-k+G-fc=k8QB}o%Gez3m|9$Fe z?Em!?N-Qe>YKj=^k4;gf#V<`=f&Gt8t;YUGrl>OG4^NRk<%g!Idg^;q0b;7gN|N|LbemeLt|Q;F-MORKed-8yXCK4v71E=fc@(>s zDJtcqgV>!ufL-ko>?R+^Zel-n)qU8#_z-sEd$FtR!ES6fcB8wn8`+87sU6t8upPVd zC$Kw7y6+PYVi$i5JN^K6(KhTtAG^TAj=9)9?_f7xRfb(d)xis7?8f_*G zGo416wi3JhXpG%ULu@OJuzP5L-A&_b3k|QIIZS)C_+zxcuJ~icQ^nf~U&ZQwyZYDa z52#EXRQ`kVhTyQlvAB+a?d)z%lgDGtJP@ZTu40n#-6O zxUuF2n&ri|#&j>X3@z{@-L~8iM~x7#s;(E50?+b!0M9%t=5ZVtK@jL+=;;;@G*=Ie zP>TW!2OT5UnC~+`2tAt{h7;OO5JV1;rmho|Sm+0a@4I*-E7E;e*L1^oqR?|Z6aO>Y zwF28R0yhdY$93b#^IhBYsPQb0{Q#ptUAq^(S74eK*^aAm+-Js|IgTDVj61$>2Ttt8 zo@+Y3?%NuX*5XLBY%7Y4FtRLP*ELW136Q#4G|IDCq(!c7FxL-kM~{6TN0G*3hwFid z?$hHa&~4YpLp(i-@ZcbJT@HG_6GR&LfO?gn_87}*Tk1X z?}ScZdXeRaw&hz<$e9+$mZq6Z(*PuOjd(pr4=hl!e8Uby*Vp6Nu?){g$J?eBgt}$1 z(B(WtPouFkJWPD=IEZ;<8on7SZ-Ug7f|RXsCvbe%@Pb&2Jm!a97)EAf`<@pt%?fmU zGskjVZrT`;wif%AX&DF^vMuJizVd01x;j`$3YTQFVK-kuil^%Sv585Btib!D?4>iLwmf~qiowx`G9{dIN;g1-53gJD8iw(9 zClrX`$>u^1Ij4uFp1giw@S%s)*YC~rTF%9d4$3)sTbsVjxpV>D+T2a#+S^jxdZuGa zB_`e&HlBJ$tC@3K4qmrnkUh|0r?u^_Tc^$q4h%l)k@x+RLZJ4yqCrdgP3srR;H z*MTE^LeyEShlMvEQZsI?i6UjV`I5OjKSN5Jo=vGF5*i6@O-`<(EmL>iGBD`6g;pnJ zJn$f;N2*SoERQ!ewCSLlaZGJAwdta{8k^{4G7hEWW-8J-W{+=~y5Z)5!AER$YDZG4 zRruDD8uRc-?L>JD<)#=`apf0leE1YU)6hUQPT!fTpB_K7|FOsSK;g4{&r_wm3!CVa z;P{t!$z9{(sr#d?B|L4*6u(Ji7;7o+muXy1gYRgg5Nou#Lv2{SRf#U_(4)snBc>yNJ*F(PU6n;jLtdlY)<7%g+@_X1>8#&d|Byyj~yKh(|8$B-Gh+0|BdDVr!CEeQQ z+Pt@%tV_*jj6@m^r3_;w`b6$_&K z!pf*tBhHzu2NO<5_WVQQ0p!HNqlEFzv?+-QBVg}G<+nrpB@`P^o-r0xnwaFQxtN%0 zI`O6T$&-G`<+C&ciVxFxNP>T5vqC>}Y~OGQ*+i$-tWv-GT3yrPFy!3zwIB|B!}S~+ z9&vD1f~OsPp*RMN?)vb}GaUG>1)*(+CJSSoT+$*dEaVOjS*n@AK|L!vu$P15@E>(Q zI2H#XG;ih)Fd@qI=hzV;BlLETZ2*}jTfvBg9F$0~vjm=ep_^`P2+|1KoKGZf67R$# zCWfI*tBsBw7ZRcnhCy<(nE%yx75wzKOWOk%s>s6vH*58+XW$ndL>3QCc#v8aJWt_z z?ZD+Zv=|(sEgr!o7A~vsSLX1Lg0ob$Y8im#@C;7-oMQd<}IhN5GD*4N(Zl0dZ|9$*l zTI45BzffIP$G;Oh4u``$^E~o^4>c`tEY2ZdN2ZQ|2-gTS78_ji;HqxgF0;93`WmDx z*Yf<>DK5*!zbx4d{>6anHU2**i+^b**e(7^CQa}kh=ut}mFh^;G5&>Qn=9VWjsN_o zr}+P(qJHs%0D*oR=IDV7N0z7j;U*DaU7}>Bf%74UZ$Z2T?iEvgNt_L6&TWKhPxC^F`Quviw1sNp{O0NSr#`F+UJOSQq>t zd0HR;a{%rGcIv{B6?Ob0TFG^jYT$eaXw=@&I9xjv@pyu z+=yEm#0JDN#@g}~x%ihQo54Q;q38I2Ul#w;Ot4%0)BN2z{>7}{1^$z#rTD*1QE&Tz zjrI>pbEcmeNKp3 zq;AE|SE!v8ivLM6Lhg@*lrE}wG|kE+M<*6b0^Jc#VT_3mkHj5ub+h?Wz_Y7|(+dg0 zx;Dppzbn!+E!8wUgtl2mQEGQ5M$p;UHxwJ~)t=N4@s4e5)U!WQqx*A>(`~h$)V_>e zyiE3%HV>w5ZFVRS?h$RS&9nZKbk*karX~vai5VXUYGQIv?yJq?<3LX=Q&Y-xcbV)` zZ5lA^-5L+3Q!S1adG9vI++XfJtBpX1fILg=3j?#1F zV^T=tEoPS1A>QJNFw2vHdUg)bEdfZ=`Tu=IeIL*6Zy&iH`1Hn=iTLs@Lzd&&wjBqK zV;c+xd;^+9!-(Tpw~<}Lb70g(CX-OtVoeKUGc@t56C*W8Y(s0HM*}N!#TQw!8S#a{ z(|hsd(c9(37ik%;z4)@EEam>h{#R1e68gQr&FX>aXIG`-zrl44NpM``k}(_w%HZ=PU?ZD@0luByw>rc*ew*9Rg;tVIn(>AGiV1?-v*I7o{BbH3My zPX2EQ*11A*ll&+8f5z*9zC)fzY8yFnmW77K&vAKE6IrZX z%o)cXa%Lg}Xz0am6o$-!WgDpzuedUI+{u#7j60f2TF0GYPd|S}D*St<>o_m~M24v| zq@dJX52;5@$8impnX#p7$O;)k($@@TGmXI+UU$e_8;y z^wJ}5`*+?YC;UrukoMqNsQ67{WvlL=q_MO%(SoG^rt7;9|7Smc2IP|B|7^-lEv@VS z#V}gh$O-K?@0J@S(oktXN~n*T;v9DgN9l(3NBC?e!BaJg$DlzacB0!FzMgXnks_}0 ztgsEi6C3FpMT}*QB;zDfWPg;zZjh*&96?Dj5t3OWF8xv&<$Wd}9j9EN6G^}djS(E4 zJXyvsiq=J&D1Ky~Hi^zz)HXE<=`D*i(IIC=zl+2VX}{BWZ4>)O&6oE`2qU*A&y z37#*M?)0LFiEGo@t-YH+M9zuQhtlY`ahXw z=ZH>P2tJaS3V{!?Kop#pzR!a3L3gx_Me%v&`-Y-zONI}6fqB|Q`W5lr8<<_%Hqpz@ z(JW&bzr$alTSht44~d=xy;d{`M>8(I00mLJQc$l(mZ4OkZHu7;pe z4K)Z*$HC|Nm1}ZWPqJhM+ipY@6Ju>%0*VWkRxW=3fqUfS;_lbMcwL9ySyj&~_~~z# zpa(9TyD9<6TUBNlQ2-?gil|sn#n~uM66l8E!HejI_ zg-&$eUvpJ1NMy-oKqAt3(A3mR6~p1*mjsD4r)a-;6mvVt+4*vYkS^+BKQzFavYQv45~Kq&ov*>2OP*QT0&$f$u~q4pIbXfV5m z0sP}M8|r*Bf~KF_R7%XYnTxVpM#L;sit<8~yb2tcHaL&N;Oe!xFq9>mfgvq!dk(|* zZoFR(hSI#KQy8{YTaY>=QClRb!-(+C8fjRY@F|?lXXfXFm8cw_$K~ zH5V_qK?#-orTt8kT11WolWuy~6#rMD9voVI*>BU8btxNo0wwVr!@`ObW$U~!4pG@I zK-oIX#X73=!N_N0xeB|uVc2dQMyP5SfL);G9pdq^ZZ+1f%Vh(yWHW4l-28ja2F~9q z$p)l(;S#Wc#NyBd4biPapWVs^l8~QSY@l(cFk1wP{;(VTCoNeL{LgyH6#rMCUSc2o z)%7X(pRrsY6-ymvhp_O60Ym9XlwpML3`$e#v4c{ShKACnC{KxEs0<}N$?k7h7Jfm+ zNnc;LJ{SD5WHaEWh?t&(|I7bL68zE}aM8e@c*c@UoyY(*tsW~+j&vgdu&e~YI+g&k z%^?oJIoA&XWlf$|1~J)i>LKB(F4=v4Jjjd*fK1n)<94UgKNKGp_Z}l`Hl-w z-*Ey&+e1Ep-#-*~rjHV^1`>0jt}K=dsCtY7$GWqALp~#rB%5Ib6cyKVMo{{FNk$;e z4VQosBopKeJUWyC>h~o{zB-iv8X#>*0E>?Q%nPRYhYjE|#s2iUYf`LNQI#CH_00$| zZ?TC28xJ|d-KF!KPEl*&k$M^1wezdB!W)k zo%`~CgYQjOH`e9G93=q&{Kku4fk;(dnvIgGK$aP5E@F${AtgUMPq-GDR8L#20k z)JMARhk9gPvoV(y$db*l0t!g)IV*VYDoIu#%@3Ds+Hb(3!+C$^MsXi;6WO_csqmjn z|1<4P^ZzUVpwQ1{wp}b-o8tXb+~r7<1^0b7W?>M))!&N)&(&ju$HS4IF~|0d&@{r( z3PXm3_oib)TF@X1SS}3k8?Vj9yDZrZ-YNG)&+$I~vMD$9OEbMi!#jbhdA26y+z~#) z*#s}4+*-#H0Ljc@GeL2n zeg1z(QSftVw~G_kC20KgbIc1URZYZW2lK0;+m`2X0L}2j*b2CZ)V4@-X_?G7Y-HUq zx#?m>Zfh1Sa{=<$3|^OuMp?2MG*YmA&!^+_|F}(VI+kWh^Pn*e`Ph|>p(5c$6v!E# ztg^8QiijCb4DLzXT|GpG?9f#{S*wnf?nusyZl8%AI3~KhB}-PjFaU97!-6LwAQA*n zHMk}UH?zQlwLxBy{t%65-h&mb& z_~B^hxTXo~y%vMTK#%V+-@=N3AsXI8eGT38jL3=%R65eHyFQo4$db+Q7|P|*a~`vH z!vk_WMw-#i!(#}<3v+!TBqjyF1jisJw?nm2UM*vSrOC0u9mHQmxQ(07T>OjfZQ)2A zuK_acB!Z%H;i9YpGQ^uZA=UrAsHiXEk^ODydtmy?8xj`u_FBVck#EPQ7TA6aWg~}s zuWuolw~1J16i6_b9%(_OY3%wNa#@fp*$fM!sOFxtpz}W| z$%3T$@FKII=HS2CB2E)Cq7k@=sGsM#DgG~K``?YJ4xnnnf6qiZ5-(tWggRTH7cqGM zxN!8f49zg10&s0|@?j2=WunZEYiYWvX}+i18e9QxxG@)wvSc%Gq%>GPhvR!cCkaPs zp0vntYz@6_f*+a68VmR?WPujS$GQ{*GDH@UAGA>3TVMZwkD}n`1G!y%=B89FprQ1g zj%uRJ=WxsOL)(g6nBv?By1K}SNnH>iGzQg4VXwn+6LLKe$|6gPISQ8AH{O&BP+77W zfKrOOo&)s#UzP-@G?$tOpfu<0s*Io6Tdj9}UF=j;f1{Yb_QYDE?hbg*yh%J3sgFD9zIrr5IrMVcw>%y(L{&-(gsu=HY)GK&R*C4;m71udniQgMhqz%0~8hH%=5X6 z&V?F7M*>j~3#Ayw7*99#$ksxXCnLv58ERq&Eosy*^m5ea+J`nfZ{E22A( zRnC--B189?CJ~wOg9to0<;8|=+;VGfPsozZK$Ma__Z*_VahJi#>%(g-u!TzuSh9>U0wUoM@j29!fIK4>!NrT4G)h$JYbInyG8l2$JbEoc+`7D+m2q5omA|J|V|_-VJj z`Sk4xQa*dPi>x=8w>XyIaRdW6JXCB{yn~Bw#69@VBR>`k0in>=9gQI%-iVCIj2z}g zh84To(CxWMktLf!%I$cQo+IV(QAwmovx|97LS~yP-ue)>xfGF$`xS-#uS^}CykEp8!b9TI zmd7S0lg_7MHBufswLdx}OiY!@le|iQP`6BC&2>Wg`qPqQUz!25AN%teN9G;*^qPxf zyPNk>3z69buWQU;qFhE2);%xz&a8QJt{j)hzBk`1G$4|&i0*ZfbZ4Vdhxo=z-1`D9 zXU2K0tB%cI8ORRq)c$u~DPE_XN00P;o8Gu7T^bLu&<_mCfs6XNk?y;&)fv7Mg`VS~ zIl!@6#aM7XYPdMF`=vSBE*d&j0+*+Q0VMz+|5F6y~#@!r>V%Poyz zgo`|vAzwkjk>SMm7z&yBq+viJGvwoUW7mM2F)|s%HnI-K5bjV6JVGYoID&#g?>PTP z);yM->&l1!^ZqDUEvSc$HX!E7uK=1tvNe(2< zhUeixv!?1!>}PX^@3-;aoo}(xwB(-?Tuq+BEuzLk+dnt)e8FFEK=~f)+yAi_R?u;9*OuI;$&$^0atm7B^L5MN8zn(0&6yS%l(f!iXiuBq zcX`_X2=U#5EHgguz7*s0{o5tSr!+|C8J~-5uMzm~%<4v~<5?wcNpfs4xY1Q)c}mpk zbh?euoV|0&gk+L%J0!limN=oJ?7cM(uifK~db&IoC)$)UJv{06E{%cb@0NsQeU|Q; zFSUoHtg5JiKcduPgF3>)N;Rs@v*;uBa}vlm-|Dj^oUfhxm|($;>1^qYfiy;FjeRNY ze!_5ur0n*+x_+-=lr(Ld__z@aJ~-CUm3`b=T!cGSt)K(UdGG! zY@5D&Yq}&pfC_pjn-%&n1R!Y7jy;(3Q2iOAzUv_f$U})+2i7YOwn0ZnWRQmn(Uec# z4GqfAYu>XpSNfJEn~`p|qHB6CeZTjbBr8RgU=LA_I_hjgDT*2y4?@PIvVLhgpsiM&CLqt@}D93rA13OpF!}D zroDamo@B6o{yNzXz5AZr!758OGg$9g&|syWNa}x|RMeCBtG_K-4@_^oH|2Ljj)hF( zR)EYxnrX%%A{u>*$8N}3s3Th0urw`jVh@GJC@3JJ9H=NSiWp@cXuiwbt@q{*Az8AS zA#^XgqUZeXb5HJ8kXAsUQiVLEbaE@hWio(eC8HY zN9t@76AvLB5cOG_r6>N z$db(<;J!Hcyj36!sLuD8;`bkmMx&m=PY?ACHa4B4OyYN>1fDlZ^8Hk zAv-{MOJv^m9GJ+F3g3@n#!=%I#alQux*A8$EiHqW`_U=ApD{K|;zgQi^awA*3&P7m zY^B|mttA4^2MRJvK9utp$oaE&mvHRGHxP&<{?eO`H6`nVN5^SFF(Jaynk7li&#`B{a2KBU{lZAQW{-yP9(r2dVdSAb=}K4I3E`u@+*M z1-(DqeXZDuL)7>{3R0;1!%&MNBV#?In2s6#uU)3VtrlcJZv8qIryQHw=)7%jHfC z{~x3cV#u#$0p32s(&4@WbGh!h$lB>4Ufl?g7XVSk$kOGaGN|D=mYt7gNwOI<+vumB zqq*cAmP4~NOY0QPZI{dKEP{)JV?yJ+PRxzzFpr@S4mwsdw0kQ-tP^p*r~Q`V|4$SJ zKOgb!;(9056WkQJx@*Q9dHn*&)CjFNkWj?XkfscIMv$)n;lYj$ft(pOLW7YP*h3;S zN7Fo=!vY$?W|7NiWXWb2je`%;b4K%?G^3H_ww*GXE~iZz?Tex(XgB{$@w4{TgJ-!Q z$-IAm{ofO&H$CXq*Z)*O5RMiMiFh1`^UyUR%b;!Q6U>beJ`zY0 z;OdY&2RwkQg8t6C!dq!0McwwmIKi2+#Fb0R{rYx z`xYif^hHU}C~jN85&SOL7Gbd)SbIRF>;!3Em05g(Ncd|9#?;QN;X_+R)xhvk4T z&HZG7Pv|{Xo*dyKQ!?C>t9)Xz^iXYdygT$a=n-%_(j#UWNrMzIftIGF_^&8xg&x!2 zmbeF|uiBP!gKJzri~)HM>3;&pN3wMIC`L9CaoRERm{=$+V4(h7RfZLSXHWL~`6fnk9D64H7+I19J4+j*&7S>y!;-3DL;x zk_}`@HQJQoe^gONiGF`u(jJ&z`#_5R>pTzfPBD+;5N3XODg)r~LNYCGXaSS}&`%=V z4+?`(!w<%C=$B@J-J?J8;cTFv zDD61SCRw#p_-856!0ZzKSyD}nO7Z`SqP{{8=x@F2f$8GM5@zt$t4Ne=2Y@b|6&+@} zVThu{s8h_4=OBVpkf*^C0UrhwTS0*p6EO@S9Eg02@zBBFarnT;a+!fF*$gxI82YB? z%-}1hPRcO@X^uD#Gf0z-q}g8x`J!6!p`>PevLc|8e8*1m(r#8gReEHMd=IkhfTrlQ z#>!~rG|!SN6X7|mh~2g##s7IlJ&yw(4T%d|FW86We4ysWaX(+61;&-qG zJJ17^D220<@3RmKT`z(N03CuAA!h)>m5?yNWSaJ|2XncAEZGbfco1FGb1pFcu~Twf zK$;aU0vDjgH*tX-r}@|kUY+Qa1yB~<-si{{8rUp|4a<(0XQ03t0-g;9w?UgR4;CEHfwrD1-TM!IJQrcIWHSi+ zIDKc5*?L*$Z~SvVEQc^@HZu>xXy`61;HR%_mYeQ1VR!-h(deX}0Yfm|_u^ ztRwffuj{A`5MTovusvwrFbQ)bhJ%o+>BztXM zWCPN?a0%Ezl5dfO@C2*FbuQ4B2^`^LY?SY+jMPTQL{3E^)7NrHhy-Z~*#-~klusoM zY+@#h%mLbWKq~(~si;qWq|x8&mz%zGd%|tr-tC!4R%ju9$AEp`(XqJaF%J2j9tRwz zKFbMBZo8rFAu)mFQU)Y=1Htx>GD$Hc`%i4o-s0_lmV}{&h!S)@w_?IP{ z!T%0)LeKI4mBN?g@Gs2-yT?DFygmNm`PU)-+tMq4`GJRuUE;qj(Hfm0_e2r@uRgBe zr@t*$4@}>+Ge!SC1SfgS)Qw2z&@9-fX zF5%yn=u9j7_^0k&>}~q4U3L6Nh%@#OzKB47Cv;rW)KmI$+lFl)<#w?gz$o?|OAioV z6v5`lL*Mh^{EH=lAMtqSu3Y@flFi_M7kZ%Q`2XA;KQ1@>%kox<40ArhoT(VH1=-&U zVCM5sSqwGJqsT$h?-(V_{1^q!nPs}WcIV<>mTU(9yV2;Lug$}$BxduGX)S>#rAvd%rM_~kE0VPjuJGP-PL{Q+Ky}1lQmTZP0?7bum;q))b zF$8I@D9aGYK;IJocmClouIqF$kia%es@wR#d}?LUus;5C(Yp8tr#C)S$G@W+NNer{ z4pRJSrfs-LVZ>Y>z|TO3M;`?N@faCM5b^Jtj%OPNk|Nor?OO;24t;;`L%H~uC7Z$j zLze>oYybS$_^Nm?0v3oGD-x>oDg-w;BthsNO~5#kPnanloT?+h<|HW73@Gs2-yT^Z`Y@3b$-+OIer})n&)h+(>sg)IKSRennXkGk+(;N5K z@vp(zm>Wn}h$2f+1H{M!g!JUdP7b$3Al`s>5IRF7|Albi>Nb3ok)a$F(<~%1WroG~ z?a#%(EZGeH_g@P99}a$34*$|juzUQ|ooCY!o)OL__%PKeD=RmB-=m%4Kc7^$_|K=7 zG%UscYDHbW_+$RVMR9uK;RK=YJm*klLNkO$4%Ii|qvrsDK7(f-vUZd6j)8(jj;$eU zx2{pr9#hv)H#hJsWGuCiUwHq+xd@ddn?dNqmja<*QT~e@LZ#VMrwDDEPm>a1CsBHd zGe616hv4?xgjQ1JHQag+R-$rzCiM!YX6y(jE7ge_dD*dMO5#Ar5dqouBvcl~6*xpmJ2WWsaX~vS^Kl6qu{;yP2TDV;1+w_YEQs93Q1&8$jrHK&^ z2m_r3_Z$yZxltk2wcC;aO18F_`epm?1a_Sv$yiS7#27A32ZpAep|NQ39?y2oeura}_HHxo$ft}*pQOQ@-0lqJ+jA}J1A2!aasL6|I5;1rC*u=@& zb{v%Eew_lJd|cBC#m87M!Y4%8va_`T9G|2E`y(N@ALiAV3J;Hkyg4{mV4sxeVk%Cs zFX+s=NjMZGLO~nwFQh3AhSHJS-$FaAY-!6F%To^r=@SwMj7~Iid z2yhG!rb6T#=crz0BUafnAa_`S#*sWQM&(k@j0g^cs4B!Y-ACrYP}entG9EaX%Y$Ue zW_Zv+bXm`N(EGo4yBrUaX2kRGpjGIV9Z{5~@gPRCw~8EK(FF2>ht*haUdV?h{I&yplTp$oNI_Q2qIK5 zMcJf7kLDs@mTU(3kD|kRj{Ng~CW(A$_9u;eB5?FfG{(<#4t~KN+Sdn}fBJ8Fc#8ke zE9&Pj@zC!-KJ9@Erw*q);1GNRP#MIIP0aog0*Gw#6|lesbk}p-02#zl2n{v|5BWm_ z59N#zUKqGA8p6TQ@C@zI!?`>_mTZOx97e-?&I85|ZIYY)r5T|#51>hZC-M<>&H;qD zAj1J>9^J?P2afMtIQ7^J{x#b~9U%A!8J32E0Remf5ipDd!Eg?s0>Qdz@Yv-L6Oa;6 zgCr0j&q!oNdH|0h%Q^g5KK>=iX7K;mrNIC5Pu?kqe`zMTDEQxVnvHdifAZ*&68{PR z;?aHle*p1+q>g_``jmtW`GMf}Z^VcZFf_xDC@rWK!u;sCu?+)+Ws*Ar*8WWQLk}uJ zI0{85Dquf$Bp3g(WHb0baw+itb$7EQ{$-iqqTv4^t9G#hphrvNUpzX+|0+eLWy9sX zO-~+8VE4}RKB|4XaI`lAT5G#nh#Ec6y2q%)h0NO_Lia7tK~!(#`AEwfxhUQN`J2T^ z5{Rf>haEYZ3p-h|8Q2}g*Xp_0{obdx$iYsU_sj!3Qk*Z$+)h_EJMGY%xN!HMCVl*3 z`Tt}aw21cqEa{ndP4T~8QP<=3`x|;-`qZ(M5gf7*kPAi-m@xU1yp8&s$heQFT}a)! z8zVd~vaKM%f}F!w7a4-#3KcD8w=x>G*hnz#Nq0Vo z&48TJv-KS0U;mlAOk`o#yAD znWi|De!x%>S1o{-3DhpCJa%j~zXLn?KTUxXeUZE(UXZ z5TM$I>A~p_B{_6Ev{0bOM zncx!OKbc&cp)for{#%@VoAE!3gqZe&HlBZN!w?!EZA|+?N1lHRCOxag_4vOl6qU>g zm-jYZeKHmNpMiGGqV(G)v;aX6`wl8{W9e^TmG6QVgvi^A$N&v_JY5^PePZ7>GzYnS zOvYX0?cq;6nadMo$!2&0rEKpxPx#z#Y?YGH2@8qHe@H^tYr3rcXYVa)N!(5{HfseLE=uf;dF7 zE{FsO33LM2aV!>-UJ%uKHLwbp=-`B>GiD-M0MYym$^D;vDxVWblFe{}r_k=6bAqq` zmLw;TW`|3_36eRu4iC|%ooE2k8-*kgb}9)pK-!Q5!o`sUvZU)TnBu>tsI@-$d)5Qf zE1ypI%iAkW7I{eV;o4D*{GbRVa^dlV#DE-rzfs`EaOaOh&q53UGKPf)mLvuegRwZY zp?QqW;Hjr``HL*s41an0Qt+4UKYFj6q#?_@JI)W>Bon0dLF)W~R0_xn+TbTNx3}RB z?WxcJ?T8<=CzUp+kALdZUT@PYKUv2=a(kiLCu9W@8!QwFLM8z$C6G(NVDR*ZS06K& zhuondoO*EFwEPI!K)7KsN*o?N{mESX%aYCD|C8vUp34_sR_>F-zcdrz&X4tyE~7c8&j5QeysxNJ%O|6~cdC{-@6E(KfyEnL7UAxW{dH_3JEhjnFVX z=o~cz4!?n?M$nw{gnl?n~duJPYWitvvH_3=-A+Usq4<+FAC!&%5hwlP>9 zku^w124LHRqXFCrk$A`nJyZ)cH5)}ibStvFFm`p{u}Kq&93!sf$NDqR=Hg$LYzF_& zqKA5p|MPz$iGOJ(D1-kb>t^%%A2A!E4;z8L1WA!Zq{W=yEdRGsp>f(Z{#!{2{Lj?? z>CgYvwLROWS3XzAzhzL0&miP(ggWFt)PRA(;aD6&2^d4(hmir*fDz`$T~rFz0tYF9 z5!{cgA8`=6zWwZTx%ihQo5BBc=$fA6|I5cUx%pq33CiG~?roj_b-#1`w^E@2+BN=L zNfG|hpmhG9P}B(`-rsuA1JkRAQ}kbHAz_$qBESGn#ZCyXLgs6>AA1p2_A%7_Ho^?x zTM*MKz_EM?Y&O({o@;2Z7Fdq|+;BenCCO&cKa5W4Ir_gaV9TLjngz~-{?<5qa>{GY ztcAQQG`mwkf#^-7`6%sUW2c%EiwPS5NeF4y;HGpxS#q7_{K?+kzA*{@mA_Ny=d#)^ zE_*&f;kyb#!Lh!!tjOf9Pc=7Ch08;+UI$R~P>&0yH*RsLvF64+?V0cgLKP2a z=D0und@c%Q$!1XaJU&m)QK)TnBvB~KisnJ#!ku9o={1vdza$TfTzyTmGR=w&t6Yoc zZx&NC@NV||sYAYF*Pm?TW(X~c_fM93T2Gm90>+&{w<8cd{RY2VgUw}H^4MN5VM?ECid z4=7Ea3F`PaJ;SzqJ8&Xc;X=kZ3i8Ix%|G-7a_xnAivKl=N`}D8eVZN+Q%3Mqh{D`~rTHku=O8%9vyf>MYPSfXK}g5}83BrU z2O)qubafsQZZV%3u3=a()}eT>VFmC9&tU|zWHXE)#24$iAn@hCkYohX+;9mPL1WsT zlM%E81I>~H>aglA5U2stq%li^|IB-)>;J!36#QIX+r?s(g8fyL-SRvH{GxUboND3O z?+3`^&yie*ixORM#c^SDb07}{9un>tQ4m{Hy3fT5!Q@UD<$_(7YzFMnr2zZk4WE}& z07!GZMFTsr#HcgOSoT)!cy7jG7{QrN7f}hK8 zn|_U_Jm9p8GQ4qw#F?%K=UWZg^i4fPu`0$O1t0_vN&q-N8MihAzyL;odz<7p*M*w0%pGZgkkH8EuHjtmzG2sqXJ)|~+9yrU>`A``ks zWFtU&f7?R(@4!T`08$g_upI`r?!p})m;K0+&9I;NQm~)#N55H41d(O!^Th#?!w!ml zJW=K^%`OQD2Fh=ldQ&;C|oNCkt+ds)i6PUHR zyh+0*AUQmdG_p+7Y;q6PJ=<9uIdz0L+4FB`Jf`Ko;ke94B5`B+He6F4qTTJ-<>P#@ z`zXc#A1MlcKC;`z>rSS;;V#!j0w09fFpkPP3=tv*M-2@f0XPnp_B!Hid>+Cl&q1jT z76ujt<%Jd|Q*J=XbR@aK9UxKOM7uk&>x>&LyjN2E|DK}Y=OedWER|E-d%hkAoTGFe zl5@bt$BL1}CqlVi1IvA!^012M9snP~qUV?T zj{EWB-ztZDY3A2C?nz8YE9$jFK1svrC0%2G6=-ydeFC2(_BVq>`7YXR!LBpxch)m0 z{=cLs`1#Om7x%r8;QG@mp_QY0csdkwkz-j%k8kP_=neSK+Eg4*6IHr{7}-4$JL5vZ z2U{NUbcHs>)3~hsLN2al$!2i<0)56_&*xuw_1ooeEzR88)W43Bcr*9mnEA){3v>%=ZJs*?SC(acxkq`c!(zxz|r!FO09|l zzojQRJKf@7Bvu$j>SW>@Ep~z|BPol^7Fy~vdv^;P%5791|M&t+yiMOQR>wbriflIy zk>5jyVUi)smy2wkR%p5?I}#(QAFL9AMJd0`7_q?)s(+ili|W3N6d9H_I+lxnS+W`Y zkD)_)j{oQX{8!}gFUtwY&Fk18q&ef@1| zdtiF)c*+H?^H45OXC^9&!krK5PB{I!@GtOO+k?W>MZr;*c{*JFG^F`6k@ts#JHWbu zoIlWs>QQApmkY>}&2WKn^hfWxz_usFx+1K|QI0dyo3>{Hh%4rCHzNp`Mo7M=AlYPL_68ULqa9!_};b zVu9;_@?uyV=;t;n#s5A<-G>MDx2}6&+ODP`M{WQt?2!+^U=&B_z}*m~JR%K R;H za{zy%C_o+oAF4nk`Hj5D4r2JITM=T3f=GL@nhSDSvKf$9@v1!s`B#4Ix8y)B&FSU= zIcd;ZGInHDYQoj{tmw;`*g@ghzrR`;D;;EGS)8}G@iiL^a>>$Q>?jDZ^Tww5-=?VB z=xP0J;XN?DYa&Jbt2*lSKzHv%E-HbeUMST2o{uB|6iyiX0YYHmJB-QR00%&wk670n zW+Or&jAHmK1yOY(7xA)WGl-`Qk3C2H_-B7x4)M}#Pa5$=Ad7ek8`#ChCs}!{v}2T4 z%SZ+%`nD0%vq0eIQKvyD2mCy$d2LSde?Ui>!b&! zcR=!+TBE9Ozv4o6=eny!q2st7k^{Jbf$|er+#`Rt?x85KW1w=VhVWe^4MR>r-GGb$ zEdX*2+AJ_9Yq@xrC7Z!Jg$ec?@9*9ApXKl_&GeQ8@1#lFS*?Y<^dvhoHcPp`z}bHu zc+Ro%pMRy2M?2f<6#u%S>Ya@I%lO*q1E&+vzq1y~JWV%|1q88!u;3$q2t4;-#tS1C zIYSUij65MI=!LwXsOH7=7=?lno&dKWju>P=(rc%4K`%{~yupyvrd%02R2way#$P=L zy>jlq%7I>*^GSoA2xKRD0^QL`ewvq#R8H{fEPeh0QJ+UTPdb-^f8Il8wZA|AFoaRx*GU+F5&)45rjIgz9_a?4 z_|s0ml*fI7U%-q zk3_;ISVLq)@ccosk?`d+xrmn~n?d{;yi?B+UwZWGa)_5^d(w!fSvHG!%0;xhdSB@w zWDVt|9kmlR`1f_a`pUKlU~Wdh*H zi#*|u@5iXx?P{hCj$pcA1)+spq9()vPqzZp0CobOYJEW;80pX^oOvaeH^`FB@P=3L zzCGs+UsnD?jyFg%NNL_cVtAG}kiPB7N+s%649KHN!%>bK zY%bzu$z~9L7BAa##J_jvU&3u06O*NfYNO*_%Kv$!X-LWe zKac9H2B-LcOi>@BC-yh#9+=+xsgxlc=9umg*k@=SVt!F8AfRGG9FBpA>NjEbb6m~F z0>OrlziuM!348*>IC9_<5Hs=~I{T?yh9FBe!w^1&SME7O_`>%8SB@b_bHz>>0x8qd zbYh|y=h+%n%-PFMSE}VnK1?|VlbHXJGGCw*;6g~!cQ z9fq0$&>KX?r_SYKU6yPH>y+fZ=UD&x`M;LKx-`R+!Fn={&c-^~>yDk|rQOi}mmV1_ zpExb?TF^JIK}@iMcqZu>~CH4!1T6PQ!X$9Qx$yqH4Te;Sp47$ z9C6=u1Iy(U*UK@*bIkR&=KGqXo0iFSD~@gE8ju4}Nz^^}YAzR$C7a;_ui_r{nVdq1cAUFvGfn&rl4np4-gh*UyAPE?PfP=_+^|f65 z%aYCD|24co&+-5D@BUjk{7W-I8T=XQj=?A{h$KH)>${GLv>vV= zn_v}=0|fvg1ewL-*Oc?Q96*+Ch69|(8~2<8j353xISwGr24y%v;!Qi71CWDGlMb-+ zH9$MUG$^}f0PTooTbttFR#cmw*WVV@1Je&psf74K;hjPhILO!q9e;oX7ms6N$b8&vRkeW11Gghb3@PLEJ*+a2r8Dp@rODrgC1H%7wfv;SA)b@Mb-S{Nb-j zLSCBhEd}IfNzok({R@R>Cl0{PLPb0Jvs#|wf3KqMUC_wyUzFPe)6P^OVF+(OZ=zzT zufx?JY5ZW_vnX+A98ypa^7e9w{5o>{dXNZg_zWWvK)_>D_hQ40Z0G#>^12fp-@>q%KiMzst0eeW{z{>r%VoQ`a%x!u^LLj;w(EI3 z^x?M4Q3h1=TuRJ|YM`8lp$>~4gFg@Kb*KyO1Pl|p6~;*A192ahB4Hourd-H{xh&xf z%rDT8qj9&?)k4eX>rYBy{+1-nrTN_w!n~cFoLwd_nt(q`0AC`|FZ9LI`Tq(eR%njf!{|h zGDfbxL7-evrgPbXEa41Wn8qjT4O^Ho8YFDt{5MIm1!?|x3D^SRWAPMxvzQDWm=)_M zJ^#031>$8={I6D23Jkogx9RHCiWL4&p*$cw^HK4a>Co2!-5$Jky~vLu$2YMo(2&gw znL1IChr2ic!@iFMe;f`z$l@P+wlb|;%!R)!;SBsQ;*0eL{xd!U3H%R#K@$Gbd~gZj zPhwWyGN0hlIsDsEn`MhzB2l2-^Tpfr@lV|#HsF7tj(^k>wa7uoH&B$zMIB7Y{aEqC znvWblNYsTjLFi(I0P{U^d%|beH4)Pn2Qf#fLf;IPi^`k1_?IP|!T+0Rd2jGP<2jhF z3ce_be`!Xzg!pggJwwB!bNsiX)~lsJyXT9yN%23RsHFY4e7EV^)XEh6&v`l&0w(5u zLx)npM_x}Pl(&5cZn~%&%mdi$9m|9!&_e7EoP0yn50ByJg9U?w3OS6C#j+|I1&J1iv&FTte`ZyuEmne>-Y%bfn7`SzfMut(ck?| znLahOD&+(Z$FSoePoD*3!+j66`k?)HQH(FNkm=7xjUHI~z!5@*Re)o{ypNDy%KO7? z4`F|%qr9cOoy!Si31>LL+xTw1;RG|$gQ*tarzJUoG)r7UPLM>-&i42rkaR8vw4+AL z^CT8$iSXa>3aIWKgt=? zq&Z4^AkhNyf3)`=0FqU8zW=#ZxR_8B7312DARxh;1B%WJK}H7{hY^I)9d4W{>FFLj zfPksc8JA?v5p!0|iV4@85VP+4yX*hyJ~z4RKG(Ny@a_Bj&aLjLzFj>XPEVsy<#3(p zuKLxv=bm5s{*vA!=G~q;K03X7Y`QO0HvJ%ey;J5~?UyZ?G!u9sdM(|ZdlUK zw4W74)omAhw{NiKN><#Yv9-*{CyMENL3u}gtJ~6Sw4B_L>+R@>HXqErf9QMHY#yVSDh8j9_+D#k`ntH7ZIFD{%aN9LqO8B2Hyj~?8UR5-ww4r{IfeLH<3*>sY$cMS@5_Xmh$(yfEZr3?dX~-sB+9DoNRR(- zQHnpeh1b3lx_9ky{pP?f8LbB=$JFE%G~>);nsSh5m&9?F+CK7tDZD;|W68&jihvos zzURATl?Fcb`iZ(mT|1ELO$j%+{#rJE&AI+(TYqn6aTtrfgT?id;ikEslKlP-y$6Bk zn*zp=^_%1lh6I?qJ3apYno|7vqkir4x`*``|H6bkz0yxX@qGGjF*py%4wOD1cZazf zur?Jr`01qfGye65{a-W2Hw=x*!#fl-DWGtJPE0z}VLIfjO6!hp5I&%1y zBOzAh>wEhDUvF>uzjyDM3Wv~=vy5|>+&V@h>19{ApaD&HQ#0IRn-WxgD zkjIP#%_2N@H+JRJ@jaKFF)_Y-B@o$wK6%>Yr6@<^?6sjS#X1|OM=zcOne~Mkw#zDY z8KL*{NK$72=Wu^{latueNQ(s9*l|7eV)c@N`rnjrL;t^oL$>Dn|JOe^l&2mO z#u+-8fMWU;V@6%Z4iss^6ydVKcyfcJstP}@(%kdtL>6RmE(J?rYBQ3^4zuZn<*C-w z^I}bXT6v*PxLQZ|nb!vZ)Rsa`n15O5*;0s#8DZO3O?B^4pVOB-*>oq>i`0t;o>No8 zjdS|qRXV4T`IJ~8mh~jXpW$m2cL4c!#Ekq)5o?qY(eOLAT{_z23=FDlKZvOnpcQCN z_b<07nIq~-(i2Sud{PVP1|CYmA%9mdQ7;{Mu1pCx&ecn~|FxHYm;Sd|IKo&7n8-g7 zA_M#mEf<$B{}!Sq=W@mJZy};0lV|dOhx{*FyqYig|6Viv58Cf$P8db0D+2+T`U&1h zff3!n{3swSNnugtkyV_=loGj&4PjnLoYPID%t=^s)l1dO2J*it;RgS|j15?G{vSD4 zz3}wujrw~fHa0DZ=ub^AF4vRir<3hY%Ex!xx|>JsxYVsY&P*qFO|R#@#zN49|HT9! z#Q$PBEua4vq6V>*^Z!Cb^%8sWzajt27O&qR%npk@@%$*MNKBG=NbV*vmpJ5q+1rEgg8z@4;eRi# zeA||3Um5KJ=^0YwVU)WOx;>U6!X)>diY~z+z$p1Z^cBfNzoI0;cQQM3L-h*v%7OfE zO1Q!QuVf$Aoc~8!Ei?W%7J?@HFP|O6|6+76U;i&eP0r_v`F|mzdWp73hcjERl5e=Y z{h#gs)BczCKem6_{-5oiwtw9IUi;?u*V)NkuzodO# z`_J3YZ9l7hS$nEI-hOJkY+uyg-afDWg!bn4W7`|sr?lg?*FLFzT>Gf@gWLCS-@AQy z`!4N2XZ*IN5 z^{UoOTQ6u`)p}m**{#c4)2%(Ni(6GIZ9Tblers#%tk#*WQ(KR2B`v@8h}OefN4G{= z4`|(|b&u9vTZgt<*6*#~S--LV$@)9%7uH`}e`)=|y2bj2^=0ex)~BtHTOYFCXT8&U zv-LXb71oQatF7l*&$6CwO<1E=WnE-lXl=93w$8LRT92|q%eEda+XGHI*1b3X_t~zb zUVWx3GnGHxy(iZjyAr+UtbVL3;Q2?plK1(Mu5?)Xa91K#AL>eA z?t@){sc-1Y1NuN$9>n{*VywNdE05;AT@ldl>5A%jcbAKLm$cw|r!?Yuhjin3yHw@B zO$u_~dO4T3$e8apKb^~)F6DClC0yPpQQbF8ae4hDm)A{jdF|7d+bMF~9wn#kk%Ew%EGI6J0~gDA zi{!ZZa@ssOW{#XP`;J^rk~4Oq9I+GRgdHyj?BR00j+5hctR+8uo+aOBtvks4fA{vC z+U?fw;r@SV{X^?-T7M1if3Nke)>m4ehx0$$`atVlt+&AUuWY@zbxmt8Tz^?>vbDRl z6Q1ARdSdIG)+RXqlvdPoTF1lh4{qJBbwukhxZSe;)B3IT&+z)st)E&yvc3bSf64ls z^-1f)@cBEeH(9T>UT(e6y2^U4^-Swh>uJ`-R%z|9F0jtE&axhBZLl6`1=b_1pf3GX2{JUK_=HKbcIlrZQ6xW-(a?-!mm81U6?g-a!bmg#ry(_2vYuyKN z{c2YZ{a3nj>c8BTWB;YDock|!<=}t5D<|=$t{nZ(4TbF*2wRaG7D=q~=hlDi8|&V; zwwkZqQdP9okhxez?B$j37IsKuUO#3?F1>b`nnx009FwtIxjEf;BEQHz`fnzIU&Ixy z_|z-as|H%lri2?-^Q+jlHMg3#{NB)NHg+9ErA*TAzAS3*R@6yHPS65{Gor%=7SGEG ztWD_ED&LW%>|e!lvif8#b63F+R9}kVob<$H-;yRaIG0ECvhG@7_6;Jims9gp`~SDU z{@?ejrOY*l$Oujc#f%*F&4GUl`>*bte{koPc6}1U*Q)lmoJwi zBv!a2ea}7qe+JH%{CjO*&sW|1_q6>KJ@le1%PEWVMTJW<&D3_t3#6@9QTd@wpIzy- z1zV-QXD1msgGCyGrQ2Q^;=H|Dy=EXEni6jC;cM8UHRr=y{{2p77@4s!T7(b9-Q3?D zNZimZ<%AT!rPSL_>c%N_Xj& zWtQ1KgE-OfZl0!0;|kSl)N2RJ6+^-ex$;`}e$C~|$X)MjCRdDw>mqVR*kfr1u^?f6 z)ii>RNhw3W80y3058xpzgCoTpo9gdSwroeCv1%cxTL>*j{V(W+G>8w}k{rDwaxc+tQ_?mOQ`jsiyn~J`J#r0x_%oZ>mJa!+*N&5kZRTNlv zk@G=P4=f9*EW|K~wH_P@}HXw)ymw_OJDMWon~d{;VQ;71Mx$n=~g zyC#Vk?@^Lv6DPK0-V%e43)%&FMHQ>psn-u=e^bH@_J94Y!T!H~!d=XizOf8kF8eRy z;7ZV8bpb>RiVN`b3fBU` zTrIOx7vL?f=+h}k94$CRRhMMU#CAr@ZIHOwXAEB&mW+mH5QnW^uih|_aZL#~821fq z)SBz|Jxs1DrPMVLlFZcNBXP~RKSI45Czars?6`J)65L-pte*QNscLRI; z|F5^V{yz`ynF@ylu}ufA#7%t%grBDV0__uD#DYkoqR8`#8~}mH2>~7L)MF}G zWiuWytRnRW^~Qk`$CPkG;=J+JkT@5;(oo_Ui{0f)oE54~&E@-r$?EI>;5& zCXUgOtPDyA4<#YsVUr;2sW+>j0dniH?$cJP9qbTGjZ)Sq?kXXen(xe-ZYR4O$j%+@J(_s);b5^ z*H1R&LSs?1%3LS_*lbjBi9G<95fqq1{*R^q-$oLTSIYg*Ke@ZE=A`R(F$*hjbaVX( zm%OYpp!c}6rI$rX%^N*5OJN$nNNiftBo$tKnc24My~flyHNS-h69t z(nw{^<}a5@$kk2X2QZl$Hkji+ETD`2+X?tMasL**&?(y1C!a z#te_BF`q7cM5Sn9{#>Vd&roczLAfS_&ZRtz6*Tr+@GA~v;Asi8U+?rai(8PK)Njmo z?-PJa(^zTuzU8{0H2}WKth>;?*38c>F5Ky_Bfssja__r8h~y;;3w;Mp`K z+&G(WVdZOdHumcm_BJ${25lJrquPW2R@yIkHkicAFrVQq(%qI(~ zh#*85z|v_%2Qc!VJ!+nCo+Rm+iG@X`-b&5y^qpC0s%PNLw!yLu2{(@6+xUhxI)++f zc6OIL)gJ3`GZAU5Cl)q8WtH;_bmqw+G1K;YD9h-Bx)>$s8j8q4OY7nPL;jcTJyTT-n1Nq;SaD)Hf&StDR|Bw8YDgT=aL1X?Gd%TbT<)3}f{s97u^Z36(Ne-u(iPRvd zmp0^o+1G>og8z@1;s3Iz(j@f@KlJHw9w(UqpG+byT{J%q5<`U|5|!O33yUfuJu%5> z=j{+C%otHrChG0#9RvB_lyHOp-@%@&IsZTBoO_t*e`6tN%>QC0^zpy^bA|lhpd@G1 zjQ<-1_0op?FZ+6sU-192GyES@VBq93-iR0?=|*L0#|3H1F`d3?#~%Zr!sXN~kpvm~ ze(c407-aHBh9rTQsCTG$4&;AR!VUg^CwsEy{D1gYk1*qZVD`uJb|xkCPL zP?9rh#{Uh1dTB%cmwi3RFZlns8UD{?7Fv-9xogKhASv0*B>j-O>@u=hJQr;Lc|#Is>mafiWm)0 z26+}q@81%(XD9$Ef3e5x07jHMfv?`B-aU~2O$j&n|K04#n)Cm!|JsoMjfJ2v#d5kD z&GXvlW+>>nge&C#1|>P8X8hkE*w6pFl(kE~VE8&%JFxd&-Q#;Q;DJug%pX5b81_L+ zGdJ;N7*JNYLCPpECZ&~xN6U&%L$oZ2s7lUkFQjCdWJD*gq9jo7R__@o1561wWWama zj5U`5_VoSDYye|HxQGmxn>5lC0B6?|ASu`#pU~lgv(kx+N48GW32Ah^DG3CE>q}|r z4VY|_>_{HS;OM{j@cNHkxdC2b{#ujTE)-p6!L=rQ$#7sSjFFG$ia z;*d`NZr}(P`v7-Qna9#=#6hpSDP2a2tgLc3OJiCSgh`gWv?o#TQSTkd)TV?RO#NPN zam|_f`mGNzV`^i`wg^)T*UU-aYqF7M!a9HG)LrQW!B<8V$(uG#Oy}ju#&m3~oM`g5 zX3v$8mV;=f?nP=Wy5OPxFB^T3U)cXA&e;FJ`9+*C)+=&KpB94jDR7*^k0PXl9lFf_ z0|u1AKvI-Q3lq%#40HgwLLma?fO@Zb-#{^8O1L2=-p8J-xtRFbL%#bWqgi0avT~Kg zgfPBNky9 zOc6C9+~E87vjJ<)_v`QVLo>cN7JiHH{W5HR>Ck^}o=%LGQ&S_48{d_VO-_!FH7$cB zTK@9At7G{uu&f^c2g;H*@x#~Bc3|&gyN{?D`|1|XIU)PB0>$&_Ek=$WP4PS@rAU|g zy`=3$c=r@JRdglKX-5_X3F*)k4a`CO$|zLtS05P2*rtRVjQs&_ZOs|``j7v_jIoWS z+aip;xN$Bl*fgW1DB|pNiV39Y$kuV1oQ@O8S(%m}L|zV|nW`T|b>O0h_P=cQL4INX zGmE8rSgUo_Q4v`@AwB>N$nr1^GLKY%gtY#`O)>i^odEQAX;$Xh(ya#!FbaaK%BzsE z2cesU>I3SAf#SlHa6??&z@DtRxY+ZL|HDjN7|YC65*KrgY!k^aOG(b?Y6yi{g8laY z7_+SoG7}8RvjcnY)^&Q4;PAlpDjIu`2kca~6QrKc;F1KBfU)2%g*-m91~3xH6UbaY zWkzTf0~=60o%%9O+*UWJ4-S+Bri2@k;DfAt%_YHe{9l+!0%LKwh$IjVdqB`#N;I$x zrqj`L$0w$C@-JS$drZ?zSfV2!)Jzf0s}`)TGI$ z47Glf;@njZ?SI+fgZ#q&_h#&WK!SokrZg)Jq`PSp(NfTf!pMznrwU6Te+N&Xbn={` zW`3D^MQkTIKthG%KyyO%A@$*bLcx@9LnwThJy~<1F!BpSp3}k9k!VRYW2)DQ9Ont+j{nAX+ z8%wrTVrtQx`(yvoerMzK<(H3a?(Ez(by=CNH2Pn&`h(o_Xc9o{ zv{ZdWeRQDEFeTg&8Xsj_)?8@Z^7+3r6B@>na}l8-oV9%a|B&@7^#AW74?$lBP6Ju- zYI^*C7i9_Y+xKhl)4E~JoL8U6Fv8FyRv-9O=g{~w_h@7oI+0Jc4)eltH=xukVys>j zyGh`9g!6K*N*Nu-^q?Fw{iEt*1DVs5aDzEN#v@x3=IqSc4t1xv7yXkVgBnYzg&9=0 zN|SKchCO>mFMHxxF-|&eXMAk3o>aU<$W3#bcH1FPe^USJz`WLxkM*Mh!c+R8?Kd6t zkLDQmx}nQFv{0nQK%QT{8yxC?c^GT<-E zGFpLqBm-771|bWGAL&GbvLq(xV!V#ss(3FWT#L;~sOL z=LJRSL}lgBAH0a9(uaON8CCxo<-K7=8XyKl88V(M&!hl2laimLq5*&_?fzr+arKFT z%xy}z!Q7v?HJIBz>>tdS+gQRaY&&RnS{(ZqR_xdE^6%5e$40hx@{32y$)=URLaM?k z3zz^aq}W_%kN=1LzhtWq)(iYU=?R9T!oa4ym)svxegZ;+31vV&tv%v2vgwD)PoffU zg0ZB99j0+bxsXExpu!FS06+oMC)6hg3I?#PrC^ zXUKTO^bnVJew6!qm9#<59ouzi$Qy)oKXyX^KJ`g;<3N@+CEQ@?8@abNXX%^#UzzE8 zW4X3UEG-8AeivS5Yo9f7(#V$asdQ?5ONFDxlr@%%E3Qzxd7*Q1{&yiN-UAUyJYs1?dm8*`zw z3paCV@kvL(ln!9L2V}s@)u+^_2eP&);Rb7e`qp6WpS|s0%~;!5zAeJqi+lg_j(+c7 z{*4JRwQFQ^S?pYKAAlxVVH7hnphV;s2j9WRn;y8A!KCM18P&AklZit4@uqSJ-_4l0g zTQkvMEFV`%G|W2L2K_&4-kqf+XLL1$!7Rc4{+~nqe-%Uk`2VAOY<^xG&;lrDY6pm3 zRZ_&CSIj|zi>o4`m1jlbfFnFlf3GUZ69#}VQJhGCj7Or?Z>T<_K0A=jO$j&H{Ij

-b*seYbNy>!sKi(4u{eRivHGARzKW4`N4_(_~ zHYx+Z>HbgtupQA9v?AZn3n}^|6(FjB5cvn9B&JX}p&tl-1l>Mp98jcz`mFlgKoMa| zxFI4w$L_4TC9vheN50so6Ogf-Ttq|&L#<){pQP!py8lmp(a`^2E#AlfUq9pjI}AK7 zDB7v2qKxB2k_Ktr6~|>*27c&zg>BQ~8@Hcdq^VOd1jtG4q|7Rt&Ok+>KBsOP$l9ia z8?1ekoGLL0*Rt~O`jPvZv9_^%TZFa6@IG)aK;g!w|G#xS>x_+@KOxlsO^{EvcnQUS zO|pE-Q2940%B{^==+OU{O+Lsk{QnIz{(mJ_e^mM82-DsxqPEWqsQ>jcpESS9_bUe< zwxZo1nSe9`4%{eY1{k$~1PDC4bkt4i^8-bLDdC1__&j^E=AuD;$WSyG%g0p`4YN+R zWa=78gIP*)M$J5dS{Td{)JyB}|FHkBZ11vP^#9%1W9$iu^4qDO()^cJzd56MV;g8c zpwSQf&PY!OgCJn?F=;=kEp_=vCBqIot~ITS6cL6Gv-{$9a+X4rO#v?Avi!#y<6O zLF75=FVq(Xa+E3I21k8?eOq&m+ViZ3m~oV`SXl{2?btcdnVi~jUZ=ctvQy8akUaU( z^s@2ksdM*~`Sb?QnsU0BKph&?SO?Z9v%P|huNk>B(Y{Af!#QH@Vfev_Az zBSkrtc1CqUPyJUdd9**?|31CBx=XiM#@h!c=G=>4_4DSA$@*J6qQmRYNa5ke>Cua~ zmb=FD*j+e^`Tn3|kt9{cw{=Yye{;cZj`48=$Uh~*> zCo>X*@HROwl(73D^DOClLGgP^;R`MB6N=gS6Q-``I$<37X;>tQ&3`#fGHpBe)fd&5 z2J)CG;RcU=N%noMd2G*bk1^vhV?nbpkI7yP2Cn*&umB z2UP2I_wPL^{>aqh%ggG@;Lh02(Q;{i#UvW9Pm!sBB9ocnE_D z%{c}8w@3cg_r%e%^%7;nM?nnxoOiv6leE;XzhJU{qY&8sl1Eyvy5rNlLh4Glf4_##-D6o z+J6yvNHcQt=+O^A7&%44h+k&>d_{eApb9r7+)&|P$N`P`pOla9wx>lv)FQp^$8{zHbf(wyLt{&3VI{uM!LOGqWq9)0TC&e& zap8>pJhw5O=y9C9AK`*l(m$BbfrE&N0Nqb)v=j!M?1)slLTknuX{CR(5J_3lT+Oox ztSC#>Bgi#Eqeqx_LI<(tnff z@62BM)WILQ0Wf=WLjyp~H3Nk?>+>lmweN|qz5%e`uslcxfZV|%27qvsHUL&d|JPsG zum9yI%h3P&?X~{Dzp_M>3}1aaaNUL7O+5qP*p$-yBByn?6DO4H#eN=DZtgS6*YN@x zf8-$@y;HJ!J%^ZupM{P~voHGhq+uLXfvvu(zBbSRFeThD0KUfh*W3Vj&IQjhH2{q5 zAUVsEQ{xk5QFrRpl0_b7?H)8NUwQXR3;_9Dttfj2fF?AL1($9Bh*WAA0NVLofaJ;y zfCum;od?S0L2~(1xja}d{P%zOPaVmB^Y#C%+Nw>4x&K@h{V$TZv;QKSIZzw?k?Vhb zhxO5sERZ1pFbe}|E)^eNV@wBC|Ld=k%k)9g|8g5^s{iXR?3WbslV#|C{q~{$XVV70 z(EpE{(f>ASymajF(Cj1ur4q@bTL-=Qe11_R6@z(5>CM8Zr0&Pdv&>CNDvE6~6-!2X zM(S(o>jU+_DdC3x|2lWF=KBAl`##%L{~Mb@B3_rS|K;SbME}<;C*Rf6|C-P|7F@di z7lG8!|5`>ZKyqdJUq*j-BrB>TKy~Nda=DLO`0sZ$|BowcoPP~p2YUzh9@c$)FEDsF zpcagd7@y<^k9ve^J7IzEt`BIpb?)rTefw;+Y@cWuk=gNSka;PRIKI7oZfkmXqQWnRiOGV@F9!$x>8Rk((!SiO}~+F2jH`KPZ_8Q2BOAYM6Y1LcA#;f7rJmK;Diz-w7~aKj^?YbqCv#pA+qVW|u(Si^otLqx$K-(bG} zm(R@+E1FQh{+AyZ>VHi_zS~0i5A{DAv3f7`|JgJ8pE`yNlbwPgihzCedf}#-SB9w< zNojvhS8#Zoc|R4!Mq~nF{8Q)XRBptN2qdU)shbD#xhdfWpWn>wuQ{Kqj~ny3vAA29 z&zG%F z0xkgp+sR!D?<*&YJevJ7LsW7G{UW1d2z~#_Buo=#*tk*Pdm%Ia3Xu-#+v=8qd~Qm( z!RNQ!8hpO~E5>|oEbbQO^QAEBTsK=X4d&?ono;M<0L{%yfDbjXRnq^17dF)Y%Vuq( z51{{_Fr)wJ^W{e#eZQj0OPw6l-^s!>w@EP0=?WC2gbrx{MikMr3lS1QMdv_=CMOI4 zuku*kqP{awE|?N-$c69R8ggOFGxwV6e`E2uuv}Ow1B=F9iT^L3o1_0Vp?>`@e>c?s znuO^8mU^A||F7e(e^9Th9??CwW|kLkjqRM7nO@GgOk!YumeMKT4}+Yc8iAMDDN*;d zjHsteJi2HmWyBACI_T4sE01WOtG=tgH;`FO2{)MKd$NO83;n-!M*mYHoHM|O%D#l= z-+@c3gPdW0iNl27DkGtv{^|{RvbwxW=*i5-$&tR=Pl@2vkgt6Ftns2#Nx|gw?y^fP~?JEwOLr6~B855myteDhAkY}Wqj8Al?${hzLrgfN{imj!rUni#PNS6Mx z9Xd}=|8@WR=j_K^)S98Uy>b>=4(TzY<)&gkjfK`OyE3l{Ph;J(&`o3gwQ(!D{kdeM z7D;Mp=G;72KL$ zdfWo4UW*lZ@3M&Zom9>L--p1Oi+~$m@#m%@z*u#!wg?ccSF_CONdR%%W>3?Oh1h7R zBA`iapbVJ3i#};!AOmJk+k&gVO$dRxTdVWq2R^rza;i~tA&ETJQcJ6<*(Y50|8J@L ztM-{(=DhYjy!*u3guUiWtkNX&Vp`baq()4nWLk|A2TZJX>0(U_YsOO~SwR<^kU^-q z9fXXrmdOig;k15H>{!VMGl2i(D$OR`(8zS>lh1v&A+2rnjPU|^QV68d8j z5S<7aAsmEZL7OyE1pj*X8Up#zUV_>yvv2=AXuG$SSoOT@5YTpxt-JC))N^ zCv>;gd0yo9>AK* zF7?7|Ol6m`2sgIE8jRU5H}r<;rk)FnuDBN`eZ}D)w3+w)kgyntljq& z_76V5{?zzC6X@m7AJuE$sonE?EPgo+qAM>Zx)%6x7RNsSCV37v#tcca3e%;cpKqBa zY0m#_5^Wf(RV77{M!xMuKEm%ugIU~=aD&BvbZfBq4U^ZJvbeFNGiLGHI+(>;L~HfK z2JVtxan~8Mzo`^lZT6qXYfDk{%@1j;bdC@);QslXwE!_$#T*oQ zQ5H2){SE5PJ+N(oE$*g{QSBSslkF2)|EqO<>%!K(tZ!NSto7>O)rZw?eyP;yhjb6K zY|>SkSI4lb!b6+JMA;mE!i<|N31e5L*Lx8iF`R;+8y&oT>RJNae-B5MbeHHsHI>|@ z>t!yZJCY3L9J+Z-n{T(u(kLK7hCblTySCxWC@5Vm$@-%HB!{F+J@MQ+GQ zox&r`=LfjLk>_Pu&Z4A!c!pV@(4m7$o7hdteiRe|J-#xT=EGQ@st8$^A2G}?_p7iZ70+?i%j(4-RZ7>zW5^Pp zpk9f`Zj-#n{7&xPbv?p*nPm3R;xi%w7(~8XI$r6M(x1A1xv5?Z`2aE!ym06O8l^1D zDF9jWlsD0TB+A2t0YYN5ums1;y@dS@ODYGwJfNINy{$*au&m%)^*u$I?`QNjNk~O@ z5?Y-kffq1=JS(xBf`HT{Mi=t?m^!79M#tD%MM)>G%uX29V*g1^h3QAcW0`8oQ^;|_ ziZY3y9F-Bd4-7hXN*l9{kJzE(QPn`nBh}PEOeI-UZuG1fDmKT8tqL<96KPXCX67j7 z1@@cdQ`;${7^;Bp&I_uuEAi+&J9T&vE=kvy^r(0_gBdgMlZwa6{sms_MU-^Yqd8=% zOziSOpNGUSRt|ANbyn(SG3Pi=*^DRE+iO}1{`H!SkKLJaUNe$C?JC3*b;{f>15SG` zp>1kFL4YEf$+&?F02JmWx!@&z=qlgwGxgT`o*ds{T3jmq>Db^c(3D^CKT^?34p5jm@R#uaVd|->J=H!@bN>acBdl*)d#%T)->DC(r?ziA z{gC#J>I}=tnMvpR0iQ~3N>D+iDvzYpG^|7J6{i}cFH1v?MwONu1F8KZV`+v7aPfQN`-@H5tW>D0^0d-J_^AB$|9m`U&zYSC@tvQ=K;Y{p6g0;Z;?SU=bsW>jx2e4g422G1^OX4 z62m87CnO8X@R_d1F4rgcRx% z!<46)^KY~OqY9~DM4YCT&j8~XBqh(~IQXSg-nCu~GCZX@B1w-t-oU9sJMcm>9%-hK zKshXFnC_RX%JY!*VVFmDm^)SAxi*PUm3mQcF_cF!NVT%L4|1QYBnD~DY%h*oG)p8Y zR0XbG;jx`0P35J0gI@(SWrRV}C)SH`e3C6uAxV~|yrc3nG(m)rW^Z`pG7$;so#>~X z=Z@_3IEsH|3hk2Jq+%z)xW4^YbkTdIbCRF7@fQv#1LBQ@p{E%!E&2OR1vPeA5;HI%3GDi zK0Px$H;Wm*12_j87P2oH333G@g09#pk~=9b)DMBTlo4!zOf0CMqj$EGamvw8%;(`r z0-u+0%Jbv|qf;?lJRhCR!ce`uSG~O^6VcF%8GL*;rZAUNm3Xn#vK=%A`;V@x&=^so-c;Wco(3A)16@q2%rDBa$T{>l99|T6uu0cJ*(^C913O1PirWC6091q}iaCmHZRGXzs~ z1sfRT7)Ui0Kel83qoROE#B6cc6hjKzhe|^yvRRI#5sJlvA`fz&1@t4fm{z8)oBe6M zF>XfsQRbvrIkY85EMsd>nLy*yihP@leg!-nmjTAeec>}^nYs*6!d{7eKZw*dH5ufR z=U^N8Xf8ycq-CW%$B_N;*gx(u7kR*;XA5&g2LqRo`~b6JCEDs0^*z~z&*wc(1q_Ky z%p*pi!>eUh;^(2qu|bP6qnrXOkg6_YiUoX}1TnbMU)Fc>R97GVS+q=W)wQ;;kUj$4xU zhVjJ7NN72e6BAWE@}+yvml%)zB#XQj&z z@d5|a7sU^Nfc=ZEjMbGjnI!g6EGZHO0|M`(EQv3K8P+D#d@0|>5kko&QAQww=M3X= zWHN#f8PkBXOzrKFK>$UV39cI>RAUB#qu@Q8z#!oZbe$F+*zzGvwFG~J6Tto=DzPgmBuep$oQmxVX<>+zMM7eZVCrU!K(jM9RIjXe zm~f+y@s8<|z94jB##ehwo~7;eCk_0s~G>gPpPQ^B%E>K4e{ z6pd_Cyo6e#hS_DDF@~%NT2TnL%aBJK*(TnYXc!JH%@F*%)VG8U#h?UcB+vi_sX-GI z)IWwohHb*bA~O^b&t4X!hhi&o^09$1Hewf_!i&`>dSr-SBVX89kYO>b9o~oWl4TJ_ zjaWxA1x6Xz5{rV4z@?e_lQH1c%+~r;{uow zJ&}3%j2yR|$yO!n%{ZNLHeOS~|I0n9zTg9aqdBRRe<5zTRlK1b-5K-u7zH6dlV|62 zppGkMXjOOup?{L5;$jN$?0rAQh9UBv_eLyJ2E+z*X#8Jl= z3w^BBD{B5fR`mY~t-oo#u~pmuqW}LxeMF6^V@3b>EC8|8;Kdv(%L@faPZPY1iZT^^ zQP_(%j!0IQ;D?t~ zOyml-G&}6vv!;So4=>{b20V5=KrDH72@euB^$i-nFDh#Ic-%?qJ-McWZ1T}t zn1AR;tSyXN96;**QPr5oZq6yed5Qzz6HZ9#axSHD0G#cmo-?HnxsTJuP9g>a-cUqQ zC`K*}#{t8dM`b&1ifRHdVHjJPdYrrj90lV)%;{sMK3p#b9gIJngYRU(smNJa1q`4- z$|5512QWf>OaUBu?jj5E7SJhZ8jOP|3I41{h5HTIKN_J$fG1)wa6>T~G|WH}C}{*N zK`i1U@C3>tu~P&WtH#6Olpj%3;gFSdRzNtR3K{r zk`l8W9}^n_W9(`5#+c4w`ARKB(DB0Qzyz4=A=1J{gMm(YY!6eR;0H$3!l>BDv4a&! z%8ZPI7NUdp5`9tt*p={C>J&NO86cnw zvic>zaDiYCC@d)k0CY+`9y{BJMJYf8kD6Nt zT0y@!h(W->7WkA!wk)VK|69G?f zGB{nOdVh}$Swv_Q8WMsRVjA$QiE=PbNcI5-9TyEhou>)R;UF7u1@J5pMhKFyiqsox zGNuy`zFLzpEq(ZI{h&ZU#LUlAoJ21uWvNIt3@acCR3Bhe1Y+tVo*2OfgpvWVpk9#- zi6CG{_NX`jhE!RiT$$uRkrlWfvt08;JWIGY1F9q>1k2(1;iQptm1C%ZN|yk0_QPr_ z0s$d>F$P1l7P<`J1LG43gv$%0fv690sPJE16dUcL`86&hd|~tJGO9NQV4JBCc(0Uq zgQ(#NBV0j6L7w5|j5ZNAdJyM0LuH677|Yl=P)d5jz<{tOuQT9E><82mCKkdSEED$@ z9~@j3YZRl_4Mnh$k%y2#jiM!Rw~(Vhf&W9vz+dmsS>0TlfI zm|FiUb*+U+P9i7sORTIE*a$2ZbCuu_E;Q;My@^u9I4d!$BZ_W#TsCqM3pa?t%yNC_ zFf9-oUOm}tC2;NVbTSJhMqV}vqzRN8Hb^QD`7 z$Jmk;hJd(v1r98b6EKC3a|saXp)EnN1Y=0b-8Gevay=dxvXSn6_;@yJatj0sP#x|1 zLNFYR3&};0nk9lJHjBbm!N5cu7f+v|0;Y$Y!dw)KGe&`610k&dfv^-@jC!0s3|(wF zL;}u%AQq%u0)jyPoEa*Z8Js3h<0ClN0BJ?Yu@tb(CBOn)2+$nbXbemb2*h9-7;^ym zi1CZ+hh#c>{R=f2)5_;tdSsZMNSY~j9#9y*Fm^d^O-7a$8K__+C?iqEcpdoKsNRwt z!6?N8!s$*j^~ri`z#6dn@ijRfV%&qWl$fp3b`hY&#cKl70C~n+%YE&w)6x}W0GN}< z7@b&C@qp(6PJpi1SKum;SU|WCS3M|8WGjF~24au#iLu4-7luPJIhtdcA&CR`!SyY< zxPaKiIRFiCh8U_IGp9v$!t~(25~dQj09P2B4_pFp@nCCB&HncLgWPSq8`$>1|k8n3cea?2K=RmfFdaY(;HfePNd;Sw&=r9qoG`*S4(+QFG3(h2#36wU_9j%H z)jLSI4jIZ?LDoT|1XHA+rHg}K5uO7N4$;T~A+uP303f~u?4J+dY2}#V-ujvf3YxD5 zdyaAdxxh_GnL>O(OiVxAl> zYQ)!*9*rMH3N?WYv^+Q$#||8qGv_fk;FOw*>BH2wL>QC9jDkZ&q8{)B$lC^{73Ub^ zlwAZT1#bpN@&OKE1oRkwCBP<0kU^+^QZFV)V1TfrB|uCFkumOw5EX*1)Im=1Q}E!3 zmw*9aC5fZ!lM5I{;Mv&sMNK8)q<}l2C%`&5fv_o-q6|;RF!Ax*eYQTr3==>e(3Q|A zk!8dr&=8)%$@ML9{4u;-LQ9yNl!3x62sD}6?hr%b{9wPy$8bqw@FS!;=SE^90G5OV z)yr!#csPh%2{94KfUAf#0Jfv3Q380JIVQa;fmv`;%q4(7(PyBj97f`Zp}L{>un9m4 z+6R~sQ;l5!K63<~1V4DQDHLSv%pflY8h&}9OD@l3=3W&eP5Fb8n)OQ3Wojnw;U zGN%2oZq~OhfoJ?+U_Viifa^HyKpY@hG=9Ova{=hUBV@^NAcmjKiKA{f;^_h(rRUaE zFwVfM5wKEuFVR>ByHNr!Y)Y;M1ryOxfwJRm;#^lA5&8iFWBMDk5l-l-pwUGX6fCzcvQ1ET}f3`~LS2FRg}K)%sAY)Ot| z4h{#Ti>Zl;C3azKtLyY)VvrdzgK=LN`;Kxz-0%cs(kWRH;@Su%2=qhV2r&Y%HcU$( zB$$Xah{(QNF9y&MnJbY&Oci_@L?`+yL0t239z|Y)(}0hneFT!NFMp%-l!TRM45MdeV3hPR=hNvOt22Y0H0>%-f)M1*^IkUc{$Oc5I3h+gwjvz!nKnegn+^d4Fa_7Gu+l?90U}?B3gP*4KxLFLE}Y<< z5fBG1e@#VN3d${rZ(PZuaUXDJ1vx{GVGa>MLYU!f@h*WE5Hmy;@v=O;hWxU6dy!#0 zWh%X50@XN9(uN$g3ad0BTPLT9F>zM(nkx<=QE5TEi3#9POV^S@y|i8oK|4?iIl?%+ zcsv+$K$37dY=w1;hQqLeQ^h|MPn!T9;Z|Y_g#3us+^b%UxXCDWFnt6UWg4hgXAtFl&}3DWII_XPih|y`f%=7%9NbK$vnwDN@6fgTqN>Vte@uZ|9JQ zg2YLt4FC*5S0n~57TILU-D)btf+)=*4#Lv|r}eS+$v)?7BKF7w;h6*fCs@J`&&&bk z1l%MhE0IBfJp0m`3QJ|*NleE}C8m#x5(t{J7Q#0iEt)}LgJ3^{pcBUhPe9z@rHQ*u z2*j!JKb9+j4kyF&Cjcrz6>K>|DdgAyfy78@j8edC;xl}UY@>u;#Y83=6~yQD@c)_k z-`}?0)Y?w|-_5i7e_S1_T@6W>16aWv3AjJ(SQbp|mK=M;CP5cqSTrM+5ouk_#~`4K zlV2gRXmN#Uj+G9!{4ss+tRz77Fd_?8NmPnJjQHV%EnzRTF+Mji4*6sZ+2FtvLjcq# z(1XO6q{HOVH5IfC=m)nAlqKm?gy@iZ;MBQ9JRD9PdA=2bSuj6peNcRo_yw1<;L3SQ zO@&xEP#fq=CR&-$nAnW)d@5bIP+lNHqUOj(!0N=VjM1lLTon#rAjZ$VdsL8$o}f8I zU-91foF|eDO(G94CcOliOs_w7LOS{pl`jJ10+uGur|)gl$6PWVF%V%2BKTmJ9BKBM z^NTev`DkRtaf~W}Dcmqpn2AkDN+AF%hybRTdRM&|@>V$ioEESJHU&^Qi9z$>DYhb@ z2#RV%LZxl=euy7|dXRZR8w=KIlD2H1;y$iTpD%0jQY5_XcE0 zfp0jKA_YL{MYr&Hu@BYN^UF2fKu~n~61q4kXzNm>-D) zwuBxhmj`vapm*0b@z@lbS8)l68UO&ig%`3ne;}u&)mmheZfiN+=01i2s9;@7h8D*a))(9iS3eVp#a9z*O$kE- z8=#Vq{VWXqL=7+2GAqY{tNEY!f1Ur=AO92dkLW-6pMC}yS)4gcC^F}PJxE@H`AgC^ zkyzN^q$CjmMgL&{!*JxmV51Tf24)aIx=huZYci(Y@V}{VTtkNh1IZZ_cb<*o_>s;5 z@&ne1h{wc1>kBcd0Eg~SFUGX4`3Lo4Q2po|$yg%wjvM4)6K5mM z1=heD!1^OXvudbLlQ#(5CqmFvy|)svGI7= z5?<#I0TVIkB&!fD$2;y^Q-K#UykbH_*co7(AYhW=f{r5Yg5f1ZK9x*9SDaRK#4^1~!6&dqz%dggPM?fDmv}Z!p0fk`E|pB>_$R2+~y~ z?ZC$L!?-Ci#R%E~6ob&%m`d2WSQx;iU^&=F)#h3L|Kw~0@Isvdyieo*efd9<9*GN0 zVv8I&oLL?cHWR@YyzfeSTN6c=;g?{vsR%HD71)vVBurj>R`nNp#ry;ejKEb&I*SBd zupK$$2ngh|&Yfqqte&%i?N4A{(ro~Lusr}{+~Q99A(WV(*wGHUtHAk@6kW73y8x&Q zIu{dx#}2{O#z4UP1rsCMO@teWfHww}`f|M((otbs&LQY84*`G-<(6X13+OK<9ElrP zp1|@fnj`|3HS$*&T@oz9C+NkHPC}v#f>GRcv;^mX01$@}cv3>2wDzLsmzV@-LHr4+ zPay7(ohWv6?yE29#Q+I{%oByer$QH#>;do47Qo{ml07VC1hyP5El^>(82y6HNv>nz zO3wTJ>cwP)hk;wrTyk=VF%ZQf?1Px5dzud@NwkVw91arr4Zs#zrDz%adlV-~iR#v5 z&`U#FF0)YY8AX`D= z4nX+?6T!pSQnL4u(pZsXUxT@an6IFGlEE0(+soO^c1!9pjXh~p3I`MIg2dr|S zFhnp6219Z2mjk&{d0u2{_Ff7s(*C+Pv1HovS=RE0nU(a^dVLuQ5Dc# zTzv3wGO|do4FPn-Cj<}SM1ivqr;*_|V|vXT129({ChnHFYKdVRqRa5YP-z56U6BtW z@?m;{7O)~lG(jt%EOD|z^_F@ucphZtAc@7hBG5z{I^qZS1Xmp%!fyjECg#Ob9dVcC z#1Km*zREeOyc0APa0&b&(%z*CLDEN4^fAAe2oTj5B66{}k(*fU97yTV!2#r_>oa0iGb{Lcqj#swujiMINtZ^Q2{W3 zZAsKcfl0;;h7hIz8S}vXcs|66h!o>zp$dq|K^wV`(*a0N7L~oFM@4G9i1WsTe6htP z{}Rbh#2*EWIFgh}@u*-{1QhWcV1KBhYbzp1e({WYV}L*;*$)&9{6$AqIrFrk#05dZ zV&qG<8Cf{kouD%sxCEOc7AJX%#IHYHlflyh$`>32aftPZA{Qo-gh)@Kc(@Q!MS_6? z9rC{LH-U@D=gCPbb3OGELFFxElTJ@3HAKoLh|F<-FrF}M2>D=c2sa`jh_`TZxniXX z2VkvBB4)B(^#2{LudBA`|KnS~XuY9zKJovri_brN{W0x;K31GVd|dJ{(U(Av;FS2C z*h3s;%pxl5FnB1W;b0SS0&61(o zk0z9uE)tNaZ|Ux=7ef{`u^ZGh@jlT&xQVz57$boFER4^Avr4cRw*qgLG!HU%@WQc0 zh``0_g*6$Xb3}_sam6ynK*MSU$OFid&=)pAg&stDke?*6A4)C2CPTy?&;+neS*hpr z$N>N9oIJv3$Z8}I&?ymXkSU2XliMgF4N*;0Mh9=iiq(&dZqTKpK3|jJO_HLCTM4X# zi~(3736a~uLSiq$)4}KjWCo-_LP$8Bj50yDq(Zk)&y_|7hpB`HVb;CcA8TLSen{(w zt!r8vtbgN&@3m6vNXE}RcOV~Fch`>xh=T|gPgD|dWzHQDH6#%jIVqxSpQuR$pOkKK z5JA`FOklS~U>X2^-ZWob=Xpwz5CRx!MTR9u3=s&F&R;Npv88my8_*So0ryHCKEM+~ z3}l0iv9Sec$6(cIy#TqAHY_n^ys!-Ck;4uqD5)Hgn8cXsc&2;}K{Q+uNw>lB!1RN4 zP*cx_(mnLuVc~=g4{#`Vi`;6CjKa~7ECH4tZXKZ2HIL|T!g~$P^z6*2ke8?p9JFq=`1;( zJZo}cD5}J6U_u1J1!}XTR0nBGRy*(khYBbI=$W{MdJm-P%yTk`G03q$kkaef0nFx8ZBJ1H&Eo!`=bw z-dYnF+yBzIL`h$z;cK`9!yUNY?SSzAA?^38_CL0N-2MW88NP-)Fx-JZ&>cA9kVDmB z4>;_wBV__bIWf6&Vtjhf5$)a&PcJ%C2HX z{I;1Gk$UIK( zd5pII)rqS0Z0noq@70N|@dIp4_vE7vJ@UBYT34PpB?F#%eN35nGle-Z_g~i0r*1uc z!?x2$wr$wB`Sg)>bAP*TL9*+LI_27Z`rhj*!8O_Cjd$3`UHPns=!de`>0k1OQ#Ws&W};Z{ z(9HbPdf>o?&;0s8YftmhGsY)&PmiXJB^+~%tbd|2mUs4~qs?2;znv&2$EPRqvhhi| zamd!w+y?w^PdTx>Gdam!H>vhM*;~+-bYikB8mFON)_w}x%BN2addmdcq<8-8C!Tfs z)=j6*K9-rypO%>cQ&SV2Y zm2OM3O;hFW8Obp7&$|y?@62xxv}8?uuvA+>-W}I{+?@|SGK#D#J^kU{zMcMbl;Bu8 zx?|xV-fiG2=YDX7Yb{c~Hk~X7--%w**@GUA=Kn+5f2G?0-u`v_m;7b;8t%Yw2W}%f zaOfd-P?mM4Lk~H$Eq}RVyM1U&|K$#?J6Ly6ce>LZCI9cx))A8bH^57xcHH7L69)|#@NOEN)6^^7*(KUh} z(jI~OEb@%$PQ#=@UNRlDBZ)vA85h!#Y=Dl;4C~0;mMf-38tk0tOirCyj^fnpIIT0e z2g_}{{8mp;!jZv8ni_fR^zJ=+l z9zWLExoc|vr{!xmj55pGu4gXu&))p(fdiYH^GvpjJEP}NFL*y48QRp5;c6W{&{+gv zM}{tSgjjFB^Fp^QBpk9G&HJ*j&^Jq_)ZTa5dE{aF2E;50N-!i{NYKVi2II1Nn% zQoWFg5_ArwFg3Yd9+5ror}|)v-g@YU&6{hP))}kDYaJrLyh49dQBLM(>%S2apVBkB zr^hDCsT1`-MXHT<#xCAeboiUnQ;f%_c9jz|Kaek&>g+Cesy*L%*T=VAy0IQR2EsNi zf~c5Gcc!E3ckWI*qbKF#yKVi`J?R7s-=HzuGt{~PB2-WHGj c@ScV{u=oy${vY!H;`=mw$8ZO3i#zcD00{ttjsO4v literal 0 HcmV?d00001 diff --git a/test/gen-server/seed.ts b/test/gen-server/seed.ts new file mode 100644 index 00000000..ef8a8b63 --- /dev/null +++ b/test/gen-server/seed.ts @@ -0,0 +1,640 @@ +/** + * + * Can run standalone as: + * ts-node test/gen-server/seed.ts serve + * By default, uses a landing.db database in current directory. + * Can prefix with database overrides, e.g. + * TYPEORM_DATABASE=:memory: + * TYPEORM_DATABASE=/tmp/test.db + * To connect to a postgres database, change ormconfig.env, or add a bunch of variables: + * export TYPEORM_CONNECTION=postgres + * export TYPEORM_HOST=localhost + * export TYPEORM_DATABASE=landing + * export TYPEORM_USERNAME=development + * export TYPEORM_PASSWORD=***** + * + * To just set up the database (migrate and add seed data), and then stop immediately, do: + * ts-node test/gen-server/seed.ts init + * To apply all migrations to the db, do: + * ts-node test/gen-server/seed.ts migrate + * To revert the last migration: + * ts-node test/gen-server/seed.ts revert + * + */ + +import {addPath} from 'app-module-path'; +import {IHookCallbackContext} from 'mocha'; +import * as path from 'path'; +import {Connection, createConnection, getConnectionManager, Repository} from 'typeorm'; + +if (require.main === module) { + addPath(path.dirname(path.dirname(__dirname))); +} + +import {AclRuleDoc, AclRuleOrg, AclRuleWs} from "app/gen-server/entity/AclRule"; +import {BillingAccount} from "app/gen-server/entity/BillingAccount"; +import {Document} from "app/gen-server/entity/Document"; +import {Group} from "app/gen-server/entity/Group"; +import {Login} from "app/gen-server/entity/Login"; +import {Organization} from "app/gen-server/entity/Organization"; +import {Product, synchronizeProducts} from "app/gen-server/entity/Product"; +import {User} from "app/gen-server/entity/User"; +import {Workspace} from "app/gen-server/entity/Workspace"; +import {EXAMPLE_WORKSPACE_NAME} from 'app/gen-server/lib/HomeDBManager'; +import {Permissions} from 'app/gen-server/lib/Permissions'; +import {runMigrations, undoLastMigration, updateDb} from 'app/server/lib/dbUtils'; +import {FlexServer} from 'app/server/lib/FlexServer'; +import * as fse from 'fs-extra'; + +const ACCESS_GROUPS = ['owners', 'editors', 'viewers', 'guests', 'members']; + +export const exampleOrgs = [ + { + name: 'NASA', + domain: 'nasa', + workspaces: [ + { + name: 'Horizon', + docs: ['Jupiter', 'Pluto', 'Beyond'] + }, + { + name: 'Rovers', + docs: ['Curiosity', 'Apathy'] + } + ] + }, + { + name: 'Primately', + domain: 'pr', + workspaces: [ + { + name: 'Fruit', + docs: ['Bananas', 'Apples'] + }, + { + name: 'Trees', + docs: ['Tall', 'Short'] + } + ] + }, + { + name: 'Flightless', + domain: 'fly', + workspaces: [ + { + name: 'Media', + docs: ['Australia', 'Antartic'] + } + ] + }, + { + name: 'Abyss', + domain: 'deep', + workspaces: [ + { + name: 'Deep', + docs: ['Unfathomable'] + } + ] + }, + { + name: 'Chimpyland', + workspaces: [ + { + name: 'Private', + docs: ['Timesheets', 'Appointments'] + }, + { + name: 'Public', + docs: [] + } + ] + }, + { + name: 'Kiwiland', + workspaces: [] + }, + { + name: 'EmptyWsOrg', + domain: 'blanky', + workspaces: [ + { + name: 'Vacuum', + docs: [] + } + ] + }, + { + name: 'EmptyOrg', + domain: 'blankiest', + workspaces: [] + }, + { + name: 'Fish', + domain: 'fish', + workspaces: [ + { + name: 'Big', + docs: [ + 'Shark' + ] + }, + { + name: 'Small', + docs: [ + 'Anchovy', + 'Herring' + ] + } + ] + }, + { + name: 'Supportland', + workspaces: [ + { + name: EXAMPLE_WORKSPACE_NAME, + docs: ['Hello World', 'Sample Example'] + }, + ] + }, + { + name: 'Shiny', + domain: 'shiny', + host: 'www.shiny-grist.io', + workspaces: [ + { + name: 'Tailor Made', + docs: ['Suits', 'Shoes'] + } + ] + }, + { + name: 'FreeTeam', + domain: 'freeteam', + product: 'teamFree', + workspaces: [ + { + name: 'FreeTeamWs', + docs: [], + } + ] + }, +]; + + +const exampleUsers: {[user: string]: {[org: string]: string}} = { + Chimpy: { + FreeTeam: 'owners', + Chimpyland: 'owners', + NASA: 'owners', + Primately: 'guests', + Fruit: 'viewers', + Flightless: 'guests', + Media: 'guests', + Antartic: 'viewers', + EmptyOrg: 'editors', + EmptyWsOrg: 'editors', + Fish: 'owners' + }, + Kiwi: { + Kiwiland: 'owners', + Flightless: 'editors', + Primately: 'viewers', + Fish: 'editors' + }, + Charon: { + NASA: 'guests', + Horizon: 'guests', + Pluto: 'viewers', + Chimpyland: 'viewers', + Fish: 'viewers', + Abyss: 'owners', + }, + // User support@ owns a workspace "Examples & Templates" in its personal org. It can be shared + // with everyone@ to let all users see it (this is not done here to avoid impacting all tests). + Support: { Supportland: 'owners' }, +}; + +interface Groups { + owners: Group; + editors: Group; + viewers: Group; + guests: Group; + members?: Group; +} + +class Seed { + public userRepository: Repository; + public groupRepository: Repository; + public groups: {[key: string]: Groups}; + + constructor(public connection: Connection) { + this.userRepository = connection.getRepository(User); + this.groupRepository = connection.getRepository(Group); + this.groups = {}; + } + + public async createGroups(parent?: Organization|Workspace): Promise { + const owners = new Group(); + owners.name = 'owners'; + const editors = new Group(); + editors.name = 'editors'; + const viewers = new Group(); + viewers.name = 'viewers'; + const guests = new Group(); + guests.name = 'guests'; + + if (parent) { + // Nest the parent groups inside the new groups + const parentGroups = this.groups[parent.name]; + owners.memberGroups = [parentGroups.owners]; + editors.memberGroups = [parentGroups.editors]; + viewers.memberGroups = [parentGroups.viewers]; + } + + await this.groupRepository.save([owners, editors, viewers, guests]); + + if (!parent) { + // Add the members group for orgs. + const members = new Group(); + members.name = 'members'; + await this.groupRepository.save(members); + return { + owners, + editors, + viewers, + guests, + members + }; + } else { + return { + owners, + editors, + viewers, + guests + }; + } + } + + public async addOrgToGroups(groups: Groups, org: Organization) { + const acl0 = new AclRuleOrg(); + acl0.group = groups.members!; + acl0.permissions = Permissions.VIEW; + acl0.organization = org; + + const acl1 = new AclRuleOrg(); + acl1.group = groups.guests; + acl1.permissions = Permissions.VIEW; + acl1.organization = org; + + const acl2 = new AclRuleOrg(); + acl2.group = groups.viewers; + acl2.permissions = Permissions.VIEW; + acl2.organization = org; + + const acl3 = new AclRuleOrg(); + acl3.group = groups.editors; + acl3.permissions = Permissions.EDITOR; + acl3.organization = org; + + const acl4 = new AclRuleOrg(); + acl4.group = groups.owners; + acl4.permissions = Permissions.OWNER; + acl4.organization = org; + + // should be able to save both together, but typeorm messes up on postgres. + await acl0.save(); + await acl1.save(); + await acl2.save(); + await acl3.save(); + await acl4.save(); + } + + public async addWorkspaceToGroups(groups: Groups, ws: Workspace) { + const acl1 = new AclRuleWs(); + acl1.group = groups.guests; + acl1.permissions = Permissions.VIEW; + acl1.workspace = ws; + + const acl2 = new AclRuleWs(); + acl2.group = groups.viewers; + acl2.permissions = Permissions.VIEW; + acl2.workspace = ws; + + const acl3 = new AclRuleWs(); + acl3.group = groups.editors; + acl3.permissions = Permissions.EDITOR; + acl3.workspace = ws; + + const acl4 = new AclRuleWs(); + acl4.group = groups.owners; + acl4.permissions = Permissions.OWNER; + acl4.workspace = ws; + + // should be able to save both together, but typeorm messes up on postgres. + await acl1.save(); + await acl2.save(); + await acl3.save(); + await acl4.save(); + } + + public async addDocumentToGroups(groups: Groups, doc: Document) { + const acl1 = new AclRuleDoc(); + acl1.group = groups.guests; + acl1.permissions = Permissions.VIEW; + acl1.document = doc; + + const acl2 = new AclRuleDoc(); + acl2.group = groups.viewers; + acl2.permissions = Permissions.VIEW; + acl2.document = doc; + + const acl3 = new AclRuleDoc(); + acl3.group = groups.editors; + acl3.permissions = Permissions.EDITOR; + acl3.document = doc; + + const acl4 = new AclRuleDoc(); + acl4.group = groups.owners; + acl4.permissions = Permissions.OWNER; + acl4.document = doc; + + await acl1.save(); + await acl2.save(); + await acl3.save(); + await acl4.save(); + } + + public async addUserToGroup(user: User, group: Group) { + await this.connection.createQueryBuilder() + .relation(Group, "memberUsers") + .of(group) + .add(user); + } + + public async addDocs(orgs: Array<{name: string, domain?: string, host?: string, product?: string, + workspaces: Array<{name: string, docs: string[]}>}>) { + let docId = 1; + for (const org of orgs) { + const o = new Organization(); + o.name = org.name; + const ba = new BillingAccount(); + ba.individual = false; + const productName = org.product || 'Free'; + ba.product = (await Product.findOne({name: productName}))!; + o.billingAccount = ba; + if (org.domain) { o.domain = org.domain; } + if (org.host) { o.host = org.host; } + await ba.save(); + await o.save(); + const grps = await this.createGroups(); + this.groups[o.name] = grps; + await this.addOrgToGroups(grps, o); + for (const workspace of org.workspaces) { + const w = new Workspace(); + w.name = workspace.name; + w.org = o; + await w.save(); + const wgrps = await this.createGroups(o); + this.groups[w.name] = wgrps; + await this.addWorkspaceToGroups(wgrps, w); + for (const doc of workspace.docs) { + const d = new Document(); + d.name = doc; + d.workspace = w; + d.id = `sample_${docId}`; + docId++; + await d.save(); + const dgrps = await this.createGroups(w); + this.groups[d.name] = dgrps; + await this.addDocumentToGroups(dgrps, d); + } + } + } + } + + public async run() { + if (await this.userRepository.findOne()) { + // we already have a user - skip seeding database + return; + } + + await this.addDocs(exampleOrgs); + await this._buildUsers(exampleUsers); + } + + // Creates benchmark data with 10 orgs, 50 workspaces per org and 20 docs per workspace. + public async runBenchmark() { + if (await this.userRepository.findOne()) { + // we already have a user - skip seeding database + return; + } + + await this.connection.runMigrations(); + + const benchmarkOrgs = _generateData(100, 50, 20); + // Create an access object giving Chimpy random access to the orgs. + const chimpyAccess: {[name: string]: string} = {}; + benchmarkOrgs.forEach((_org: any) => { + const zeroToThree = Math.floor(Math.random() * 4); + chimpyAccess[_org.name] = ACCESS_GROUPS[zeroToThree]; + }); + + await this.addDocs(benchmarkOrgs); + await this._buildUsers({ Chimpy: chimpyAccess }); + } + + private async _buildUsers(userAccessMap: {[user: string]: {[org: string]: string}}) { + for (const name of Object.keys(userAccessMap)) { + const user = new User(); + user.name = name; + user.apiKey = "api_key_for_" + name.toLowerCase(); + await user.save(); + const login = new Login(); + login.displayEmail = login.email = name.toLowerCase() + "@getgrist.com"; + login.user = user; + await login.save(); + const personal = await Organization.findOne({name: name + "land"}); + if (personal) { + personal.owner = user; + await personal.save(); + } + for (const org of Object.keys(userAccessMap[name])) { + await this.addUserToGroup(user, (this.groups[org] as any)[userAccessMap[name][org]]); + } + } + } +} + +// When running mocha on several test files at once, we need to reset our database connection +// if it exists. This is a little ugly since it is stored globally. +export async function removeConnection() { + if (getConnectionManager().connections.length > 0) { + if (getConnectionManager().connections.length > 1) { + throw new Error("unexpected number of connections"); + } + await getConnectionManager().connections[0].close(); + // There is no official way to delete connections that I've found. + (getConnectionManager().connections as any) = []; + } +} + +export async function createInitialDb(connection?: Connection, migrateAndSeedData: boolean = true) { + // In jenkins tests, we may want to reset the database to a clean + // state. If so, TEST_CLEAN_DATABASE will have been set. How to + // clean the database depends on what kind of database it is. With + // postgres, it suffices to recreate our schema ("public", the + // default). With sqlite, it suffices to delete the file -- but we + // are only allowed to do this if there is no connection open to it + // (so we fail if a connection has already been made). If the + // sqlite db is in memory (":memory:") there's nothing to delete. + const uncommitted = !connection; // has user already created a connection? + // if so we won't be able to delete sqlite db + connection = connection || await createConnection(); + const opt = connection.driver.options; + if (process.env.TEST_CLEAN_DATABASE) { + if (opt.type === 'sqlite') { + const database = (opt as any).database; + // Only dbs on disk need to be deleted + if (database !== ':memory:') { + // We can only delete on-file dbs if no connection is open to them + if (!uncommitted) { + throw Error("too late to clean sqlite db"); + } + await removeConnection(); + if (await fse.pathExists(database)) { + await fse.unlink(database); + } + connection = await createConnection(); + } + } else if (opt.type === 'postgres') { + // recreate schema, destroying everything that was inside it + await connection.query("DROP SCHEMA public CASCADE;"); + await connection.query("CREATE SCHEMA public;"); + } else { + throw new Error(`do not know how to clean a ${opt.type} db`); + } + } + + // Finally - actually initialize the database. + if (migrateAndSeedData) { + await updateDb(connection); + await addSeedData(connection); + } +} + +// add some test data to the database. +export async function addSeedData(connection: Connection) { + await synchronizeProducts(connection, true); + await connection.transaction(async tr => { + const seed = new Seed(tr.connection); + await seed.run(); + }); +} + +export async function createBenchmarkDb(connection?: Connection) { + connection = connection || await createConnection(); + await updateDb(connection); + await connection.transaction(async tr => { + const seed = new Seed(tr.connection); + await seed.runBenchmark(); + }); +} + +export async function createServer(port: number, initDb = createInitialDb): Promise { + const flexServer = new FlexServer(port); + flexServer.addJsonSupport(); + await flexServer.start(); + await flexServer.initHomeDBManager(); + flexServer.addAccessMiddleware(); + flexServer.addApiMiddleware(); + flexServer.addHomeApi(); + flexServer.addApiErrorHandlers(); + await initDb(flexServer.getHomeDBManager().connection); + flexServer.summary(); + return flexServer; +} + +export async function createBenchmarkServer(port: number): Promise { + return createServer(port, createBenchmarkDb); +} + +// Generates a random dataset of orgs, workspaces and docs. The number of workspaces +// given is per org, and the number of docs given is per workspace. +function _generateData(numOrgs: number, numWorkspaces: number, numDocs: number) { + if (numOrgs < 1 || numWorkspaces < 1 || numDocs < 0) { + throw new Error('_generateData error: Invalid arguments'); + } + const example = []; + for (let i = 0; i < numOrgs; i++) { + const workspaces = []; + for (let j = 0; j < numWorkspaces; j++) { + const docs = []; + for (let k = 0; k < numDocs; k++) { + const docIndex = (i * numWorkspaces * numDocs) + (j * numDocs) + k; + docs.push(`doc-${docIndex}`); + } + const workspaceIndex = (i * numWorkspaces) + j; + workspaces.push({ + name: `ws-${workspaceIndex}`, + docs + }); + } + example.push({ + name: `org-${i}`, + domain: `org-${i}`, + workspaces + }); + } + return example; +} + +/** + * To set up TYPEORM_* environment variables for testing, call this in a before() call of a test + * suite, using setUpDB(this); + */ +export function setUpDB(context?: IHookCallbackContext) { + if (!process.env.TYPEORM_DATABASE) { + process.env.TYPEORM_DATABASE = ":memory:"; + } else { + if (context) { context.timeout(60000); } + } +} + +async function main() { + const cmd = process.argv[2]; + if (cmd === 'init') { + const connection = await createConnection(); + await createInitialDb(connection); + return; + } else if (cmd === 'benchmark') { + const connection = await createConnection(); + await createInitialDb(connection, false); + await createBenchmarkDb(connection); + return; + } else if (cmd === 'migrate') { + process.env.TYPEORM_LOGGING = 'true'; + const connection = await createConnection(); + await runMigrations(connection); + return; + } else if (cmd === 'revert') { + process.env.TYPEORM_LOGGING = 'true'; + const connection = await createConnection(); + await undoLastMigration(connection); + return; + } else if (cmd === 'serve') { + const home = await createServer(3000); + // tslint:disable-next-line:no-console + console.log(`Home API demo available at ${home.getOwnUrl()}`); + return; + } + // tslint:disable-next-line:no-console + console.log("Call with: init | migrate | revert | serve | benchmark"); +} + +if (require.main === module) { + main().catch(e => { + // tslint:disable-next-line:no-console + console.log(e); + }); +} diff --git a/test/gen-server/testUtils.ts b/test/gen-server/testUtils.ts new file mode 100644 index 00000000..81dc18a1 --- /dev/null +++ b/test/gen-server/testUtils.ts @@ -0,0 +1,103 @@ +import {GristLoadConfig} from 'app/common/gristUrls'; +import {BillingAccount} from 'app/gen-server/entity/BillingAccount'; +import {Organization} from 'app/gen-server/entity/Organization'; +import {Product} from 'app/gen-server/entity/Product'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {INotifier} from 'app/server/lib/INotifier'; +import {AxiosRequestConfig} from "axios"; +import {delay} from 'bluebird'; + +/** + * Returns an AxiosRequestConfig, that identifies the user with `username` on a server running + * against a database using `test/gen-server/seed.ts`. Also tells axios not to raise exception on + * failed request. + */ +export function configForUser(username: string): AxiosRequestConfig { + const config: AxiosRequestConfig = { + responseType: 'json', + validateStatus: (status: number) => true, + headers: { + 'X-Requested-With': 'XMLHttpRequest', + } + }; + if (username !== 'Anonymous') { + config.headers.Authorization = 'Bearer api_key_for_' + username.toLowerCase(); + } + return config; +} + +/** + * Create a new user and return their personal org. + */ +export async function createUser(dbManager: HomeDBManager, name: string): Promise { + const username = name.toLowerCase(); + const email = `${username}@getgrist.com`; + const user = await dbManager.getUserByLogin(email, {email, name}); + if (!user) { throw new Error('failed to create user'); } + user.apiKey = `api_key_for_${username}`; + await user.save(); + const userHome = (await dbManager.getOrg({userId: user.id}, null)).data; + if (!userHome) { throw new Error('failed to create personal org'); } + return userHome; +} + +/** + * Associate a given org with a given product. + */ +export async function setPlan(dbManager: HomeDBManager, org: {billingAccount?: {id: number}}, + productName: string) { + const product = await dbManager.connection.manager.findOne(Product, {where: {name: productName}}); + if (!product) { throw new Error(`cannot find product ${productName}`); } + if (!org.billingAccount) { throw new Error('must join billingAccount'); } + await dbManager.connection.createQueryBuilder() + .update(BillingAccount) + .set({product}) + .where('id = :bid', {bid: org.billingAccount.id}) + .execute(); +} + +/** + * Returns the window.gristConfig object extracted from the raw HTML of app.html page. + */ +export function getGristConfig(page: string): Partial { + const match = /window\.gristConfig = ([^;]*)/.exec(page); + if (!match) { throw new Error('cannot find grist config'); } + return JSON.parse(match[1]); +} + +/** + * Waits for all pending (back-end) notifications to complete. Notifications are + * started during request handling, but may not complete fully during it. + */ +export async function waitForAllNotifications(notifier: INotifier, maxWait: number = 1000) { + const start = Date.now(); + while (Date.now() - start < maxWait) { + if (!notifier.testPending) { return; } + await delay(1); + } + throw new Error('waitForAllNotifications timed out'); +} + +// count the number of rows in a table +export async function getRowCount(dbManager: HomeDBManager, tableName: string): Promise { + const result = await dbManager.connection.query(`select count(*) as ct from ${tableName}`); + return parseInt(result[0].ct, 10); +} + +// gather counts for all significant tables - handy as a sanity check on deletions +export async function getRowCounts(dbManager: HomeDBManager) { + return { + aclRules: await getRowCount(dbManager, 'acl_rules'), + docs: await getRowCount(dbManager, 'docs'), + groupGroups: await getRowCount(dbManager, 'group_groups'), + groupUsers: await getRowCount(dbManager, 'group_users'), + groups: await getRowCount(dbManager, 'groups'), + logins: await getRowCount(dbManager, 'logins'), + orgs: await getRowCount(dbManager, 'orgs'), + users: await getRowCount(dbManager, 'users'), + workspaces: await getRowCount(dbManager, 'workspaces'), + billingAccounts: await getRowCount(dbManager, 'billing_accounts'), + billingAccountManagers: await getRowCount(dbManager, 'billing_account_managers'), + products: await getRowCount(dbManager, 'products') + }; +} diff --git a/test/server/customUtil.ts b/test/server/customUtil.ts new file mode 100644 index 00000000..91a4175a --- /dev/null +++ b/test/server/customUtil.ts @@ -0,0 +1,66 @@ +import {getAppRoot} from 'app/server/lib/places'; +import {fromCallback} from 'bluebird'; +import * as express from 'express'; +import * as http from 'http'; +import {AddressInfo, Socket} from 'net'; +import * as path from 'path'; +import {fixturesRoot} from 'test/server/testUtils'; + +export interface Serving { + url: string; + shutdown: () => void; +} + + +// Adds static files from a directory. +// By default exposes /fixture/sites +export function addStatic(app: express.Express, rootDir?: string) { + // mix in a copy of the plugin api + app.use(/^\/(grist-plugin-api.js)$/, (req, res) => + res.sendFile(req.params[0], {root: + path.resolve(getAppRoot(), "static")})); + app.use(express.static(rootDir || path.resolve(fixturesRoot, "sites"), { + setHeaders: (res) => { + res.set("Access-Control-Allow-Origin", "*"); + } + })); +} + +// Serve from a directory. +export async function serveStatic(rootDir: string): Promise { + return serveSomething(app => addStatic(app, rootDir)); +} + +// Serve a string of html. +export async function serveSinglePage(html: string): Promise { + return serveSomething(app => { + app.get('', (req, res) => res.send(html)); + }); +} + +export function serveCustomViews(): Promise { + return serveStatic(path.resolve(fixturesRoot, "sites")); +} + +export async function serveSomething(setup: (app: express.Express) => void, port= 0): Promise { + const app = express(); + const server = http.createServer(app); + await fromCallback((cb: any) => server.listen(port, cb)); + + const connections = new Set(); + server.on('connection', (conn) => { + connections.add(conn); + conn.on('close', () => connections.delete(conn)); + }); + + function shutdown() { + server.close(); + for (const conn of connections) { conn.destroy(); } + } + + port = (server.address() as AddressInfo).port; + app.set('port', port); + setup(app); + const url = `http://localhost:${port}`; + return {url, shutdown}; +} diff --git a/test/server/docTools.ts b/test/server/docTools.ts new file mode 100644 index 00000000..32d7be52 --- /dev/null +++ b/test/server/docTools.ts @@ -0,0 +1,243 @@ +import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {DummyAuthorizer} from 'app/server/lib/Authorizer'; +import {create} from 'app/server/lib/create'; +import {DocManager} from 'app/server/lib/DocManager'; +import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession'; +import {DocStorageManager} from 'app/server/lib/DocStorageManager'; +import {GristServer} from 'app/server/lib/GristServer'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import {getAppRoot} from 'app/server/lib/places'; +import {PluginManager} from 'app/server/lib/PluginManager'; +import {createTmpDir as createTmpUploadDir, FileUploadInfo, globalUploadSet} from 'app/server/lib/uploads'; +import * as testUtils from 'test/server/testUtils'; + +import {assert} from 'chai'; +import * as fse from 'fs-extra'; +import {tmpdir} from 'os'; +import * as path from 'path'; +import * as tmp from 'tmp'; + +tmp.setGracefulCleanup(); + +// it is sometimes useful in debugging to turn off automatic cleanup of docs and workspaces. +const noCleanup = Boolean(process.env.NO_CLEANUP); + +/** + * Use from a test suite to get an object with convenient methods for creating ActiveDocs: + * + * createDoc(docName): creates a new empty document. + * loadFixtureDoc(docName): loads a copy of a fixture document. + * loadDoc(docName): loads a given document, e.g. previously created with createDoc(). + * createFakeSession(): creates a fake DocSession for use when applying user actions. + * + * Also available are accessors for the created "managers": + * getDocManager() + * getStorageManager() + * getPluginManager() + * + * It also takes care of cleaning up any created ActiveDocs. + * @param persistAcrossCases Don't shut down created ActiveDocs between test cases. + * @param useFixturePlugins Use the plugins in `test/fixtures/plugins` + */ +export function createDocTools(options: {persistAcrossCases?: boolean, + useFixturePlugins?: boolean, + storageManager?: IDocStorageManager, + server?: GristServer} = {}) { + let tmpDir: string; + let docManager: DocManager; + + async function doBefore() { + tmpDir = await createTmpDir(); + const pluginManager = options.useFixturePlugins ? await createFixturePluginManager() : undefined; + docManager = await createDocManager({tmpDir, pluginManager, storageManager: options.storageManager, + server: options.server}); + } + + async function doAfter() { + // Clean up at the end of the test suite (in addition to the optional per-test cleanup). + await testUtils.captureLog('info', () => docManager.shutdownAll()); + assert.equal(docManager.numOpenDocs(), 0); + await globalUploadSet.cleanupAll(); + + // Clean up the temp directory. + if (!noCleanup) { + await fse.remove(tmpDir); + } + } + + // Allow using outside of mocha + if (typeof before !== "undefined") { + before(doBefore); + after(doAfter); + + // Check after each test case that all ActiveDocs got shut down. + afterEach(async function() { + if (!options.persistAcrossCases) { + await docManager.shutdownAll(); + assert.equal(docManager.numOpenDocs(), 0); + } + }); + } + + const systemSession = makeExceptionalDocSession('system'); + return { + /** create a fake session for use when applying user actions to a document */ + createFakeSession(): DocSession { + return {client: null, authorizer: new DummyAuthorizer('editors', 'doc')} as any as DocSession; + }, + + /** create a throw-away, empty document for testing purposes */ + async createDoc(docName: string): Promise { + return docManager.createNewEmptyDoc(systemSession, docName); + }, + + /** load a copy of a fixture document for testing purposes */ + async loadFixtureDoc(docName: string): Promise { + const copiedDocName = await testUtils.useFixtureDoc(docName, docManager.storageManager); + return this.loadDoc(copiedDocName); + }, + + /** load a copy of a local document at an arbitrary path on disk for testing purposes */ + async loadLocalDoc(srcPath: string): Promise { + const copiedDocName = await testUtils.useLocalDoc(srcPath, docManager.storageManager); + return this.loadDoc(copiedDocName); + }, + + /** like `loadFixtureDoc`, but lets you rename the document on disk */ + async loadFixtureDocAs(docName: string, alias: string): Promise { + const copiedDocName = await testUtils.useFixtureDoc(docName, docManager.storageManager, alias); + return this.loadDoc(copiedDocName); + }, + + /** Loads a given document, e.g. previously created with createDoc() */ + async loadDoc(docName: string): Promise { + return docManager.fetchDoc(systemSession, docName); + }, + + getDocManager() { return docManager; }, + getStorageManager() { return docManager.storageManager; }, + getPluginManager() { return docManager.pluginManager; }, + + /** Setup that needs to be done before using the tools, typically called by mocha */ + before() { return doBefore(); }, + + /** Teardown that needs to be done after using the tools, typically called by mocha */ + after() { return doAfter(); }, + }; +} + +/** + * Returns a DocManager for tests, complete with a PluginManager and DocStorageManager. + * @param options.pluginManager The PluginManager to use; defaults to using a real global singleton + * that loads built-in modules. + */ +export async function createDocManager( + options: {tmpDir?: string, pluginManager?: PluginManager, + storageManager?: IDocStorageManager, + server?: GristServer} = {}): Promise { + // Set Grist home to a temporary directory, and wipe it out on exit. + const tmpDir = options.tmpDir || await createTmpDir(); + const docStorageManager = options.storageManager || new DocStorageManager(tmpDir); + const pluginManager = options.pluginManager || await getGlobalPluginManager(); + const store = getDocWorkerMap(); + const internalPermitStore = store.getPermitStore('1'); + const externalPermitStore = store.getPermitStore('2'); + return new DocManager(docStorageManager, pluginManager, null, options.server || { + ...createDummyGristServer(), + getPermitStore() { return internalPermitStore; }, + getExternalPermitStore() { return externalPermitStore; }, + getStorageManager() { return docStorageManager; }, + }); +} + +export function createDummyGristServer(): GristServer { + return { + create, + getHost() { return 'localhost:4242'; }, + getHomeUrl() { return 'http://localhost:4242'; }, + getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); }, + getMergedOrgUrl() { return 'http://localhost:4242'; }, + getOwnUrl() { return 'http://localhost:4242'; }, + getPermitStore() { throw new Error('no permit store'); }, + getExternalPermitStore() { throw new Error('no external permit store'); }, + getGristConfig() { return { homeUrl: '', timestampMs: 0 }; }, + getOrgUrl() { return Promise.resolve(''); }, + getResourceUrl() { return Promise.resolve(''); }, + getSessions() { throw new Error('no sessions'); }, + getComm() { throw new Error('no comms'); }, + getHosts() { throw new Error('no hosts'); }, + getHomeDBManager() { throw new Error('no db'); }, + getStorageManager() { throw new Error('no storage manager'); }, + getNotifier() { throw new Error('no notifier'); }, + getDocTemplate() { throw new Error('no doc template'); }, + getTag() { return 'tag'; }, + sendAppPage() { return Promise.resolve(); }, + }; +} + +export async function createTmpDir(): Promise { + const tmpRootDir = process.env.TESTDIR || tmpdir(); + await fse.mkdirs(tmpRootDir); + return fse.realpath(await tmp.dirAsync({ + dir: tmpRootDir, + prefix: 'grist_test_', + unsafeCleanup: true, + keep: noCleanup, + })); +} + +/** + * Creates a file with the given name (and simple dummy content) in dirPath, and returns + * FileUploadInfo for it. + */ +export async function createFile(dirPath: string, name: string): Promise { + const absPath = path.join(dirPath, name); + await fse.outputFile(absPath, `${name}:${name}\n`); + return { + absPath, + origName: name, + size: (await fse.stat(absPath)).size, + ext: path.extname(name), + }; +} + +/** + * Creates an upload with the given filenames (containg simple dummy content), in the + * globalUploadSet, and returns its uploadId. The upload is registered with the given accessId + * (userId), and the same id must be used to retrieve it. + */ +export async function createUpload(fileNames: string[], accessId: string|null): Promise { + const {tmpDir, cleanupCallback} = await createTmpUploadDir({}); + const files = await Promise.all(fileNames.map((name) => createFile(tmpDir, name))); + return globalUploadSet.registerUpload(files, tmpDir, cleanupCallback, accessId); +} + + +let _globalPluginManager: PluginManager|null = null; + +// Helper to create a singleton PluginManager. This includes loading built-in plugins. Since most +// tests don't make any use of it, it's fine to reuse a single one. For tests that need a custom +// one, pass one into createDocManager(). +export async function getGlobalPluginManager(): Promise { + if (!_globalPluginManager) { + const appRoot = getAppRoot(); + _globalPluginManager = new PluginManager(appRoot); + await _globalPluginManager.initialize(); + } + return _globalPluginManager; +} + +// Path to the folder where builtIn plugins leave in test/fixtures +export const builtInFolder = path.join(testUtils.fixturesRoot, 'plugins/builtInPlugins'); + +// Path to the folder where installed plugins leave in test/fixtures +export const installedFolder = path.join(testUtils.fixturesRoot, 'plugins/installedPlugins'); + +// Creates a plugin manager which loads the plugins in `test/fixtures/plugins` +async function createFixturePluginManager() { + const p = new PluginManager(builtInFolder, installedFolder); + p.appRoot = getAppRoot(); + await p.initialize(); + return p; +} diff --git a/test/server/gristClient.ts b/test/server/gristClient.ts new file mode 100644 index 00000000..324a803a --- /dev/null +++ b/test/server/gristClient.ts @@ -0,0 +1,167 @@ +import { DocAction } from 'app/common/DocActions'; +import { FlexServer } from 'app/server/lib/FlexServer'; +import axios from 'axios'; +import pick = require('lodash/pick'); +import * as WebSocket from 'ws'; + +interface GristRequest { + reqId: number; + method: string; + args: any[]; +} + +interface GristResponse { + reqId: number; + error?: string; + errorCode?: string; + data?: any; +} + +interface GristMessage { + type: 'clientConnect' | 'docUserAction'; + docFD: number; + data: any; +} + +export class GristClient { + public messages: GristMessage[] = []; + + private _requestId: number = 0; + private _pending: Array = []; + private _consumer: () => void; + private _ignoreTrivialActions: boolean = false; + + constructor(public ws: any) { + ws.onmessage = (data: any) => { + const msg = pick(JSON.parse(data.data), + ['reqId', 'error', 'errorCode', 'data', 'type', 'docFD']); + if (this._ignoreTrivialActions && msg.type === 'docUserAction' && + msg.data?.actionGroup?.internal === true && + msg.data?.docActions?.length === 0) { + return; + } + this._pending.push(msg); + if (this._consumer) { this._consumer(); } + }; + } + + // After a document is opened, the sandbox recomputes its formulas and sends any changes. + // The client will receive an update even if there are no changes. This may be useful in + // the future to know that the document is up to date. But for testing, this asynchronous + // message can be awkward. Call this method to ignore it. + public ignoreTrivialActions() { + this._ignoreTrivialActions = true; + } + + public flush() { + this._pending = []; + } + + public shift() { + return this._pending.shift(); + } + + public count() { + return this._pending.length; + } + + public async read(): Promise { + for (;;) { + if (this._pending.length) { + return this._pending.shift(); + } + await new Promise(resolve => this._consumer = resolve); + } + } + + public async readMessage(): Promise { + const result = await this.read(); + if (!result.type) { + throw new Error(`message looks wrong: ${JSON.stringify(result)}`); + } + return result; + } + + public async readResponse(): Promise { + this.messages = []; + for (;;) { + const result = await this.read(); + if (result.reqId === undefined) { + this.messages.push(result); + continue; + } + if (result.reqId !== this._requestId) { + throw new Error("unexpected request id"); + } + return result; + } + } + + // Helper to read the next docUserAction ignoring anything else (e.g. a duplicate clientConnect). + public async readDocUserAction(): Promise { + while (true) { // eslint-disable-line no-constant-condition + const msg = await this.readMessage(); + if (msg.type === 'docUserAction') { + return msg.data.docActions; + } + } + } + + public async send(method: string, ...args: any[]): Promise { + const p = this.readResponse(); + this._requestId++; + const req: GristRequest = { + reqId: this._requestId, + method, + args + }; + this.ws.send(JSON.stringify(req)); + const result = await p; + return result; + } + + public async close() { + this.ws.terminate(); + this.ws.close(); + } + + public async openDocOnConnect(docId: string) { + const msg = await this.readMessage(); + if (msg.type !== 'clientConnect') { throw new Error('expected clientConnect'); } + const openDoc = await this.send('openDoc', docId); + if (openDoc.error) { throw new Error('error in openDocOnConnect'); } + return openDoc; + } +} + +export async function openClient(server: FlexServer, email: string, org: string, + emailHeader?: string): Promise { + const headers: Record = {}; + if (!emailHeader) { + const resp = await axios.get(`${server.getOwnUrl()}/test/session`); + const cookie = resp.headers['set-cookie'][0]; + if (email !== 'anon@getgrist.com') { + const cid = decodeURIComponent(cookie.split('=')[1].split(';')[0]); + const comm = server.getComm(); + const sessionId = comm.getSessionIdFromCookie(cid); + const scopedSession = comm.getOrCreateSession(sessionId, {org}); + const profile = { email, email_verified: true, name: "Someone" }; + await scopedSession.updateUserProfile({} as any, profile); + } + headers.Cookie = cookie; + } else { + headers[emailHeader] = email; + } + const ws = new WebSocket('ws://localhost:' + server.getOwnPort() + `/o/${org}`, { + headers + }); + await new Promise(function(resolve, reject) { + ws.on('open', function() { + resolve(ws); + }); + ws.on('error', function(err: any) { + reject(err); + }); + }); + return new GristClient(ws); +} diff --git a/test/server/lib/Authorizer.ts b/test/server/lib/Authorizer.ts new file mode 100644 index 00000000..6c4af796 --- /dev/null +++ b/test/server/lib/Authorizer.ts @@ -0,0 +1,305 @@ +import {parseUrlId} from 'app/common/gristUrls'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {DocManager} from 'app/server/lib/DocManager'; +import {FlexServer} from 'app/server/lib/FlexServer'; +import axios from 'axios'; +import {assert} from 'chai'; +import {toPairs} from 'lodash'; +import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed'; +import {configForUser, getGristConfig} from 'test/gen-server/testUtils'; +import {createDocTools} from 'test/server/docTools'; +import {openClient} from 'test/server/gristClient'; +import * as testUtils from 'test/server/testUtils'; +import * as uuidv4 from 'uuid/v4'; + +let serverUrl: string; +let server: FlexServer; +let dbManager: HomeDBManager; + +async function activateServer(home: FlexServer, docManager: DocManager) { + await home.initHomeDBManager(); + home.addHosts(); + home.addDocWorkerMap(); + home.addAccessMiddleware(); + dbManager = home.getHomeDBManager(); + await home.loadConfig({}); + home.addSessions(); + home.addHealthCheck(); + docManager.testSetHomeDbManager(dbManager); + home.testSetDocManager(docManager); + await home.start(); + home.addAccessMiddleware(); + home.addApiMiddleware(); + home.addJsonSupport(); + await home.addLandingPages(); + home.addHomeApi(); + await home.addDoc(); + home.addApiErrorHandlers(); + serverUrl = home.getOwnUrl(); +} + +const chimpy = configForUser('Chimpy'); +const charon = configForUser('Charon'); + +const fixtures: {[docName: string]: string|null} = { + Bananas: 'Hello.grist', + Pluto: 'Hello.grist', +}; + +describe('Authorizer', function() { + + testUtils.setTmpLogLevel('fatal'); + + server = new FlexServer(0, 'test docWorker'); + const docTools = createDocTools({persistAcrossCases: true, useFixturePlugins: false, + server}); + const docs: {[name: string]: {id: string}} = {}; + + // Loads the fixtures documents so that they are available to the doc worker under the correct + // names. + async function loadFixtureDocs() { + for (const [docName, fixtureDoc] of toPairs(fixtures)) { + const docId = String(await dbManager.testGetId(docName)); + if (fixtureDoc) { + await docTools.loadFixtureDocAs(fixtureDoc, docId); + } else { + await docTools.createDoc(docId); + } + docs[docName] = {id: docId}; + } + } + + let oldEnv: testUtils.EnvironmentSnapshot; + before(async function() { + this.timeout(5000); + setUpDB(this); + oldEnv = new testUtils.EnvironmentSnapshot(); + process.env.GRIST_PROXY_AUTH_HEADER = 'X-email'; + await createInitialDb(); + await activateServer(server, docTools.getDocManager()); + await loadFixtureDocs(); + }); + + after(async function() { + const messages = await testUtils.captureLog('warn', async () => { + await server.close(); + await removeConnection(); + }); + assert.lengthOf(messages, 0); + oldEnv.restore(); + }); + + // TODO XXX Is it safe to remove this support now? + // (It used to be implemented in getDocAccessInfo() in Authorizer.ts). + it.skip("viewer gets redirect by title", async function() { + const resp = await axios.get(`${serverUrl}/o/pr/doc/Bananas`, chimpy); + assert.equal(resp.status, 200); + assert.equal(getGristConfig(resp.data).assignmentId, 'sample_6'); + assert.match(resp.request.res.responseUrl, /\/doc\/sample_6$/); + const resp2 = await axios.get(`${serverUrl}/o/nasa/doc/Pluto`, chimpy); + assert.equal(resp2.status, 200); + assert.equal(getGristConfig(resp2.data).assignmentId, 'sample_2'); + assert.match(resp2.request.res.responseUrl, /\/doc\/sample_2$/); + }); + + it("stranger gets consistent refusal regardless of title", async function() { + const resp = await axios.get(`${serverUrl}/o/pr/doc/Bananas`, charon); + assert.equal(resp.status, 404); + assert.notMatch(resp.data, /sample_6/); + const resp2 = await axios.get(`${serverUrl}/o/pr/doc/Bananas2`, charon); + assert.equal(resp2.status, 404); + assert.notMatch(resp.data, /sample_6/); + assert.deepEqual(resp.data, resp2.data); + }); + + it("viewer can access title", async function() { + const resp = await axios.get(`${serverUrl}/o/pr/doc/sample_6`, chimpy); + assert.equal(resp.status, 200); + const config = getGristConfig(resp.data); + assert.equal(config.getDoc![config.assignmentId!].name, 'Bananas'); + }); + + it("stranger cannot access title", async function() { + const resp = await axios.get(`${serverUrl}/o/pr/doc/sample_6`, charon); + assert.equal(resp.status, 403); + assert.notMatch(resp.data, /Bananas/); + }); + + it("viewer cannot access document from wrong org", async function() { + const resp = await axios.get(`${serverUrl}/o/nasa/doc/sample_6`, chimpy); + assert.equal(resp.status, 404); + }); + + it("websocket allows openDoc for viewer", async function() { + const cli = await openClient(server, 'chimpy@getgrist.com', 'pr'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + const openDoc = await cli.send("openDoc", "sample_6"); + assert.equal(openDoc.error, undefined); + assert.match(JSON.stringify(openDoc.data), /Table1/); + await cli.close(); + }); + + it("websocket forbids openDoc for stranger", async function() { + const cli = await openClient(server, 'charon@getgrist.com', 'pr'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + const openDoc = await cli.send("openDoc", "sample_6"); + assert.match(openDoc.error!, /No view access/); + assert.equal(openDoc.data, undefined); + assert.match(openDoc.errorCode!, /AUTH_NO_VIEW/); + await cli.close(); + }); + + it("websocket forbids applyUserActions for viewer", async function() { + const cli = await openClient(server, 'charon@getgrist.com', 'nasa'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + const openDoc = await cli.openDocOnConnect("sample_2"); + assert.equal(openDoc.error, undefined); + const nonce = uuidv4(); + const applyUserActions = await cli.send("applyUserActions", + 0, + [["UpdateRecord", "Table1", 1, {A: nonce}], {}]); + assert.lengthOf(cli.messages, 0); // no user actions pushed to client + assert.match(applyUserActions.error!, /No write access/); + assert.match(applyUserActions.errorCode!, /AUTH_NO_EDIT/); + const fetchTable = await cli.send("fetchTable", 0, "Table1"); + assert.equal(fetchTable.error, undefined); + assert.notInclude(JSON.stringify(fetchTable.data), nonce); + await cli.close(); + }); + + it("websocket allows applyUserActions for editor", async function() { + const cli = await openClient(server, 'chimpy@getgrist.com', 'nasa'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + const openDoc = await cli.openDocOnConnect("sample_2"); + assert.equal(openDoc.error, undefined); + const nonce = uuidv4(); + const applyUserActions = await cli.send("applyUserActions", + 0, + [["UpdateRecord", "Table1", 1, {A: nonce}]]); + assert.lengthOf(cli.messages, 1); // user actions pushed to client + assert.equal(applyUserActions.error, undefined); + const fetchTable = await cli.send("fetchTable", 0, "Table1"); + assert.equal(fetchTable.error, undefined); + assert.include(JSON.stringify(fetchTable.data), nonce); + await cli.close(); + }); + + it("can keep different simultaneous clients of a doc straight", async function() { + const editor = await openClient(server, 'chimpy@getgrist.com', 'nasa'); + assert.equal((await editor.readMessage()).type, 'clientConnect'); + const viewer = await openClient(server, 'charon@getgrist.com', 'nasa'); + assert.equal((await viewer.readMessage()).type, 'clientConnect'); + const stranger = await openClient(server, 'kiwi@getgrist.com', 'nasa'); + assert.equal((await stranger.readMessage()).type, 'clientConnect'); + + editor.ignoreTrivialActions(); + viewer.ignoreTrivialActions(); + stranger.ignoreTrivialActions(); + assert.equal((await editor.send("openDoc", "sample_2")).error, undefined); + assert.equal((await viewer.send("openDoc", "sample_2")).error, undefined); + assert.match((await stranger.send("openDoc", "sample_2")).error!, /No view access/); + + const action = [0, [["UpdateRecord", "Table1", 1, {A: "foo"}]]]; + assert.equal((await editor.send("applyUserActions", ...action)).error, undefined); + assert.match((await viewer.send("applyUserActions", ...action)).error!, /No write access/); + // Different message here because sending actions without a doc being open. + assert.match((await stranger.send("applyUserActions", ...action)).error!, /Invalid/); + }); + + it("previewer has view access to docs", async function() { + const cli = await openClient(server, 'thumbnail@getgrist.com', 'nasa'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + const openDoc = await cli.send("openDoc", "sample_2"); + assert.equal(openDoc.error, undefined); + const nonce = uuidv4(); + const applyUserActions = await cli.send("applyUserActions", + 0, + [["UpdateRecord", "Table1", 1, {A: nonce}], {}]); + assert.lengthOf(cli.messages, 0); // no user actions pushed to client + assert.match(applyUserActions.error!, /No write access/); + assert.match(applyUserActions.errorCode!, /AUTH_NO_EDIT/); + const fetchTable = await cli.send("fetchTable", 0, "Table1"); + assert.equal(fetchTable.error, undefined); + assert.notInclude(JSON.stringify(fetchTable.data), nonce); + await cli.close(); + }); + + it("viewer can fork doc", async function() { + const cli = await openClient(server, 'charon@getgrist.com', 'nasa'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + const openDoc = await cli.send("openDoc", "sample_2"); + assert.equal(openDoc.error, undefined); + const result = await cli.send("fork", 0); + assert.equal(result.data.docId, result.data.urlId); + const parts = parseUrlId(result.data.docId); + assert.equal(parts.trunkId, "sample_2"); + assert.isAbove(parts.forkId!.length, 4); + assert.equal(parts.forkUserId, await dbManager.testGetId('Charon') as number); + }); + + it("anon can fork doc", async function() { + // anon does not have access to doc initially + const cli = await openClient(server, 'anon@getgrist.com', 'nasa'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + let openDoc = await cli.send("openDoc", "sample_2"); + assert.match(openDoc.error!, /No view access/); + + // grant anon access to doc and retry + await dbManager.updateDocPermissions({ + userId: await dbManager.testGetId('Chimpy') as number, + urlId: 'sample_2', + org: 'nasa' + }, {users: {"anon@getgrist.com": "viewers"}}); + dbManager.flushDocAuthCache(); + openDoc = await cli.send("openDoc", "sample_2"); + assert.equal(openDoc.error, undefined); + + // make a fork + const result = await cli.send("fork", 0); + assert.equal(result.data.docId, result.data.urlId); + const parts = parseUrlId(result.data.docId); + assert.equal(parts.trunkId, "sample_2"); + assert.isAbove(parts.forkId!.length, 4); + assert.equal(parts.forkUserId, undefined); + }); + + it("can set user via GRIST_PROXY_AUTH_HEADER", async function() { + // User can access a doc by setting header. + const docUrl = `${serverUrl}/o/pr/api/docs/sample_6`; + const resp = await axios.get(docUrl, { + headers: {'X-email': 'chimpy@getgrist.com'} + }); + assert.equal(resp.data.name, 'Bananas'); + + // Unknown user is denied. + await assert.isRejected(axios.get(docUrl, { + headers: {'X-email': 'notchimpy@getgrist.com'} + })); + + // User can access a doc via websocket by setting header. + let cli = await openClient(server, 'chimpy@getgrist.com', 'pr', 'X-email'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + let openDoc = await cli.send("openDoc", "sample_6"); + assert.equal(openDoc.error, undefined); + assert.match(JSON.stringify(openDoc.data), /Table1/); + await cli.close(); + + // Unknown user is denied. + cli = await openClient(server, 'notchimpy@getgrist.com', 'pr', 'X-email'); + cli.ignoreTrivialActions(); + assert.equal((await cli.readMessage()).type, 'clientConnect'); + openDoc = await cli.send("openDoc", "sample_6"); + assert.match(openDoc.error!, /No view access/); + assert.equal(openDoc.data, undefined); + assert.match(openDoc.errorCode!, /AUTH_NO_VIEW/); + await cli.close(); + }); +}); diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts new file mode 100644 index 00000000..48e80b66 --- /dev/null +++ b/test/server/lib/DocApi.ts @@ -0,0 +1,2589 @@ +import {ActionSummary} from 'app/common/ActionSummary'; +import {BulkColValues} from 'app/common/DocActions'; +import {arrayRepeat} from 'app/common/gutil'; +import {DocState, UserAPIImpl} from 'app/common/UserAPI'; +import {AddOrUpdateRecord} from 'app/plugin/DocApiTypes'; +import {teamFreeFeatures} from 'app/gen-server/entity/Product'; +import {CellValue, GristObjCode} from 'app/plugin/GristData'; +import {applyQueryParameters, docDailyApiUsageKey} from 'app/server/lib/DocApi'; +import * as log from 'app/server/lib/log'; +import {exitPromise} from 'app/server/lib/serverUtils'; +import {connectTestingHooks, TestingHooksClient} from 'app/server/lib/TestingHooks'; +import axios, {AxiosResponse} from 'axios'; +import {delay} from 'bluebird'; +import * as bodyParser from 'body-parser'; +import {assert} from 'chai'; +import {ChildProcess, execFileSync, spawn} from 'child_process'; +import * as FormData from 'form-data'; +import * as fse from 'fs-extra'; +import * as _ from 'lodash'; +import fetch from 'node-fetch'; +import {tmpdir} from 'os'; +import * as path from 'path'; +import {createClient, RedisClient} from 'redis'; +import {configForUser} from 'test/gen-server/testUtils'; +import {serveSomething, Serving} from 'test/server/customUtil'; +import * as testUtils from 'test/server/testUtils'; +import clone = require('lodash/clone'); +import defaultsDeep = require('lodash/defaultsDeep'); + +const chimpy = configForUser('Chimpy'); +const kiwi = configForUser('Kiwi'); +const charon = configForUser('Charon'); +const nobody = configForUser('Anonymous'); +const support = configForUser('support'); + +// some doc ids +const docIds: {[name: string]: string} = { + ApiDataRecordsTest: 'sample_7', + Timesheets: 'sample_13', + Bananas: 'sample_6', + Antartic: 'sample_11' +}; + +// A testDir of the form grist_test_{USER}_{SERVER_NAME} +const username = process.env.USER || "nobody"; +const tmpDir = path.join(tmpdir(), `grist_test_${username}_docapi`); + +let dataDir: string; +let suitename: string; +let serverUrl: string; +let homeUrl: string; +let hasHomeApi: boolean; +let home: TestServer; +let docs: TestServer; +let userApi: UserAPIImpl; + +describe('DocApi', function() { + this.timeout(20000); + testUtils.setTmpLogLevel('error'); + const oldEnv = clone(process.env); + + before(async function() { + // Create the tmp dir removing any previous one + await fse.remove(tmpDir); + await fse.mkdirs(tmpDir); + log.warn(`Test logs and data are at: ${tmpDir}/`); + + // Let's create a sqlite db that we can share with servers that run in other processes, hence + // not an in-memory db. Running seed.ts directly might not take in account the most recent value + // for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different + // configuration (in-memory for instance). Spawning a process is one way to make sure that the + // latest value prevail. + process.env.TYPEORM_DATABASE = path.join(tmpDir, 'landing.db'); + const seed = await testUtils.getBuildFile('test/gen-server/seed.js'); + execFileSync('node', [seed, 'init'], { + env: process.env, + stdio: 'inherit' + }); + }); + + after(() => { + Object.assign(process.env, oldEnv); + }); + + /** + * Doc api tests are run against three different setup: + * - a merged server: a single server serving both as a home and doc worker + * - two separated servers: requests are sent to a home server which then forward them to a doc worker + * - a doc worker: request are sent directly to the doc worker (note that even though it is not + * used for testing we starts anyway a home server, needed for setting up the test cases) + * + * Future tests must be added within the testDocApi() function. + */ + + describe("should work with a merged server", async () => { + setup('merged', async () => { + home = docs = await startServer('home,docs'); + homeUrl = serverUrl = home.serverUrl; + hasHomeApi = true; + }); + testDocApi(); + }); + + // the way these tests are written, non-merged server requires redis. + if (process.env.TEST_REDIS_URL) { + describe("should work with a home server and a docworker", async () => { + setup('separated', async () => { + home = await startServer('home'); + docs = await startServer('docs', home.serverUrl); + homeUrl = serverUrl = home.serverUrl; + hasHomeApi = true; + }); + testDocApi(); + }); + + describe("should work directly with a docworker", async () => { + setup('docs', async () => { + home = await startServer('home'); + docs = await startServer('docs', home.serverUrl); + homeUrl = home.serverUrl; + serverUrl = docs.serverUrl; + hasHomeApi = false; + }); + testDocApi(); + }); + } + + describe("QueryParameters", async () => { + + function makeExample() { + return { + id: [ 1, 2, 3, 7, 8, 9 ], + color: ['red', 'yellow', 'white', 'blue', 'black', 'purple'], + spin: [ 'up', 'up', 'down', 'down', 'up', 'up'], + }; + } + + it("supports ascending sort", async function() { + assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['color']}, null), { + id: [8, 7, 9, 1, 3, 2], + color: ['black', 'blue', 'purple', 'red', 'white', 'yellow'], + spin: ['up', 'down', 'up', 'up', 'down', 'up'] + }); + }); + + it("supports descending sort", async function() { + assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-id']}, null), { + id: [9, 8, 7, 3, 2, 1], + color: ['purple', 'black', 'blue', 'white', 'yellow', 'red'], + spin: ['up', 'up', 'down', 'down', 'up', 'up'], + }); + }); + + it("supports multi-key sort", async function() { + assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-spin', 'color']}, null), { + id: [8, 9, 1, 2, 7, 3], + color: ['black', 'purple', 'red', 'yellow', 'blue', 'white'], + spin: ['up', 'up', 'up', 'up', 'down', 'down'], + }); + }); + + it("does not freak out sorting mixed data", async function() { + const example = { + id: [ 1, 2, 3, 4, 5, 6, 7, 8, 9], + mixed: ['red', 'green', 'white', 2.5, 1, null, ['zing', 3] as any, 5, 'blue'] + }; + assert.deepEqual(applyQueryParameters(example, {sort: ['mixed']}, null), { + mixed: [1, 2.5, 5, null, ['zing', 3] as any, 'blue', 'green', 'red', 'white'], + id: [5, 4, 8, 6, 7, 9, 2, 1, 3], + }); + }); + + it("supports limit", async function() { + assert.deepEqual(applyQueryParameters(makeExample(), {limit: 1}), + { id: [1], color: ['red'], spin: ['up'] }); + }); + + it("supports sort and limit", async function() { + assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-color'], limit: 2}, null), + { id: [2, 3], color: ['yellow', 'white'], spin: ['up', 'down'] }); + }); + }); +}); + +// Contains the tests. This is where you want to add more test. +function testDocApi() { + + it("guesses types of new columns", async () => { + const userActions = [ + ['AddTable', 'GuessTypes', []], + // Make 5 blank columns of type Any + ['AddColumn', 'GuessTypes', 'Date', {}], + ['AddColumn', 'GuessTypes', 'DateTime', {}], + ['AddColumn', 'GuessTypes', 'Bool', {}], + ['AddColumn', 'GuessTypes', 'Numeric', {}], + ['AddColumn', 'GuessTypes', 'Text', {}], + // Add string values from which the initial type will be guessed + ['AddRecord', 'GuessTypes', null, { + Date: "1970-01-02", + DateTime: "1970-01-02 12:00", + Bool: "true", + Numeric: "1.2", + Text: "hello", + }], + ]; + const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy); + assert.equal(resp.status, 200); + + // Check that the strings were parsed to typed values + assert.deepEqual( + (await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/records`, chimpy)).data, + { + records: [ + { + id: 1, + fields: { + Date: 24 * 60 * 60, + DateTime: 36 * 60 * 60, + Bool: true, + Numeric: 1.2, + Text: "hello", + }, + }, + ], + }, + ); + + // Check the column types + assert.deepEqual( + (await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/columns`, chimpy)) + .data.columns.map((col: any) => col.fields.type), + ["Date", "DateTime:UTC", "Bool", "Numeric", "Text"], + ); + }); + + for (const mode of ['logged in', 'anonymous']) { + for (const content of ['with content', 'without content']) { + it(`POST /api/docs ${content} creates an unsaved doc when ${mode}`, async function() { + const user = (mode === 'logged in') ? chimpy : nobody; + const formData = new FormData(); + formData.append('upload', 'A,B\n1,2\n3,4\n', 'table1.csv'); + const config = defaultsDeep({headers: formData.getHeaders()}, user); + let resp = await axios.post(`${serverUrl}/api/docs`, + ...(content === 'with content' ? [formData, config] : [null, user])); + assert.equal(resp.status, 200); + const urlId = resp.data; + if (mode === 'logged in') { + assert.match(urlId, /^new~[^~]*~[0-9]+$/); + } else { + assert.match(urlId, /^new~[^~]*$/); + } + + // Access information about that document should be sane for current user + resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, user); + assert.equal(resp.status, 200); + assert.equal(resp.data.name, 'Untitled'); + assert.equal(resp.data.workspace.name, 'Examples & Templates'); + assert.equal(resp.data.access, 'owners'); + if (mode === 'anonymous') { + resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, chimpy); + assert.equal(resp.data.access, 'owners'); + } else { + resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, charon); + assert.equal(resp.status, 403); + resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, nobody); + assert.equal(resp.status, 403); + } + + // content was successfully stored + resp = await axios.get(`${serverUrl}/api/docs/${urlId}/tables/Table1/data`, user); + if (content === 'with content') { + assert.deepEqual(resp.data, { id: [ 1, 2 ], manualSort: [ 1, 2 ], A: [ 1, 3 ], B: [ 2, 4 ] }); + } else { + assert.deepEqual(resp.data, { id: [], manualSort: [], A: [], B: [], C: [] }); + } + }); + } + } + + it("GET /docs/{did}/tables/{tid}/data retrieves data in column format", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, { + id: [1, 2, 3, 4], + A: ['hello', '', '', ''], + B: ['', 'world', '', ''], + C: ['', '', '', ''], + D: [null, null, null, null], + E: ['HELLO', '', '', ''], + manualSort: [1, 2, 3, 4] + }); + }); + + it("GET /docs/{did}/tables/{tid}/records retrieves data in records format", async function () { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/records`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, + { + records: + [ + { + id: 1, + fields: { + A: 'hello', + B: '', + C: '', + D: null, + E: 'HELLO', + }, + }, + { + id: 2, + fields: { + A: '', + B: 'world', + C: '', + D: null, + E: '', + }, + }, + { + id: 3, + fields: { + A: '', + B: '', + C: '', + D: null, + E: '', + }, + }, + { + id: 4, + fields: { + A: '', + B: '', + C: '', + D: null, + E: '', + }, + }, + ] + }); + }); + + it("GET /docs/{did}/tables/{tid}/records handles errors and hidden columns", async function () { + let resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/records`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, + { + "records": [ + { + "id": 1, + "fields": { + "A": null, + "B": "Hi", + "C": 1, + }, + "errors": { + "A": "ZeroDivisionError" + } + } + ] + } + ); + + // /data format for comparison: includes manualSort, gristHelper_Display, and ["E", "ZeroDivisionError"] + resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/data`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, + { + "id": [ + 1 + ], + "manualSort": [ + 1 + ], + "A": [ + [ + "E", + "ZeroDivisionError" + ] + ], + "B": [ + "Hi" + ], + "C": [ + 1 + ], + "gristHelper_Display": [ + "Hi" + ] + } + ); + }); + + it("GET /docs/{did}/tables/{tid}/columns retrieves columns", async function () { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/columns`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, + { + columns: [ + { + id: 'A', + fields: { + colRef: 2, + parentId: 1, + parentPos: 1, + type: 'Text', + widgetOptions: '', + isFormula: false, + formula: '', + label: 'A', + untieColIdFromLabel: false, + summarySourceCol: 0, + displayCol: 0, + visibleCol: 0, + rules: null, + recalcWhen: 0, + recalcDeps: null + } + }, + { + id: 'B', + fields: { + colRef: 3, + parentId: 1, + parentPos: 2, + type: 'Text', + widgetOptions: '', + isFormula: false, + formula: '', + label: 'B', + untieColIdFromLabel: false, + summarySourceCol: 0, + displayCol: 0, + visibleCol: 0, + rules: null, + recalcWhen: 0, + recalcDeps: null + } + }, + { + id: 'C', + fields: { + colRef: 4, + parentId: 1, + parentPos: 3, + type: 'Text', + widgetOptions: '', + isFormula: false, + formula: '', + label: 'C', + untieColIdFromLabel: false, + summarySourceCol: 0, + displayCol: 0, + visibleCol: 0, + rules: null, + recalcWhen: 0, + recalcDeps: null + } + }, + { + id: 'D', + fields: { + colRef: 5, + parentId: 1, + parentPos: 3, + type: 'Any', + widgetOptions: '', + isFormula: true, + formula: '', + label: 'D', + untieColIdFromLabel: false, + summarySourceCol: 0, + displayCol: 0, + visibleCol: 0, + rules: null, + recalcWhen: 0, + recalcDeps: null + } + }, + { + id: 'E', + fields: { + colRef: 6, + parentId: 1, + parentPos: 4, + type: 'Any', + widgetOptions: '', + isFormula: true, + formula: '$A.upper()', + label: 'E', + untieColIdFromLabel: false, + summarySourceCol: 0, + displayCol: 0, + visibleCol: 0, + rules: null, + recalcWhen: 0, + recalcDeps: null + } + } + ] + } + ); + }); + + it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent doc", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /document not found/i); + }); + + it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent table", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /table not found/i); + }); + + it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent doc", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /document not found/i); + }); + + it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent table", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /table not found/i); + }); + + it("GET /docs/{did}/tables/{tid}/data supports filters", async function() { + function makeQuery(filters: {[colId: string]: any[]}) { + const query = "filter=" + encodeURIComponent(JSON.stringify(filters)); + return axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data?${query}`, chimpy); + } + function checkResults(resp: AxiosResponse, expectedData: any) { + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, expectedData); + } + + checkResults(await makeQuery({B: ['world']}), { + id: [2], A: [''], B: ['world'], C: [''], D: [null], E: [''], manualSort: [2], + }); + + // Can query by id + checkResults(await makeQuery({id: [1]}), { + id: [1], A: ['hello'], B: [''], C: [''], D: [null], E: ['HELLO'], manualSort: [1], + }); + + checkResults(await makeQuery({B: [''], A: ['']}), { + id: [3, 4], A: ['', ''], B: ['', ''], C: ['', ''], D: [null, null], E: ['', ''], manualSort: [3, 4], + }); + + // Empty filter is equivalent to no filter and should return full data. + checkResults(await makeQuery({}), { + id: [1, 2, 3, 4], + A: ['hello', '', '', ''], + B: ['', 'world', '', ''], + C: ['', '', '', ''], + D: [null, null, null, null], + E: ['HELLO', '', '', ''], + manualSort: [1, 2, 3, 4] + }); + + // An impossible filter should succeed but return an empty set of rows. + checkResults(await makeQuery({B: ['world'], C: ['Neptune']}), { + id: [], A: [], B: [], C: [], D: [], E: [], manualSort: [], + }); + + // An invalid filter should return an error + { + const resp = await makeQuery({BadCol: ['']}); + assert.equal(resp.status, 400); + assert.match(resp.data.error, /BadCol/); + } + + { + const resp = await makeQuery({B: 'world'} as any); + assert.equal(resp.status, 400); + assert.match(resp.data.error, /filter values must be arrays/); + } + }); + + for (const mode of ['url', 'header']) { + it(`GET /docs/{did}/tables/{tid}/data supports sorts and limits in ${mode}`, async function() { + function makeQuery(sort: string[]|null, limit: number|null) { + const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`); + const config = configForUser('chimpy'); + if (mode === 'url') { + if (sort) { url.searchParams.append('sort', sort.join(',')); } + if (limit) { url.searchParams.append('limit', String(limit)); } + } else { + if (sort) { config.headers['x-sort'] = sort.join(','); } + if (limit) { config.headers['x-limit'] = String(limit); } + } + return axios.get(url.href, config); + } + function checkResults(resp: AxiosResponse, expectedData: any) { + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, expectedData); + } + + checkResults(await makeQuery(['-id'], null), { + id: [4, 3, 2, 1], + A: ['', '', '', 'hello'], + B: ['', '', 'world', ''], + C: ['', '', '', ''], + D: [null, null, null, null], + E: ['', '', '', 'HELLO'], + manualSort: [4, 3, 2, 1] + }); + + checkResults(await makeQuery(['-id'], 2), { + id: [4, 3], + A: ['', ''], + B: ['', ''], + C: ['', ''], + D: [null, null], + E: ['', ''], + manualSort: [4, 3] + }); + }); + } + + it("GET /docs/{did}/tables/{tid}/data respects document permissions", async function() { + // as not part of any group kiwi cannot fetch Timesheets + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, kiwi); + assert.equal(resp.status, 403); + }); + + it("GET /docs/{did}/tables/{tid}/data returns matches /not found/ for bad table id", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /not found/); + }); + + it("POST /docs/{did}/apply applies user actions", async function() { + const userActions = [ + ['AddTable', 'Foo', [{id: 'A'}, {id: 'B'}]], + ['BulkAddRecord', 'Foo', [1, 2], {A: ["Santa", "Bob"], B: [1, 11]}] + ]; + const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual( + (await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, + {id: [1, 2], A: ['Santa', 'Bob'], B: ['1', '11'], manualSort: [1, 2]}); + }); + + it("POST /docs/{did}/apply respects document permissions", async function() { + const userActions = [ + ['AddTable', 'FooBar', [{id: 'A'}]] + ]; + let resp: AxiosResponse; + + // as a guest chimpy cannot edit Bananas + resp = await axios.post(`${serverUrl}/api/docs/${docIds.Bananas}/apply`, userActions, chimpy); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {error: 'No write access'}); + + // check that changes did not apply + resp = await axios.get(`${serverUrl}/api/docs/${docIds.Bananas}/tables/FooBar/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /not found/); + + // as not in any group kiwi cannot edit TestDoc + resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, kiwi); + assert.equal(resp.status, 403); + + // check that changes did not apply + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/FooBar/data`, chimpy); + assert.equal(resp.status, 404); + assert.match(resp.data.error, /not found/); + + }); + + it("POST /docs/{did}/tables/{tid}/data adds records", async function() { + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, { + A: ['Alice', 'Felix'], + B: [2, 22] + }, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, [3, 4]); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy); + assert.deepEqual(resp.data, { + id: [1, 2, 3, 4], + A: ['Santa', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + }); + + it("POST /docs/{did}/tables/{tid}/records adds records", async function() { + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, { + records: [ + {fields: {A: 'John', B: 55}}, + {fields: {A: 'Jane', B: 0}}, + ] + }, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, { + records: [ + {id: 5}, + {id: 6}, + ] + }); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, + { + records: + [ + { + id: 1, + fields: { + A: 'Santa', + B: '1', + }, + }, + { + id: 2, + fields: { + A: 'Bob', + B: '11', + }, + }, + { + id: 3, + fields: { + A: 'Alice', + B: '2', + }, + }, + { + id: 4, + fields: { + A: 'Felix', + B: '22', + }, + }, + { + id: 5, + fields: { + A: 'John', + B: '55', + }, + }, + { + id: 6, + fields: { + A: 'Jane', + B: '0', + }, + }, + ] + }); + }); + + it("POST /docs/{did}/tables/{tid}/data/delete deletes records", async function() { + let resp = await axios.post( + `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data/delete`, + [3, 4, 5, 6], + chimpy, + ); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, null); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy); + assert.deepEqual(resp.data, { + id: [1, 2], + A: ['Santa', 'Bob'], + B: ["1", "11"], + manualSort: [1, 2] + }); + + // restore rows + await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, { + A: ['Alice', 'Felix'], + B: [2, 22] + }, chimpy); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy); + assert.deepEqual(resp.data, { + id: [1, 2, 3, 4], + A: ['Santa', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + }); + + function checkError(status: number, test: RegExp|object, resp: AxiosResponse, message?: string) { + assert.equal(resp.status, status); + if (test instanceof RegExp) { + assert.match(resp.data.error, test, message); + } else { + try { + assert.deepEqual(resp.data, test, message); + } catch(err) { + console.log(JSON.stringify(resp.data)); + console.log(JSON.stringify(test)); + throw err; + } + } + } + + it("parses strings in user actions", async () => { + // Create a test document. + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const docId = await userApi.newDoc({name: 'testdoc'}, ws1); + const docUrl = `${serverUrl}/api/docs/${docId}`; + const recordsUrl = `${docUrl}/tables/Table1/records`; + + // Make the column numeric, delete the other columns we don't care about + await axios.post(`${docUrl}/apply`, [ + ['ModifyColumn', 'Table1', 'A', {type: 'Numeric'}], + ['RemoveColumn', 'Table1', 'B'], + ['RemoveColumn', 'Table1', 'C'], + ], chimpy); + + // Add/update some records without and with string parsing + // Specifically test: + // 1. /apply, with an AddRecord + // 2. POST /records (BulkAddRecord) + // 3. PATCH /records (BulkUpdateRecord) + // Send strings that look like currency which need string parsing to become numbers + for (const queryParams of ['?noparse=1', '']) { + await axios.post(`${docUrl}/apply${queryParams}`, [ + ['AddRecord', 'Table1', null, {'A': '$1'}], + ], chimpy); + + const response = await axios.post(`${recordsUrl}${queryParams}`, + { + records: [ + {fields: {'A': '$2'}}, + {fields: {'A': '$3'}}, + ] + }, + chimpy); + + // Update $3 -> $4 + const rowId = response.data.records[1].id; + await axios.patch(`${recordsUrl}${queryParams}`, + { + records: [ + {id: rowId, fields: {'A': '$4'}} + ] + }, + chimpy); + } + + // Check the results + const resp = await axios.get(recordsUrl, chimpy); + assert.deepEqual(resp.data, { + records: + [ + // Without string parsing + {id: 1, fields: {A: '$1'}}, + {id: 2, fields: {A: '$2'}}, + {id: 3, fields: {A: '$4'}}, + + // With string parsing + {id: 4, fields: {A: 1}}, + {id: 5, fields: {A: 2}}, + {id: 6, fields: {A: 4}}, + ] + } + ); + }); + + describe("PUT /docs/{did}/tables/{tid}/records", async function() { + it("should add or update records", async function() { + // create sample document for testing + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const docId = await userApi.newDoc({name: 'BlankTest'}, wid); + const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`; + + async function check(records: AddOrUpdateRecord[], expectedTableData: BulkColValues, params: any={}) { + const resp = await axios.put(url, {records}, {...chimpy, params}); + assert.equal(resp.status, 200); + const table = await userApi.getTable(docId, "Table1"); + delete table.manualSort; + delete table.C; + assert.deepStrictEqual(table, expectedTableData); + } + + // Add 3 new records, since the table is empty so nothing matches `requires` + await check( + [ + { + require: {A: 1}, + }, + { + // Since no record with A=2 is found, create a new record, + // but `fields` overrides `require` for the value when creating, + // so the new record has A=3 + require: {A: 2}, + fields: {A: 3}, + }, + { + require: {A: 4}, + fields: {B: 5}, + }, + ], + {id: [1, 2, 3], A: [1, 3, 4], B: [0, 0, 5]} + ); + + // Update all three records since they all match the `require` values here + await check( + [ + { + // Does nothing + require: {A: 1}, + }, + { + // Changes A from 3 to 33 + require: {A: 3}, + fields: {A: 33}, + }, + { + // Changes B from 5 to 6 in the third record where A=4 + require: {A: 4}, + fields: {B: 6}, + }, + ], + {id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]} + ); + + // This would normally add a record, but noadd suppresses that + await check([ + { + require: {A: 100}, + }, + ], + {id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]}, + {noadd: "1"}, + ); + + // This would normally update A from 1 to 11, bot noupdate suppresses that + await check([ + { + require: {A: 1}, + fields: {A: 11}, + }, + ], + {id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]}, + {noupdate: "1"}, + ); + + // There are 2 records with B=0, update them both to B=1 + // Use onmany=all to specify that they should both be updated + await check([ + { + require: {B: 0}, + fields: {B: 1}, + }, + ], + {id: [1, 2, 3], A: [1, 33, 4], B: [1, 1, 6]}, + {onmany: "all"} + ); + + // In contrast to the above, the default behaviour for no value of onmany + // is to only update the first matching record, + // so only one of the records with B=1 is updated to B=2 + await check([ + { + require: {B: 1}, + fields: {B: 2}, + }, + ], + {id: [1, 2, 3], A: [1, 33, 4], B: [2, 1, 6]}, + ); + + // By default, strings in `require` and `fields` are parsed based on column type, + // so these dollar amounts are treated as currency + // and parsed as A=4 and A=44 + await check([ + { + require: {A: "$4"}, + fields: {A: "$44"}, + }, + ], + {id: [1, 2, 3], A: [1, 33, 44], B: [2, 1, 6]}, + ); + + // Turn off the default string parsing with noparse=1 + // Now we need A=44 to actually be a number to match, + // A="$44" wouldn't match and would create a new record. + // Because A="$55" isn't parsed, the raw string is stored in the table. + await check([ + { + require: {A: 44}, + fields: {A: "$55"}, + }, + ], + {id: [1, 2, 3], A: [1, 33, "$55"], B: [2, 1, 6]}, + {noparse: 1} + ); + + await check([ + // First three records already exist and nothing happens + {require: {A: 1}}, + {require: {A: 33}}, + {require: {A: "$55"}}, + // Without string parsing, A="$33" doesn't match A=33 and a new record is created + {require: {A: "$33"}}, + ], + {id: [1, 2, 3, 4], A: [1, 33, "$55", "$33"], B: [2, 1, 6, 0]}, + {noparse: 1} + ); + + // Checking that updating by `id` works. + await check([ + { + require: {id: 3}, + fields: {A: "66"}, + }, + ], + {id: [1, 2, 3, 4], A: [1, 33, 66, "$33"], B: [2, 1, 6, 0]}, + ); + + // allow_empty_require option with empty `require` updates all records + await check([ + { + require: {}, + fields: {A: 99, B: 99}, + }, + ], + {id: [1, 2, 3, 4], A: [99, 99, 99, 99], B: [99, 99, 99, 99]}, + {allow_empty_require: "1", onmany: "all"}, + ); + }); + + it("should 404 for missing tables", async () => { + checkError(404, /Table not found "Bad_Foo_"/, + await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/records`, + {records: [{require: {id: 1}}]}, chimpy)); + }); + + it("should 400 for missing columns", async () => { + checkError(400, /Invalid column "no_such_column"/, + await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, + {records: [{require: {no_such_column: 1}}]}, chimpy)); + }); + + it("should 400 for an incorrect onmany parameter", async function() { + checkError(400, + /onmany parameter foo should be one of first,none,all/, + await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, + {records: [{require: {id: 1}}]}, {...chimpy, params: {onmany: "foo"}})); + }); + + it("should 400 for an empty require without allow_empty_require", async function() { + checkError(400, + /require is empty but allow_empty_require isn't set/, + await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, + {records: [{require: {}}]}, chimpy)); + }); + + it("should validate request schema", async function() { + const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`; + const test = async (payload: any, error: { error: string, details: string }) => { + const resp = await axios.put(url, payload, chimpy); + checkError(400, error, resp); + }; + await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'}); + await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'}); + await test({records: [{fields: {}}]}, + { + error: 'Invalid payload', + details: 'Error: ' + + 'body.records[0] is not a AddOrUpdateRecord; ' + + 'body.records[0].require is missing', + }); + await test({records: [{require: {id: "1"}}]}, + { + error: 'Invalid payload', + details: 'Error: ' + + 'body.records[0] is not a AddOrUpdateRecord; ' + + 'body.records[0].require.id is not a number', + }); + }); + }); + + describe("POST /docs/{did}/tables/{tid}/records", async function() { + it("POST should have good errors", async () => { + checkError(404, /not found/, + await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`, + { A: ['Alice', 'Felix'], B: [2, 22] }, chimpy)); + + checkError(400, /Invalid column "Bad"/, + await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, + { A: ['Alice'], Bad: ['Monthy'] }, chimpy)); + + // Other errors should also be maximally informative. + checkError(400, /Error manipulating data/, + await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, + { A: ['Alice'], B: null }, chimpy)); + }); + + it("validates request schema", async function() { + const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`; + const test = async(payload: any, error: {error: string, details: string}) => { + const resp = await axios.post(url, payload, chimpy); + checkError(400, error, resp); + }; + await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'}); + await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'}); + // All column types are allowed, except Arrays (or objects) without correct code. + const testField = async (A: any) => { + await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details: + 'Error: body.records[0] is not a NewRecord; '+ + 'body.records[0].fields.A is not a CellValue; '+ + 'body.records[0].fields.A is none of number, '+ + 'string, boolean, null, 1 more; body.records[0].'+ + 'fields.A[0] is not a GristObjCode; body.records[0]'+ + '.fields.A[0] is not a valid enum value'}); + }; + // test no code at all + await testField([]); + // test invalid code + await testField(['ZZ']); + }); + + it("allows to create a blank record", async function() { + // create sample document for testing + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const docId = await userApi.newDoc({ name : 'BlankTest'}, wid); + // Create two blank records + const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`; + const resp = await axios.post(url, {records: [{}, { fields: {}}]}, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, { records : [{id: 1}, {id: 2}]}); + }); + + it("allows to create partial records", async function() { + // create sample document for testing + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const docId = await userApi.newDoc({ name : 'BlankTest'}, wid); + const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`; + // create partial records + const resp = await axios.post(url, {records: [{fields: { A: 1}}, { fields: {B: 2}}, {}]}, chimpy); + assert.equal(resp.status, 200); + const table = await userApi.getTable(docId, "Table1"); + delete table.manualSort; + assert.deepStrictEqual( + table, + { id: [1, 2, 3], A: [1, null, null], B: [null, 2, null], C:[null, null, null]}); + }); + + it("allows CellValue as a field", async function() { + // create sample document + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const docId = await userApi.newDoc({ name : 'PostTest'}, wid); + const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`; + const testField = async(A?: CellValue, message?: string) =>{ + const resp = await axios.post(url, {records: [{ fields: { A } }]}, chimpy); + assert.equal(resp.status, 200, message ?? `Error for code ${A}`); + }; + // test allowed types for a field + await testField(1); // ints + await testField(1.2); // floats + await testField("string"); // strings + await testField(true); // true and false + await testField(false); + await testField(null); // null + // encoded values (though not all make sense) + for (const code of [ + GristObjCode.List, + GristObjCode.Dict, + GristObjCode.DateTime, + GristObjCode.Date, + GristObjCode.Skip, + GristObjCode.Censored, + GristObjCode.Reference, + GristObjCode.ReferenceList, + GristObjCode.Exception, + GristObjCode.Pending, + GristObjCode.Unmarshallable, + GristObjCode.Versions, + ]) { + await testField([code]); + } + }); + }); + + it("POST /docs/{did}/tables/{tid}/data respects document permissions", async function() { + let resp: AxiosResponse; + const data = { + A: ['Alice', 'Felix'], + B: [2, 22] + }; + + // as a viewer charon cannot edit TestDoc + resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {error: 'No write access'}); + + // as not part of any group kiwi cannot edit TestDoc + resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {error: 'No view access'}); + + // check that TestDoc did not change + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy); + assert.deepEqual(resp.data, { + id: [1, 2, 3, 4], + A: ['Santa', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + }); + + describe("PATCH /docs/{did}/tables/{tid}/records", function() { + it("updates records", async function () { + let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, { + records: [ + { + id: 1, + fields: { + A: 'Father Christmas', + }, + }, + ], + }, chimpy); + assert.equal(resp.status, 200); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy); + // check that rest of the data is left unchanged + assert.deepEqual(resp.data, { + records: + [ + { + id: 1, + fields: { + A: 'Father Christmas', + B: '1', + }, + }, + { + id: 2, + fields: { + A: 'Bob', + B: '11', + }, + }, + { + id: 3, + fields: { + A: 'Alice', + B: '2', + }, + }, + { + id: 4, + fields: { + A: 'Felix', + B: '22', + }, + }, + ] + }); + }); + + it("validates request schema", async function() { + const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`; + async function failsWithError(payload: any, error: { error: string, details?: string }){ + const resp = await axios.patch(url, payload, chimpy); + checkError(400, error, resp); + } + + await failsWithError({}, {error: 'Invalid payload', details: 'Error: body.records is missing'}); + + await failsWithError({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'}); + + await failsWithError({records: []}, {error: 'Invalid payload', details: + 'Error: body.records[0] is not a Record; body.records[0] is not an object'}); + + await failsWithError({records: [{}]}, {error: 'Invalid payload', details: + 'Error: body.records[0] is not a Record\n '+ + 'body.records[0].id is missing\n '+ + 'body.records[0].fields is missing'}); + + await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details: + 'Error: body.records[0] is not a Record\n' + + ' body.records[0].id is not a number\n' + + ' body.records[0].fields is missing'}); + + await failsWithError( + {records: [{id: 1, fields: {A : 1}}, {id: 2, fields: {B: 3}}]}, + {error: 'PATCH requires all records to have same fields'}); + + // Test invalid object codes + const fieldIsNotValid = async (A: any) => { + await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details: + 'Error: body.records[0] is not a Record; '+ + 'body.records[0].fields.A is not a CellValue; '+ + 'body.records[0].fields.A is none of number, '+ + 'string, boolean, null, 1 more; body.records[0].'+ + 'fields.A[0] is not a GristObjCode; body.records[0]'+ + '.fields.A[0] is not a valid enum value'}); + }; + await fieldIsNotValid([]); + await fieldIsNotValid(['ZZ']); + }); + + it("allows CellValue as a field", async function() { + // create sample document for testing + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const docId = await userApi.newDoc({ name : 'PatchTest'}, wid); + const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`; + // create record for patching + const id = (await axios.post(url, { records: [{}] }, chimpy)).data.records[0].id; + const testField = async(A?: CellValue, message?: string) =>{ + const resp = await axios.patch(url, {records: [{ id, fields: { A } }]}, chimpy); + assert.equal(resp.status, 200, message ?? `Error for code ${A}`); + }; + await testField(1); + await testField(1.2); + await testField("string"); + await testField(true); + await testField(false); + await testField(null); + for (const code of [ + GristObjCode.List, + GristObjCode.Dict, + GristObjCode.DateTime, + GristObjCode.Date, + GristObjCode.Skip, + GristObjCode.Censored, + GristObjCode.Reference, + GristObjCode.ReferenceList, + GristObjCode.Exception, + GristObjCode.Pending, + GristObjCode.Unmarshallable, + GristObjCode.Versions, + ]) { + await testField([code]); + } + }); + }); + + describe("PATCH /docs/{did}/tables/{tid}/data", function() { + + it("updates records", async function() { + let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, { + id: [1], + A: ['Santa Klaus'], + }, chimpy); + assert.equal(resp.status, 200); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy); + // check that rest of the data is left unchanged + assert.deepEqual(resp.data, { + id: [1, 2, 3, 4], + A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + + }); + + it("throws 400 for invalid row ids", async function() { + + // combination of valid and invalid ids fails + let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, { + id: [1, 5], + A: ['Alice', 'Felix'] + }, chimpy); + assert.equal(resp.status, 400); + assert.match(resp.data.error, /Invalid row id 5/); + + // only invalid ids also fails + resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, { + id: [10, 5], + A: ['Alice', 'Felix'] + }, chimpy); + assert.equal(resp.status, 400); + assert.match(resp.data.error, /Invalid row id 10/); + + // check that changes related to id 1 did not apply + assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, { + id: [1, 2, 3, 4], + A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + }); + + it("throws 400 for invalid column", async function() { + const resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, { + id: [1], + A: ['Alice'], + C: ['Monthy'] + }, chimpy); + assert.equal(resp.status, 400); + assert.match(resp.data.error, /Invalid column "C"/); + }); + + it("respects document permissions", async function() { + let resp: AxiosResponse; + const data = { + id: [1], + A: ['Santa'], + }; + + // check data + assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, { + id: [1, 2, 3, 4], + A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + + // as a viewer charon cannot patch TestDoc + resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {error: 'No write access'}); + + // as not part of any group kiwi cannot patch TestDoc + resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {error: 'No view access'}); + + // check that changes did not apply + assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, { + id: [1, 2, 3, 4], + A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'], + B: ["1", "11", "2", "22"], + manualSort: [1, 2, 3, 4] + }); + }); + + }); + + describe('attachments', function() { + it("POST /docs/{did}/attachments adds attachments", async function() { + let formData = new FormData(); + formData.append('upload', 'foobar', "hello.doc"); + formData.append('upload', '123456', "world.jpg"); + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData, + defaultsDeep({headers: formData.getHeaders()}, chimpy)); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, [1, 2]); + + // Another upload gets the next number. + formData = new FormData(); + formData.append('upload', 'abcdef', "hello.png"); + resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData, + defaultsDeep({headers: formData.getHeaders()}, chimpy)); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, [3]); + }); + + it("GET /docs/{did}/attachments/{id} returns attachment metadata", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2`, chimpy); + assert.equal(resp.status, 200); + assert.include(resp.data, {fileName: "world.jpg", fileSize: 6}); + assert.match(resp.data.timeUploaded, /^\d{4}-\d{2}-\d{2}T/); + }); + + it("GET /docs/{did}/attachments/{id}/download downloads attachment contents", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`, + {...chimpy, responseType: 'arraybuffer'}); + assert.equal(resp.status, 200); + assert.deepEqual(resp.headers['content-type'], 'image/jpeg'); + assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"'); + assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600'); + assert.deepEqual(resp.data, Buffer.from('123456')); + }); + + it("GET /docs/{did}/attachments/{id}/download works after doc shutdown", async function() { + // Check that we can download when ActiveDoc isn't currently open. + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/force-reload`, null, chimpy); + assert.equal(resp.status, 200); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`, + {...chimpy, responseType: 'arraybuffer'}); + assert.equal(resp.status, 200); + assert.deepEqual(resp.headers['content-type'], 'image/jpeg'); + assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"'); + assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600'); + assert.deepEqual(resp.data, Buffer.from('123456')); + }); + + it("GET /docs/{did}/attachments/{id}... returns 404 when attachment not found", async function() { + let resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22`, chimpy); + checkError(404, /Attachment not found: 22/, resp); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo`, chimpy); + checkError(404, /Attachment not found: moo/, resp); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22/download`, chimpy); + checkError(404, /Attachment not found: 22/, resp); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo/download`, chimpy); + checkError(404, /Attachment not found: moo/, resp); + }); + + it("POST /docs/{did}/attachments produces reasonable errors", async function() { + // Check that it produces reasonable errors if we try to use it with non-form-data + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, [4, 5, 6], chimpy); + assert.equal(resp.status, 415); // Wrong content-type + + // Check for an error if there is no data included. + const formData = new FormData(); + resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData, + defaultsDeep({headers: formData.getHeaders()}, chimpy)); + assert.equal(resp.status, 400); + // TODO The error here is "stream ended unexpectedly", which isn't really reasonable. + }); + + it("POST/GET /docs/{did}/attachments respect document permissions", async function() { + const formData = new FormData(); + formData.append('upload', 'xyzzz', "wrong.png"); + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData, + defaultsDeep({headers: formData.getHeaders()}, kiwi)); + checkError(403, /No view access/, resp); + + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3`, kiwi); + checkError(403, /No view access/, resp); + + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3/download`, kiwi); + checkError(403, /No view access/, resp); + }); + + it("POST /docs/{did}/attachments respects untrusted content-type only if valid", async function() { + const formData = new FormData(); + formData.append('upload', 'xyz', {filename: "foo", contentType: "application/pdf"}); + formData.append('upload', 'abc', {filename: "hello.png", contentType: "invalid/content-type"}); + formData.append('upload', 'def', {filename: "world.doc", contentType: "text/plain\nbad-header: 1\n\nEvil"}); + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData, + defaultsDeep({headers: formData.getHeaders()}, chimpy)); + assert.equal(resp.status, 200); + assert.deepEqual(resp.data, [4, 5, 6]); + + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/4/download`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.headers['content-type'], 'application/pdf'); // A valid content-type is respected + assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="foo.pdf"'); + assert.deepEqual(resp.data, 'xyz'); + + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/5/download`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.headers['content-type'], 'image/png'); // Did not pay attention to invalid header + assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="hello.png"'); + assert.deepEqual(resp.data, 'abc'); + + resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/6/download`, chimpy); + assert.equal(resp.status, 200); + assert.deepEqual(resp.headers['content-type'], 'application/msword'); // Another invalid header ignored + assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.doc"'); + assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600'); + assert.deepEqual(resp.headers['bad-header'], undefined); // Attempt to hack in more headers didn't work + assert.deepEqual(resp.data, 'def'); + }); + }); + + it("GET /docs/{did}/download serves document", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, chimpy); + assert.equal(resp.status, 200); + assert.match(resp.data, /grist_Tables_column/); + }); + + it("GET /docs/{did}/download respects permissions", async function() { + // kiwi has no access to TestDoc + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, kiwi); + assert.equal(resp.status, 403); + assert.notMatch(resp.data, /grist_Tables_column/); + }); + + it("GET /docs/{did}/download/csv serves CSV-encoded document", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy); + assert.equal(resp.status, 200); + assert.equal(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n'); + + const resp2 = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Foo`, chimpy); + assert.equal(resp2.status, 200); + assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n'); + }); + + it("GET /docs/{did}/download/csv respects permissions", async function() { + // kiwi has no access to TestDoc + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi); + assert.equal(resp.status, 403); + assert.notEqual(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n'); + }); + + it("GET /docs/{did}/download/csv returns 404 if tableId is invalid", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=MissingTableId`, chimpy); + assert.equal(resp.status, 404); + assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' }); + }); + + it("GET /docs/{did}/download/csv returns 404 if viewSectionId is invalid", async function() { + const resp = await axios.get( + `${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1&viewSection=9999`, chimpy); + assert.equal(resp.status, 404); + assert.deepEqual(resp.data, { error: 'No record 9999 in table _grist_Views_section' }); + }); + + it("GET /docs/{did}/download/csv returns 400 if tableId is missing", async function() { + const resp = await axios.get( + `${serverUrl}/api/docs/${docIds.TestDoc}/download/csv`, chimpy); + assert.equal(resp.status, 400); + assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' }); + }); + + it('POST /workspaces/{wid}/import handles empty filenames', async function() { + if (!process.env.TEST_REDIS_URL) { this.skip(); } + const worker1 = await userApi.getWorkerAPI('import'); + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const fakeData1 = await testUtils.readFixtureDoc('Hello.grist'); + const uploadId1 = await worker1.upload(fakeData1, '.grist'); + const resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1}, + configForUser('Chimpy')); + assert.equal(resp.status, 200); + assert.equal(resp.data.title, 'Untitled upload'); + assert.equal(typeof resp.data.id, 'string'); + assert.notEqual(resp.data.id, ''); + }); + + it("document is protected during upload-and-import sequence", async function() { + if (!process.env.TEST_REDIS_URL) { this.skip(); } + // Prepare an API for a different user. + const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, { + headers: {Authorization: 'Bearer api_key_for_kiwi'}, + fetch : fetch as any, + newFormData: () => new FormData() as any, + logger: log + }); + // upload something for Chimpy and something else for Kiwi. + const worker1 = await userApi.getWorkerAPI('import'); + const fakeData1 = await testUtils.readFixtureDoc('Hello.grist'); + const uploadId1 = await worker1.upload(fakeData1, 'upload.grist'); + const worker2 = await kiwiApi.getWorkerAPI('import'); + const fakeData2 = await testUtils.readFixtureDoc('Favorite_Films.grist'); + const uploadId2 = await worker2.upload(fakeData2, 'upload2.grist'); + + // Check that kiwi only has access to their own upload. + let wid = (await kiwiApi.getOrgWorkspaces('current')).find((w) => w.name === 'Big')!.id; + let resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1}, + configForUser('Kiwi')); + assert.equal(resp.status, 403); + assert.deepEqual(resp.data, {error: "access denied"}); + + resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId2}, + configForUser('Kiwi')); + assert.equal(resp.status, 200); + + // Check that chimpy has access to their own upload. + wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1}, + configForUser('Chimpy')); + assert.equal(resp.status, 200); + }); + + it('limits parallel requests', async function() { + // Launch 30 requests in parallel and see how many are honored and how many + // return 429s. The timing of this test is a bit delicate. We close the doc + // to increase the odds that results won't start coming back before all the + // requests have passed authorization. May need to do something more sophisticated + // if this proves unreliable. + await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy); + const reqs = [...Array(30).keys()].map( + i => axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy)); + const responses = await Promise.all(reqs); + assert.lengthOf(responses.filter(r => r.status === 200), 10); + assert.lengthOf(responses.filter(r => r.status === 429), 20); + }); + + it('allows forced reloads', async function() { + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy); + assert.equal(resp.status, 200); + // Check that support cannot force a reload. + resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, support); + assert.equal(resp.status, 403); + if (hasHomeApi) { + // Check that support can force a reload through housekeeping api. + resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, support); + assert.equal(resp.status, 200); + // Check that regular user cannot force a reload through housekeeping api. + resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, chimpy); + assert.equal(resp.status, 403); + } + }); + + it('allows assignments', async function() { + let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, chimpy); + assert.equal(resp.status, 200); + // Check that support cannot force an assignment. + resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, support); + assert.equal(resp.status, 403); + if (hasHomeApi) { + // Check that support can force an assignment through housekeeping api. + resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, support); + assert.equal(resp.status, 200); + // Check that regular user cannot force an assignment through housekeeping api. + resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, chimpy); + assert.equal(resp.status, 403); + } + }); + + it('honors urlIds', async function() { + // Make a document with a urlId + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid1'}, ws1); + try { + // Make sure an edit made by docId is visible when accessed via docId or urlId + let resp = await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, { + A: ['Apple'], B: [99] + }, chimpy); + resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[0], 'Apple'); + resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[0], 'Apple'); + // Make sure an edit made by urlId is visible when accessed via docId or urlId + resp = await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, { + A: ['Orange'], B: [42] + }, chimpy); + resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[1], 'Orange'); + resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[1], 'Orange'); + } finally { + await userApi.deleteDoc(doc1); + } + }); + + it('filters urlIds by org', async function() { + // Make two documents with same urlId + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1); + const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, { + headers: {Authorization: 'Bearer api_key_for_chimpy'}, + fetch : fetch as any, + newFormData: () => new FormData() as any, + logger: log + }); + const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id; + const doc2 = await nasaApi.newDoc({name: 'testdoc2', urlId: 'urlid'}, ws2); + try { + // Place a value in "docs" doc + await axios.post(`${serverUrl}/o/docs/api/docs/urlid/tables/Table1/data`, { + A: ['Apple'], B: [99] + }, chimpy); + // Place a value in "nasa" doc + await axios.post(`${serverUrl}/o/nasa/api/docs/urlid/tables/Table1/data`, { + A: ['Orange'], B: [99] + }, chimpy); + // Check the values made it to the right places + let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[0], 'Apple'); + resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[0], 'Orange'); + } finally { + await userApi.deleteDoc(doc1); + await nasaApi.deleteDoc(doc2); + } + }); + + it('allows docId access to any document from merged org', async function() { + // Make two documents + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1); + const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, { + headers: {Authorization: 'Bearer api_key_for_chimpy'}, + fetch : fetch as any, + newFormData: () => new FormData() as any, + logger: log + }); + const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id; + const doc2 = await nasaApi.newDoc({name: 'testdoc2'}, ws2); + try { + // Should fail to write to a document in "docs" from "nasa" url + let resp = await axios.post(`${serverUrl}/o/nasa/api/docs/${doc1}/tables/Table1/data`, { + A: ['Apple'], B: [99] + }, chimpy); + assert.equal(resp.status, 404); + // Should successfully write to a document in "nasa" from "docs" url + resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/tables/Table1/data`, { + A: ['Orange'], B: [99] + }, chimpy); + assert.equal(resp.status, 200); + // Should fail to write to a document in "nasa" from "pr" url + resp = await axios.post(`${serverUrl}/o/pr/api/docs/${doc2}/tables/Table1/data`, { + A: ['Orange'], B: [99] + }, chimpy); + assert.equal(resp.status, 404); + } finally { + await userApi.deleteDoc(doc1); + await nasaApi.deleteDoc(doc2); + } + }); + + it("GET /docs/{did}/replace replaces one document with another", async function() { + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1); + const doc2 = await userApi.newDoc({name: 'testdoc2'}, ws1); + const doc3 = await userApi.newDoc({name: 'testdoc2'}, ws1); + await userApi.updateDocPermissions(doc2, {users: {'kiwi@getgrist.com': 'editors'}}); + await userApi.updateDocPermissions(doc3, {users: {'kiwi@getgrist.com': 'viewers'}}); + try { + // Put some material in doc3 + let resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc3}/tables/Table1/data`, { + A: ['Orange'] + }, chimpy); + assert.equal(resp.status, 200); + + // Kiwi can replace doc2 with doc3 + resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, { + sourceDocId: doc3 + }, kiwi); + assert.equal(resp.status, 200); + resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy); + assert.equal(resp.data.A[0], 'Orange'); + + // Kiwi can't replace doc1 with doc3, no write access to doc1 + resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc1}/replace`, { + sourceDocId: doc3 + }, kiwi); + assert.equal(resp.status, 403); + + // Kiwi can't replace doc2 with doc1, no read access to doc1 + resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, { + sourceDocId: doc1 + }, kiwi); + assert.equal(resp.status, 403); + } finally { + await userApi.deleteDoc(doc1); + await userApi.deleteDoc(doc2); + } + }); + + it("GET /docs/{did}/snapshots retrieves a list of snapshots", async function() { + const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/snapshots`, chimpy); + assert.equal(resp.status, 200); + assert.isAtLeast(resp.data.snapshots.length, 1); + assert.hasAllKeys(resp.data.snapshots[0], ['docId', 'lastModified', 'snapshotId']); + }); + + it("POST /docs/{did}/states/remove removes old states", async function() { + // Check doc has plenty of states. + let resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy); + assert.equal(resp.status, 200); + const states: DocState[] = resp.data.states; + assert.isAbove(states.length, 5); + + // Remove all but 3. + resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 3}, chimpy); + assert.equal(resp.status, 200); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy); + assert.equal(resp.status, 200); + assert.lengthOf(resp.data.states, 3); + assert.equal(resp.data.states[0].h, states[0].h); + assert.equal(resp.data.states[1].h, states[1].h); + assert.equal(resp.data.states[2].h, states[2].h); + + // Remove all but 1. + resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 1}, chimpy); + assert.equal(resp.status, 200); + resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy); + assert.equal(resp.status, 200); + assert.lengthOf(resp.data.states, 1); + assert.equal(resp.data.states[0].h, states[0].h); + }); + + it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function() { + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1); + const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1); + const doc1 = userApi.getDocAPI(docId1); + const doc2 = userApi.getDocAPI(docId2); + + // Stick some content in column A so it has a defined type + // so diffs are smaller and simpler. + await doc2.addRows('Table1', {A: [0]}); + + let comp = await doc1.compareDoc(docId2); + assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary']); + assert.equal(comp.summary, 'unrelated'); + assert.equal(comp.parent, null); + assert.hasAllKeys(comp.left, ['n', 'h']); + assert.hasAllKeys(comp.right, ['n', 'h']); + assert.equal(comp.left.n, 1); + assert.equal(comp.right.n, 2); + + await doc1.replace({sourceDocId: docId2}); + + comp = await doc1.compareDoc(docId2); + assert.equal(comp.summary, 'same'); + assert.equal(comp.left.n, 2); + assert.deepEqual(comp.left, comp.right); + assert.deepEqual(comp.left, comp.parent); + assert.equal(comp.details, undefined); + + comp = await doc1.compareDoc(docId2, { detail: true }); + assert.deepEqual(comp.details, { + leftChanges: { tableRenames: [], tableDeltas: {} }, + rightChanges: { tableRenames: [], tableDeltas: {} } + }); + + await doc1.addRows('Table1', {A: [1]}); + comp = await doc1.compareDoc(docId2); + assert.equal(comp.summary, 'left'); + assert.equal(comp.left.n, 3); + assert.equal(comp.right.n, 2); + assert.deepEqual(comp.right, comp.parent); + assert.equal(comp.details, undefined); + + comp = await doc1.compareDoc(docId2, { detail: true }); + assert.deepEqual(comp.details!.rightChanges, + { tableRenames: [], tableDeltas: {} }); + const addA1: ActionSummary = { + tableRenames: [], + tableDeltas: { Table1: { + updateRows: [], + removeRows: [], + addRows: [ 2 ], + columnDeltas: { + A: { [2]: [null, [1]] }, + manualSort: { [2]: [null, [2]] }, + }, + columnRenames: [], + } } + }; + assert.deepEqual(comp.details!.leftChanges, addA1); + + await doc2.addRows('Table1', {A: [1]}); + comp = await doc1.compareDoc(docId2); + assert.equal(comp.summary, 'both'); + assert.equal(comp.left.n, 3); + assert.equal(comp.right.n, 3); + assert.equal(comp.parent!.n, 2); + assert.equal(comp.details, undefined); + + comp = await doc1.compareDoc(docId2, { detail: true }); + assert.deepEqual(comp.details!.leftChanges, addA1); + assert.deepEqual(comp.details!.rightChanges, addA1); + + await doc1.replace({sourceDocId: docId2}); + + comp = await doc1.compareDoc(docId2); + assert.equal(comp.summary, 'same'); + assert.equal(comp.left.n, 3); + assert.deepEqual(comp.left, comp.right); + assert.deepEqual(comp.left, comp.parent); + assert.equal(comp.details, undefined); + + comp = await doc1.compareDoc(docId2, { detail: true }); + assert.deepEqual(comp.details, { + leftChanges: { tableRenames: [], tableDeltas: {} }, + rightChanges: { tableRenames: [], tableDeltas: {} } + }); + + await doc2.addRows('Table1', {A: [2]}); + comp = await doc1.compareDoc(docId2); + assert.equal(comp.summary, 'right'); + assert.equal(comp.left.n, 3); + assert.equal(comp.right.n, 4); + assert.deepEqual(comp.left, comp.parent); + assert.equal(comp.details, undefined); + + comp = await doc1.compareDoc(docId2, { detail: true }); + assert.deepEqual(comp.details!.leftChanges, + { tableRenames: [], tableDeltas: {} }); + const addA2: ActionSummary = { + tableRenames: [], + tableDeltas: { Table1: { + updateRows: [], + removeRows: [], + addRows: [ 3 ], + columnDeltas: { + A: { [3]: [null, [2]] }, + manualSort: { [3]: [null, [3]] }, + }, + columnRenames: [], + } } + }; + assert.deepEqual(comp.details!.rightChanges, addA2); + }); + + it("GET /docs/{did}/compare tracks changes within a doc", async function() { + // Create a test document. + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const docId = await userApi.newDoc({name: 'testdoc'}, ws1); + const doc = userApi.getDocAPI(docId); + + // Give the document some history. + await doc.addRows('Table1', {A: ['a1'], B: ['b1']}); + await doc.addRows('Table1', {A: ['a2'], B: ['b2']}); + await doc.updateRows('Table1', {id: [1], A: ['A1']}); + + // Examine the most recent change, from HEAD~ to HEAD. + let comp = await doc.compareVersion('HEAD~', 'HEAD'); + assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']); + assert.equal(comp.summary, 'right'); + assert.deepEqual(comp.parent, comp.left); + assert.notDeepEqual(comp.parent, comp.right); + assert.hasAllKeys(comp.left, ['n', 'h']); + assert.hasAllKeys(comp.right, ['n', 'h']); + assert.equal(comp.left.n, 3); + assert.equal(comp.right.n, 4); + assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} }); + assert.deepEqual(comp.details!.rightChanges, { + tableRenames: [], + tableDeltas: { + Table1: { + updateRows: [1], + removeRows: [], + addRows: [], + columnDeltas: { + A: { [1]: [['a1'], ['A1']] } + }, + columnRenames: [], + } + } + }); + + // Check we get the same result with actual hashes. + assert.notMatch(comp.left.h, /HEAD/); + assert.notMatch(comp.right.h, /HEAD/); + const comp2 = await doc.compareVersion(comp.left.h, comp.right.h); + assert.deepEqual(comp, comp2); + + // Check that comparing the HEAD with itself shows no changes. + comp = await doc.compareVersion('HEAD', 'HEAD'); + assert.equal(comp.summary, 'same'); + assert.deepEqual(comp.parent, comp.left); + assert.deepEqual(comp.parent, comp.right); + assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} }); + assert.deepEqual(comp.details!.rightChanges, { tableRenames: [], tableDeltas: {} }); + + // Examine the combination of the last two changes. + comp = await doc.compareVersion('HEAD~~', 'HEAD'); + assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']); + assert.equal(comp.summary, 'right'); + assert.deepEqual(comp.parent, comp.left); + assert.notDeepEqual(comp.parent, comp.right); + assert.hasAllKeys(comp.left, ['n', 'h']); + assert.hasAllKeys(comp.right, ['n', 'h']); + assert.equal(comp.left.n, 2); + assert.equal(comp.right.n, 4); + assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} }); + assert.deepEqual(comp.details!.rightChanges, { + tableRenames: [], + tableDeltas: { + Table1: { + updateRows: [1], + removeRows: [], + addRows: [2], + columnDeltas: { + A: { [1]: [['a1'], ['A1']], + [2]: [null, ['a2']] }, + B: { [2]: [null, ['b2']] }, + manualSort: { [2]: [null, [2]] }, + }, + columnRenames: [], + } + } + }); + }); + + it('doc worker endpoints ignore any /dw/.../ prefix', async function() { + const docWorkerUrl = docs.serverUrl; + let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy); + assert.equal(resp.status, 200); + assert.containsAllKeys(resp.data, ['A', 'B', 'C']); + + resp = await axios.get(`${docWorkerUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy); + assert.equal(resp.status, 200); + assert.containsAllKeys(resp.data, ['A', 'B', 'C']); + + if (docWorkerUrl !== homeUrl) { + resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy); + assert.equal(resp.status, 200); + assert.containsAllKeys(resp.data, ['A', 'B', 'C']); + + resp = await axios.get(`${homeUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy); + assert.equal(resp.status, 404); + } + }); + + it("POST /docs/{did}/tables/{tid}/_subscribe validates inputs", async function () { + async function check(requestBody: any, status: number, error: string) { + const resp = await axios.post( + `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`, + requestBody, chimpy + ); + assert.equal(resp.status, status); + assert.deepEqual(resp.data, {error}); + } + + await check({}, 400, "eventTypes must be a non-empty array"); + await check({eventTypes: 0}, 400, "eventTypes must be a non-empty array"); + await check({eventTypes: []}, 400, "eventTypes must be a non-empty array"); + await check({eventTypes: ["foo"]}, 400, "Allowed values in eventTypes are: add,update"); + await check({eventTypes: ["add"]}, 400, "Bad request: url required"); + await check({eventTypes: ["add"], url: "https://evil.com"}, 403, "Provided url is forbidden"); + await check({eventTypes: ["add"], url: "http://example.com"}, 403, "Provided url is forbidden"); // not https + await check({eventTypes: ["add"], url: "https://example.com", isReadyColumn: "bar"}, 404, `Column not found "bar"`); + }); + + it("POST /docs/{did}/tables/{tid}/_unsubscribe validates inputs", async function() { + const subscribeResponse = await axios.post( + `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`, + {eventTypes: ["add"], url: "https://example.com"}, chimpy + ); + assert.equal(subscribeResponse.status, 200); + const {triggerId, unsubscribeKey, webhookId} = subscribeResponse.data; + + async function check(requestBody: any, status: number, responseBody: any) { + const resp = await axios.post( + `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_unsubscribe`, + requestBody, chimpy + ); + assert.equal(resp.status, status); + if (status !== 200) { + responseBody = {error: responseBody}; + } + assert.deepEqual(resp.data, responseBody); + } + + await check({triggerId: 999}, 404, `Trigger not found "999"`); + await check({triggerId, webhookId: "foo"}, 404, `Webhook not found "foo"`); + await check({triggerId, webhookId}, 400, 'Bad request: id and unsubscribeKey both required'); + await check({triggerId, webhookId, unsubscribeKey: "foo"}, 401, 'Wrong unsubscribeKey'); + + // Actually unsubscribe + await check({triggerId, webhookId, unsubscribeKey}, 200, {success: true}); + + // Trigger is now deleted! + await check({triggerId, webhookId, unsubscribeKey}, 404, `Trigger not found "${triggerId}"`); + }); + + describe("Daily API Limit", () => { + let redisClient: RedisClient; + let workspaceId: number; + let freeTeamApi: UserAPIImpl; + + before(async function() { + if (!process.env.TEST_REDIS_URL) { this.skip(); } + redisClient = createClient(process.env.TEST_REDIS_URL); + freeTeamApi = makeUserApi('freeteam'); + workspaceId = await getWorkspaceId(freeTeamApi, 'FreeTeamWs'); + }); + + it("limits daily API usage", async function() { + // Make a new document in a free team site, currently the only product which limits daily API usage. + const docId = await freeTeamApi.newDoc({name: 'TestDoc'}, workspaceId); + const key = docDailyApiUsageKey(docId); + const limit = teamFreeFeatures.baseMaxApiUnitsPerDocumentPerDay!; + // Rather than making 5000 requests, set a high count directly in redis. + await redisClient.setAsync(key, String(limit - 2)); + + // Make three requests. The first two should succeed since we set the count to `limit - 2`. + // Wait a little after each request to allow time for the local cache to be updated with the redis count. + let response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`, chimpy); + assert.equal(response.status, 200); + await delay(100); + + response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`, chimpy); + assert.equal(response.status, 200); + await delay(100); + + // The count should now have reached the limit, and the key should expire in one day. + assert.equal(await redisClient.ttlAsync(key), 86400); + assert.equal(await redisClient.getAsync(key), String(limit)); + + // Making the same request a third time should fail. + response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`, chimpy); + assert.equal(response.status, 429); + assert.deepEqual(response.data, {error: `Exceeded daily limit for document ${docId}`}); + }); + + after(async () => { + await redisClient.quitAsync(); + }); + }); + + describe("Webhooks", () => { + let serving: Serving; // manages the test webhook server + + let requests: WebhookRequests; + + let receivedLastEvent: Promise; + + // Requests corresponding to adding 200 rows, sent in two batches of 100 + const expected200AddEvents = [ + _.range(100).map(i => ({ + id: 9 + i, manualSort: 9 + i, A3: 200 + i, B3: true, + })), + _.range(100).map(i => ({ + id: 109 + i, manualSort: 109 + i, A3: 300 + i, B3: true, + })), + ]; + + // Every event is sent to three webhook URLs which differ by the subscribed eventTypes + // Each request is an array of one or more events. + // Multiple events caused by the same action bundle get batched into a single request. + const expectedRequests: WebhookRequests = { + "add": [ + [{id: 1, A: 1, B: true, C: null, manualSort: 1}], + [{id: 2, A: 4, B: true, C: null, manualSort: 2}], + + // After isReady (B) went to false and then true again + // we treat this as creation even though it's really an update + [{id: 2, A: 7, B: true, C: null, manualSort: 2}], + + // From the big applies + [{id: 3, A3: 13, B3: true, manualSort: 3}, + {id: 5, A3: 15, B3: true, manualSort: 5}], + [{id: 7, A3: 18, B3: true, manualSort: 7}], + + ...expected200AddEvents, + ], + "update": [ + [{id: 2, A: 8, B: true, C: null, manualSort: 2}], + + // From the big applies + [{id: 1, A3: 101, B3: true, manualSort: 1}], + ], + "add,update": [ + // add + [{id: 1, A: 1, B: true, C: null, manualSort: 1}], + [{id: 2, A: 4, B: true, C: null, manualSort: 2}], + [{id: 2, A: 7, B: true, C: null, manualSort: 2}], + + // update + [{id: 2, A: 8, B: true, C: null, manualSort: 2}], + + // from the big applies + [{id: 1, A3: 101, B3: true, manualSort: 1}, // update + {id: 3, A3: 13, B3: true, manualSort: 3}, // add + {id: 5, A3: 15, B3: true, manualSort: 5}], // add + + [{id: 7, A3: 18, B3: true, manualSort: 7}], // add + + ...expected200AddEvents, + ] + }; + + let redisMonitor: any; + let redisCalls: any[]; + + before(async function() { + if (!process.env.TEST_REDIS_URL) { this.skip(); } + requests = { + "add,update": [], + "add": [], + "update": [], + }; + + let resolveReceivedLastEvent: () => void; + receivedLastEvent = new Promise(r => { + resolveReceivedLastEvent = r; + }); + + // TODO test retries on failure and slowness in a new test + serving = await serveSomething(app => { + app.use(bodyParser.json()); + app.post('/:eventTypes', async ({body, params: {eventTypes}}, res) => { + requests[eventTypes as keyof WebhookRequests].push(body); + res.sendStatus(200); + if ( + _.flattenDeep(_.values(requests)).length >= + _.flattenDeep(_.values(expectedRequests)).length + ) { + resolveReceivedLastEvent(); + } + }); + }, webhooksTestPort); + + redisCalls = []; + redisMonitor = createClient(process.env.TEST_REDIS_URL); + redisMonitor.monitor(); + redisMonitor.on("monitor", (_time: any, args: any, _rawReply: any) => { + redisCalls.push(args); + }); + }); + + after(async function() { + if (!process.env.TEST_REDIS_URL) { this.skip(); } + serving.shutdown(); + await redisMonitor.quitAsync(); + }); + + it("delivers expected payloads from combinations of changes, with retrying and batching", async function() { + // Create a test document. + const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; + const docId = await userApi.newDoc({name: 'testdoc'}, ws1); + const doc = userApi.getDocAPI(docId); + + // For some reason B is turned into Numeric even when given bools + await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [ + ['ModifyColumn', 'Table1', 'B', {type: 'Bool'}], + ], chimpy); + + // Make a webhook for every combination of event types + const subscribeResponses = []; + const webhookIds: Record = {}; + for (const eventTypes of [ + ["add"], + ["update"], + ["add", "update"], + ]) { + const {data, status} = await axios.post( + `${serverUrl}/api/docs/${docId}/tables/Table1/_subscribe`, + {eventTypes, url: `${serving.url}/${eventTypes}`, isReadyColumn: "B"}, chimpy + ); + assert.equal(status, 200); + subscribeResponses.push(data); + webhookIds[data.webhookId] = String(eventTypes); + } + + // Add and update some rows, trigger some events + // Values of A where B is true and thus the record is ready are [1, 4, 7, 8] + // So those are the values seen in expectedEvents + await doc.addRows("Table1", { + A: [1, 2], + B: [true, false], // 1 is ready, 2 is not ready yet + }); + await doc.updateRows("Table1", {id: [2], A: [3]}); // still not ready + await doc.updateRows("Table1", {id: [2], A: [4], B: [true]}); // ready! + await doc.updateRows("Table1", {id: [2], A: [5], B: [false]}); // not ready again + await doc.updateRows("Table1", {id: [2], A: [6]}); // still not ready + await doc.updateRows("Table1", {id: [2], A: [7], B: [true]}); // ready! + await doc.updateRows("Table1", {id: [2], A: [8]}); // still ready! + + // The end result here is additions for column A (now A3) with values [13, 15, 18] + // and an update for 101 + await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [ + ['BulkAddRecord', 'Table1', [3, 4, 5, 6], {A: [9, 10, 11, 12], B: [true, true, false, false]}], + ['BulkUpdateRecord', 'Table1', [1, 2, 3, 4, 5, 6], { + A: [101, 102, 13, 14, 15, 16], + B: [true, false, true, false, true, false], + }], + + ['RenameColumn', 'Table1', 'A', 'A3'], + ['RenameColumn', 'Table1', 'B', 'B3'], + + ['RenameTable', 'Table1', 'Table12'], + + // FIXME a double rename A->A2->A3 doesn't seem to get summarised correctly + // ['RenameColumn', 'Table12', 'A2', 'A3'], + // ['RenameColumn', 'Table12', 'B2', 'B3'], + + ['RemoveColumn', 'Table12', 'C'], + ], chimpy); + + // FIXME record changes after a RenameTable in the same bundle + // don't appear in the action summary + await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [ + ['AddRecord', 'Table12', 7, {A3: 17, B3: false}], + ['UpdateRecord', 'Table12', 7, {A3: 18, B3: true}], + + ['AddRecord', 'Table12', 8, {A3: 19, B3: true}], + ['UpdateRecord', 'Table12', 8, {A3: 20, B3: false}], + + ['AddRecord', 'Table12', 9, {A3: 20, B3: true}], + ['RemoveRecord', 'Table12', 9], + ], chimpy); + + // Add 200 rows. These become the `expected200AddEvents` + await doc.addRows("Table12", { + A3: _.range(200, 400), + B3: arrayRepeat(200, true), + }); + + await receivedLastEvent; + + // Unsubscribe + await Promise.all(subscribeResponses.map(async subscribeResponse => { + const unsubscribeResponse = await axios.post( + `${serverUrl}/api/docs/${docId}/tables/Table12/_unsubscribe`, + subscribeResponse, chimpy + ); + assert.equal(unsubscribeResponse.status, 200); + assert.deepEqual(unsubscribeResponse.data, {success: true}); + })); + + // Further changes should generate no events because the triggers are gone + await doc.addRows("Table12", { + A3: [88, 99], + B3: [true, false], + }); + + assert.deepEqual(requests, expectedRequests); + + // Check that the events were all pushed to the redis queue + const queueRedisCalls = redisCalls.filter(args => args[1] === "webhook-queue-" + docId); + const redisPushes = _.chain(queueRedisCalls) + .filter(args => args[0] === "rpush") // Array<["rpush", key, ...events: string[]]> + .flatMap(args => args.slice(2)) // events: string[] + .map(JSON.parse) // events: WebhookEvent[] + .groupBy('id') // {[webHookId: string]: WebhookEvent[]} + .mapKeys((_value, key) => webhookIds[key]) // {[eventTypes: 'add'|'update'|'add,update']: WebhookEvent[]} + .mapValues(group => _.map(group, 'payload')) // {[eventTypes: 'add'|'update'|'add,update']: RowRecord[]} + .value(); + const expectedPushes = _.mapValues(expectedRequests, value => _.flatten(value)); + assert.deepEqual(redisPushes, expectedPushes); + + // Check that the events were all removed from the redis queue + const redisTrims = queueRedisCalls.filter(args => args[0] === "ltrim") + .map(([,, start, end]) => { + assert.equal(end, '-1'); + start = Number(start); + assert.isTrue(start > 0); + return start; + }); + const expectedTrims = Object.values(redisPushes).map(value => value.length); + assert.equal( + _.sum(redisTrims), + _.sum(expectedTrims), + ); + + }); + }); + + // PLEASE ADD MORE TESTS HERE +} + +interface WebhookRequests { + add: object[][]; + update: object[][]; + "add,update": object[][]; +} + +function setup(name: string, cb: () => Promise) { + let api: UserAPIImpl; + + before(async function() { + suitename = name; + dataDir = path.join(tmpDir, `${suitename}-data`); + await fse.mkdirs(dataDir); + await setupDataDir(dataDir); + await cb(); + + // create TestDoc as an empty doc into Private workspace + userApi = api = makeUserApi('docs-1'); + const wid = await getWorkspaceId(api, 'Private'); + docIds.TestDoc = await api.newDoc({name: 'TestDoc'}, wid); + }); + + after(async function() { + // remove TestDoc + await api.deleteDoc(docIds.TestDoc); + delete docIds.TestDoc; + + // stop all servers + await home.stop(); + await docs.stop(); + }); +} + +function makeUserApi(org: string) { + return new UserAPIImpl(`${home.serverUrl}/o/${org}`, { + headers: {Authorization: 'Bearer api_key_for_chimpy'}, + fetch: fetch as any, + newFormData: () => new FormData() as any, + logger: log + }); +} + +async function getWorkspaceId(api: UserAPIImpl, name: string) { + const workspaces = await api.getOrgWorkspaces('current'); + return workspaces.find((w) => w.name === name)!.id; +} + +async function startServer(serverTypes: string, _homeUrl?: string): Promise { + const server = new TestServer(serverTypes); + await server.start(_homeUrl); + return server; +} + +const webhooksTestPort = 34365; + +class TestServer { + public testingSocket: string; + public testingHooks: TestingHooksClient; + public serverUrl: string; + public stopped = false; + + private _server: ChildProcess; + private _exitPromise: Promise; + + constructor(private _serverTypes: string) {} + + public async start(_homeUrl?: string) { + + // put node logs into files with meaningful name that relate to the suite name and server type + const fixedName = this._serverTypes.replace(/,/, '_'); + const nodeLogPath = path.join(tmpDir, `${suitename}-${fixedName}-node.log`); + const nodeLogFd = await fse.open(nodeLogPath, 'a'); + const serverLog = process.env.VERBOSE ? 'inherit' : nodeLogFd; + + // use a path for socket that relates to suite name and server types + this.testingSocket = path.join(tmpDir, `${suitename}-${fixedName}.socket`); + + // env + const env = { + GRIST_DATA_DIR: dataDir, + GRIST_INST_DIR: tmpDir, + GRIST_SERVERS: this._serverTypes, + // with port '0' no need to hard code a port number (we can use testing hooks to find out what + // port server is listening on). + GRIST_PORT: '0', + GRIST_TESTING_SOCKET: this.testingSocket, + GRIST_DISABLE_S3: 'true', + REDIS_URL: process.env.TEST_REDIS_URL, + APP_HOME_URL: _homeUrl, + ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`, + ...process.env + }; + + const main = await testUtils.getBuildFile('app/server/mergedServerMain.js'); + this._server = spawn('node', [main, '--testingHooks'], { + env, + stdio: ['inherit', serverLog, serverLog] + }); + + this._exitPromise = exitPromise(this._server); + + // Try to be more helpful when server exits by printing out the tail of its log. + this._exitPromise.then((code) => { + if (this._server.killed) { return; } + log.error("Server died unexpectedly, with code", code); + const output = execFileSync('tail', ['-30', nodeLogPath]); + log.info(`\n===== BEGIN SERVER OUTPUT ====\n${output}\n===== END SERVER OUTPUT =====`); + }) + .catch(() => undefined); + + await this._waitServerReady(30000); + log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`); + } + + public async stop() { + if (this.stopped) { return; } + log.info("Stopping node server: " + this._serverTypes); + this.stopped = true; + this._server.kill(); + this.testingHooks.close(); + await this._exitPromise; + } + + public async isServerReady(): Promise { + // Let's wait for the testingSocket to be created, then get the port the server is listening on, + // and then do an api check. This approach allow us to start server with GRIST_PORT set to '0', + // which will listen on first available port, removing the need to hard code a port number. + try { + + // wait for testing socket + while (!(await fse.pathExists(this.testingSocket))) { + await delay(200); + } + + // create testing hooks and get own port + this.testingHooks = await connectTestingHooks(this.testingSocket); + const port: number = await this.testingHooks.getOwnPort(); + this.serverUrl = `http://localhost:${port}`; + + // wait for check + return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok; + } catch (err) { + return false; + } + } + + + private async _waitServerReady(ms: number) { + // It's important to clear the timeout, because it can prevent node from exiting otherwise, + // which is annoying when running only this test for debugging. + let timeout: any; + const maxDelay = new Promise((resolve) => { + timeout = setTimeout(resolve, 30000); + }); + try { + await Promise.race([ + this.isServerReady(), + this._exitPromise.then(() => { throw new Error("Server exited while waiting for it"); }), + maxDelay, + ]); + } finally { + clearTimeout(timeout); + } + } +} + +async function setupDataDir(dir: string) { + // we'll be serving Hello.grist content for various document ids, so let's make copies of it in + // tmpDir + await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Timesheets + '.grist')); + await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Bananas + '.grist')); + await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Antartic + '.grist')); + + await testUtils.copyFixtureDoc( + 'ApiDataRecordsTest.grist', + path.resolve(dir, docIds.ApiDataRecordsTest + '.grist')); +} diff --git a/test/server/testUtils.ts b/test/server/testUtils.ts index 6cd7b85e..2d55c996 100644 --- a/test/server/testUtils.ts +++ b/test/server/testUtils.ts @@ -169,7 +169,7 @@ export function assertMatchArray(stringArray: string[], regexArray: RegExp[]) { * @param {String} errCode - Error code to check against `err.code` from the caller. * @param {RegExp} errRegexp - Regular expression to check against `err.message` from the caller. */ -export function expectRejection(promise: Promise, errCode: number, errRegexp: RegExp) { +export function expectRejection(promise: Promise, errCode: number|string, errRegexp: RegExp) { return promise .then(function() { assert(false, "Expected promise to return an error: " + errCode); @@ -307,4 +307,11 @@ export class EnvironmentSnapshot { } } +export async function getBuildFile(relativePath: string): Promise { + if (await fse.pathExists(path.join('_build', relativePath))) { + return path.join('_build', relativePath); + } + return path.join('_build', 'core', relativePath); +} + export { assert }; diff --git a/test/tsconfig.json b/test/tsconfig.json index cfec0a3a..ac7c07c9 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -9,6 +9,6 @@ ], "references": [ { "path": "../app" }, - { "path": "../stubs/app" }, + { "path": "../stubs/app" } ] } diff --git a/yarn.lock b/yarn.lock index 2c733fe3..fb576dd5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -741,6 +741,11 @@ anymatch@~3.1.1: normalize-path "^3.0.0" picomatch "^2.0.4" +app-module-path@2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/app-module-path/-/app-module-path-2.2.0.tgz#641aa55dfb7d6a6f0a8141c4b9c0aa50b6c24dd5" + integrity sha1-ZBqlXft9am8KgUHEucCqULbCTdU= + app-root-path@^2.0.1: version "2.2.1" resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.2.1.tgz#d0df4a682ee408273583d43f6f79e9892624bc9a"