From ab39b9cca3d4c031cdc642445ac3a332c562f549 Mon Sep 17 00:00:00 2001 From: cascade <215315028+nystar1@users.noreply.github.com> Date: Fri, 18 Jul 2025 02:27:19 +0530 Subject: [PATCH] feat: multipart, 206, FetchHttpHandler etc. - Large files are uploaded through multipart to the bucket - Slack files are downloaded in chunks if downloading at once fails - use FetchHttpHandler --- bun.lockb | Bin 96106 -> 98096 bytes package.json | 39 +++--- src/api/upload.js | 2 +- src/storage.js | 340 +++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 339 insertions(+), 42 deletions(-) diff --git a/bun.lockb b/bun.lockb index bf33312398b5482990922c4dfa58ec763aa830eb..14bf8d2c02b95e264d81f11a8600986a645090f0 100755 GIT binary patch delta 19486 zcmeHPcUVo90rZGQ`|DJFYFfXP^wi9ZPX_eu zGa$Xcw0HU`kH!JzB&jkg=b$k)`cX4BAT6a%W@`VGK53z8$^BEL6_6XCPtF*Y)*I!C z%wP5}AJj`yc}O!;hGy33mol{XfKLXbLwn0sk}5$SX{YKhg0BdEH+bsB(Oz|FV4Y7= z`=@3~*&5#pZONVsx+0ngO5SCrH;jNv>Oc6C%oKFc1WMTNWvQbUEN4KefrvpG0^$n3 zYz%rqPL8BzB&Vk*50j3e5yj3Ql;oMi2By?W&5$O+I?2tt`ls z4$er)_%LN4Z+Ae|>nzJyfzH%;8Cc51Gf=8NIBj6+z!XXPMdSagR<~@|=x7f$G^bgI za#hXSJ=IpxMm0bopzzC536uu08kC&B;-yZee#sdFK24LPJ>W_HBPcoFcYx~LG4SNg zdab?<+7sW=N9|`kXymu{0f_r95<(j-uY)}{yjy;E-=4ZDMNeTeH0iGN>3F;2|xP~Ml5SCx+ND{_p zIRgsAmW|L?1+5KAj$KAQa)5!7QgH!q3N(Ya@SUv=$^tT12h=)-TdI*WgOc6Ds85bwj#C{u21?t^m(3wX8_O82LN8G2pqa)y1mrcd zJ5jDt=MUX~{XL}N=(vTKRxk0``DdSz+pnH%IQrb9M>99JnKXIawa+X|hSlwwKDpxR zO6{&BMHZy@jtl&w!$~%2kw^@;PQRTYK}|Fy;U+ne0A z*_QZgIalZ3YzMJv_Q@ymEEUe_|6lJJlP+RHA zGsDN=;1 zn)IWwm$qjMDo5$_QEI^!L`TaNnU^uf&>4GsEcX*u4Ku-E#i@}o90iAI49?#eX|TsK zP-`x(+lVE4nG7=_A-%*FqrZGs;APS|IxsKq7`Z)5^ft+3S&p|!zY7bqAzR=arT+`1 zSk|+0lpz{>B!$d|C8KE?%keSEBUpit$#4|Qx2@VK9CF3tZmn{jk%k`NT5zt=EmB^@ z3aXg&*CA`hs<=nVHCUprN$$sTd`*U&%EDQs#83c^9M!Qxhe*8#rhOx7p??RZ7)q-l z2PJKAl_T{9yat8K8Ed>AuhA4GEf_A5dfGFZu%3=lh8-xWAtRrDVTt}G*`MY3oAjNq z97CwB-i%T`ih#VHB~~-ZS6EIplitIf+rXsiz>L*RhUwrN^3l<>{FdcFrpF$n_KGPW zH)qBGll(bL3@{lAp&^fTJURZ@TWCUJ`uay262Ykx5{Z+iu*5)<{vc#L_WJhNVB2%w zx1pp)m#n{Fi9sfP3v3?E*aF8W{e6_;c(xj13ntHC9ntIut{L|e*7d8wA%@85EtF`2 zV#>Qk8a#0z(SiaO-N*(^91r5W9 zaO&is5XCWLEt7txFIlYO7G*d=CDfI;1NPYU)Wp#2jVb1mJtO5&EU~sp{+i{~HW}_i zqq<-;2*Bouy_2RcCMW~fUU0gn{%metlYKR@ zDyha~aO4%%X!S_LL2zAE4&$j`U6K++YrPpp0Em4yOI|5usVb*Y>nDd7_ezk=I)NTkk3aDNUHaK8Yz-pPg?IW9VI9l0M*7CD?9c|@?`QXSw?3d8p1xLA7 zxh9x`>EP6XtpQg&XuU&}B(-5Z8%OE8qVyi?84{)6gi=o`=^Ufk`KTCu62cNi0>e?1 z)agyzveR31Oki6!v_OfrB|a(jpMy*0M(?4N!i{#saNgxc_r|dEZ^h`HOp^3IuQULq z4!pDvr2)KD7yAsV!>_R@sj}ZtQd@LvieG%(p;IV9tOiQ&jDYmzt%eqtuAtPLyWoV` zn88bPQF@2Bzl{&3jeQBzLEJ%|RCB7zr>|yib4^bVf z6z^dsDRvJ25T)AK0XW6tz-L-JqU7FijVFrNQOP1njI|497vr3`A6c6I zpHLe9M6Ld-XhrB|0;u0vT0NrVUyjBTCH-9dM&b@F1VfH31yIG60D6d)2W%n+4^e8E zM+}}8=cXL^sb;}zzvA>9-#K1RW-RE zC^--aO7&}kQoUN5ysjoUhoOK!QUf5W5Uwd2fl?}>HF=CiV?k*Sw9;sTCT|Z)4s-*h z4&T${JvI8irtb@iKTv~Y+JfH&N)bB%O7;(f(ngvODpUS{K!F~j6v7LjRPhQZ8MqGW0Qws!{z#8B`BPBp z@Hr^em*Fk(HX5}9C3yu<+7>)Dxwj^-3Th_Lf>5A!8wN@pHUg!MH3^gqCWBJFexT$) zszyHoC5O{NsohXeYL~0=OF^mQji9uQcY@NS+^6YLY|9Ohk$=yYf6o?7gNXooUPb>s zTd;8cJzM@gTcm%_mcKt+irBRd9c76{bnFV8&G8M6V>*#!Hplllj_XX4T#g@eobWD5 zmUCRhahEP6S;w($S1KoQyp>~zZY1f$@g9!d6G@WB@ga_VyOU%H$0s-rdXFTdI6lL1 zXb+N%=lC+m5j{zg&G8M6W0FWRo8$W&$GuOIT#g@eobUljmUCRhahGJ0tmD|W7nPGZ z-pa8O+!I9G~Ghv@c1~zG?Mh;cn`IV4qHcDqBW>3ADKRx02@PgHr@*T4zc4Bx}I$(wM;q2k4csMX>ga=zNGPbm8B)dJ5 zRG}lS{GoSQ@z``+k{w(5@!|Coy7&D4S!ONQm>L6DZ?1kI?fW0?H-G%3`%mlY4PW<7 zw*jjTHCsGrVaB!JoAo{7vZdGX;F`&kzG{-y=AgtbkMxLt+onazzMIaI{U7Haw;Z~? zWZ}&CkG5YOHge0kKi0n3G;~|ls(U9rxPDOorO)};huZ>L?cA5#)-kqXe5R$tFH7gR z{swo%c(T|NGdZ0-$|HVF;f{g%`;(?TthPzl+PM3A{+;8`9)8|tU0>hwwTE`u_pp`m z^ftx4;YikX)bqSy+fGI8H1wU@HZUtc{piYNKe?+vFV!TBVOhh;Y>e51y*0urOyk(3 z5u~~g)jey~1lE)XH`n69mW{G%HHo$1ekNpju(f6@)fCpjLI#VV+G4R%O=WTxsn(75 zU@2KvsxR0^u1Xr?!Q4h$siw2!(bURetOxtnTE&?27*g$lYVa7VR;s~ zDIVZMOR>)P^Gm$T5 zr8>Z-Gintx3;D8Eeao88B-MSWmd&(kb(lTms@&Pg*DNd55w>_1wMtNsuh~|rW30n$ zQWZh9#aeZo$qK2~%|X5tt5)B$ja-$KgM7`gQk`PSbEuWWT;$7I^#gOxA=MtJ2Ip9{ zI?eWTRoXn{Yp#{*EE_bJTKUdLzN}RR%zs{d>0SCf8)hxIz)o2UF0$bH@k;&>c~0K6 z`CpV{?Hz2`qDA)haroMR-|LpVtoimjlIHDN5^c+-&2P`E{I$~Fz9#+}0zd7k%2k<5 zAFQ^w--QpPXiL)hy1e;o6NAy{VEK9%)MFw35_@sxEoM1q<5u)SeMoVV@6tf5!&w*p zL6W4W0Q7T*K0wgdFM893UBe#lT-#@P`~A8GX2j;1Ybg(PWf|{p3BKivX%e zUqTK-MgzK}$>_6AAINAxm$?jws3QtBw8mGskpJ5g{*8~kzpBZcL7M=`fkFT^q0h}L z(Hl8%4L}{c0KP(j9Qdy$bJb+zz;#XL2AXGv6>{K)CUgf~tvPd3lX-yBw;y_L0jLvC z0ChqR+}319&^Xj18+SCB7wCA%$i~l_Y$$~TPt8Kifga}q*w}NPvK_m3ZUFnBz{mU- zRL=oLfENJjPX_1!yN*MK9k0P6Fry z8)dX6AQ(^!P#aJOP#53_2msKRx{3fNz$LV!Pm1*Yat0t9FdUEt@B`3?Rv!S({~Z1y z6@`I-48VTCH-MIay?}jy$$-xRUjU{7MgTC|_}><^25kkPPs~w(x`28Be?T=rb$~a( z2jB;w>`}()8z#+;k94r~2@3QvbuJY10P_J00J(rTz)V0izyxRp2mk~EmI9Uof&euD zI{~WzUjfzt)&kZ6)&n*GHUfeHwE%g5+JHwil5YWr0kq2c0BXaTY5)(w50DQASOBzW zXpzu-CCJn2;xGTCSOCe^YCtyPpdSw)8Sw<&3Q-Zyz`H7CADr`($K(ZGbSY;Z4Uq*v zJ`D$arokvsiaPNlHA-b6qxxB;zaY82UhfE|GC0LlzyY8zlHfFey(WHo>?uoAEWK-pOaPyk;7<^pB|rUNDc zrT}IE#sTKguxZru0ZRaj0F-)~b2Rs+0ww?`8e;)8!tnrVOl1O5CTFMv;>jjOY9fHz zin`R69H9Bl+fmr4AVMjOm99j_$UwGMS9B`s(#S-ap**7$FXVz!6ltDQ8>(v-p&+9a zUJ4t9iyWXvGn^jO}-&WKJ0rFOzDMI3F+#o?vK1i~|FM23Zo z_=Nz`5y@!;VgQR#E^UK4qKFB-NGH`<3Q!$E{7FE@MV^GWWo494QD+UxUuo?`ry@*M zf>Me_3dp{h8c=1L@xlSBL`EpQRCzC;bQpJor@DIpv^NNYG%^Y^O=b}(VPB+&mOIHb zyXIm~g_m^F2?xq{F7hvyBsn3RCWmPtqK=S@NQh2L^W$j!|6N7O+c5y;OE^iRrVho( zY1E~i;<<%P*cSB!PlM#`(SNbmq`Uw)4=4bf1DpnY41oS73^)Y%8bG@&)!zv)?*pPOHWxr)r(KM;(-nYafPc1){{1d0=0oYoz(F&A zClJCP&|w~cMke4a$~1~I05Qta5hy+S((-)N|N96a{`*kze=`DN4)Iiq{EJ+Q{FlzX z7~$*kPf-zv4;iJ&CYCErN?}_>Ps|Z|8}oft=oj6E2G6dB%Xmp_VLcuNrf_*tjp zS|HZ_X6*nusbOeX{m`)D8zbkQB)C3`eS4NpjzXP^XxWOby4lW~wg+c$)pwsc?z4I2 zQ=Oa?7FrJtr5xsbtAV_NwY=5No0gj+Y8|_$@76SEZv<*J2yGZz58Du1d&}reURQuV zCAxPFv%~YMI=LQ<42ARxE4*ct@33dEwSm>WZN$GZg@pCG?Ij;!BX1kM&&uj=z9kLL zBrII^Boemkqj7!uw@({UKZhLy4q(@Bx088qnuR`hyySx{>rR%pxLxzLe(_S9BP%v? z3lZeqEmq}cqqn$)bE@0wlWiU?ysne$ht>~m0PiAMho6n|ZuT*1i90!u(`Ju$Ex)`v ztker_Kz?Mig+Cj0E(Z4X&+&31v%8xWF7C_xxMSthWfO8JAmlW<6gP5?_iUiMw&Z(s z22D5xvefBPeqQX6+e1-+yYbVn%U2WkK{>2)bde7)BF752hztL-D#^wn+ zc@#Me{aLmj&C=KfV9^YHn%VxR?lxf!rAaLKemnU=-lz9JmgQya)I+1Zjy?ay=q)b8 zgd9D6ckM}+mK1_83=cmwSp5e^{A=>U-;HtsyYz?*n;#g%|BHN6O1l)^Z@4xt6cb0w zO7TS-?+<>PKK4EL_T0UCp$#SJ89R=a;+~J~E&uP@m9L0&!2K|CU)%_qHzGuT{lFiE zJP{ETo2tzFq0w91AgVB=ule+qqS`c#Fm4{ZcODwU#f_tRyJ~NX8Md*B>Qe*~DefdC zoFDQuZQ!0-s)kaW$`&9Bd`wzI_=xjtezlRk#a*J!D}VaTF>>7_Z7AsQrLgGioLKsA zedHz#p0Hnk^OC2sN{{B1w#nmnTOQ3uKDLTz9dR|OOvJ)*lz6C%@kym>Tijw7J#gfd^{H19yutn_qKZG@THOm5{+Vp1*PHw+JzVm{w#uB6eipC}e~Ny*#qFrj z=mROYCU@$>tMM7Vhb{b*7R8P~8^}Mhd%#zi{}ZFk<=)~J)u#2f?8o|XQ%VN7o(*^+ z%=^5?3&ep(EldlZX7n|+5gQ7Jt6c+H#@68;&89vzdW$PsH@~{xDQ#1ZD^0S-G_Uyr zG~&|Mg{KWLK^&hM+b{7`Z@o0F{6X~M5Qow5JREZwoos`~8w*QZDuD(&DJ`FTXevP? zE~sz~-+`|9Df4aQFy*RF_R@XiuS|20J$1uDWjRcD&0jePswy?1&+Bb@Rad?%C&#`W@DupgRWIFsMzy@i$PO#Zn(MK+~7r0cei#lT+MQ^WL+w zLyL<|XO}efuc5^H!*PDWjYeAi!szKW3Y|-8w5`F~7kNgA8*zc1{&X6&>c!xa3gXV( zBbRTlT^)NxDbcVREVszhM_jUVEnhXfN60j6(|m?wpFUJW*sV7Mf)yWesjqQ<^yXQC z=L5BVFipRx$sWA$^byw#fBJFXeY?Y{^Gh5#SyQR#3X?Z#Do$>o&uc0VDuOx&Drm3d_QxIxpPxm`I|tnL$dd9j1b~IGVk->eKfVc3?9R zQ%;*|Vs4=!9~~0?ja`X1I?a$y%uvyHsZm(XMQ!3Jm+fUQA8||daGkFU_hcPfr1fr{ zj?#lK?xw6;+SC^F(ns9L6*m*PWqyoU%@5(Nh?E$mwsK}yz_B6f2;HDCrF{iNx%5cX zGelUijyDZo`OIBzs8$l@#0bPRdAX73L0e?;Vh)O#K$OQ;8`7KPyq>mrv_%@`Bktlp z%?`Yil4_*)f7%DJ4|Iu8F4Iwo9!l1&HnDxgm0joLSs#Aa$$9BZS@RK>c_X&w{U?3G zjLt9D5Z8PwCx5g1`5(rMCC8x{RPS0!TPIAkQ;n2Sw3*=`*-*XI#Ip;U?F=!-?&U1U z)1lsvRtXOE`K4FO-#RPC_=ZDHoM9q$A}Y2nm>$}>P5zZW=+sl{rId+UVwCEB8dls| zYbJUUGfGUBoM@$p&gHT@z1WV|Mk}@4UqmoP%*!<9 zTTyWWgrlOd@T9E#@Axw9B3?h?_cm4Ug}<|WrGN7Iu)QTq0aL$+H-b}UioGGbVyLBh z=PO_2()yTwd2#*Ss71P7*N+xjr-yeBg@XPf4Wy{e!XEk(SB`;sV&Z!vS+h)#wd3~Uh#G$8jIueAFaZ$^0st44v%fP z5*jMUmanJrb4DpcL*=GQg^se&f0qUT4{)J2pFWN|v?^ zddf+#yWy)U%T^IC5 z&>ZD5Fhic&e)6O#lcsCa{Y|<(Z0YX`Xkr(JWOUPW6Otg(rp%Z;JqrUwLJ0q3JV9p1 z^9nExR1WTg_eP+r=ipB88OWr(%-r0}nc7YmF_2ooZGYzns&;f-!j!b zB^I6-Y^G+v7+9mJyClT4YUVT()DW1HxCA~YQ3#sB9}YE(X<}wx&O_On_6N}9?*nG= z6LQSpc7kR!D=huL!4K)q;pRA#!AAg|`M_AoW}bx@fGec~GmB3mOw!XLOv8yH&@WgY zCR+P7bwgw>f8OLtW0=9_<_esNF@t%yg{EOV&sp&4YeY+vJ_O7`k6QXUtuzh8d9J{Y zD`)_d&TXw}VZeU|rv47_Zv)PY)-=TH_UwTJ)?b^png*F?J!mQ}0d5RD8%#{p^IAJi zL;RjCz;yfo_2opUjS3_zGTxd7g5mhM$oLITP;O97$eJ`I=~F* zB<#3B_W(2SAYhK;0p>(YF>5yq&YfZwV&3%J>={!u?E%mv{=y+c4Y0hNu z?PjFUgU_NV0%ibHdzouI5SWgOp%0N~M{hHbwZJUG8Q^2JJcBKH0x|0v1C5WD*9+cfH=-+jachy`LOUc;v&c{Pv`OSEEtjrJB=3p?_xCV}%2@6}|qr z|E|(HAKLU}QC#1pUl8SJ+X`oZOP?gt(H;}UXaht!+6>`r;L_KKbhIZ$G1_oZjy793 z16}$mk&gClQH(ZFl%wq@oDE%$$Ne;|Gv=g=3qi^HAyM4WW&6oboN1VD=YmpfpV`Hkpj7yFHgY-c$5o@!+EEB94!&xLs0PXUN1_}WEpZ*`CjCbR zChKEG`fV{tt`BPdN|BH(vKaj?tL9>){b#N#J28z(z*EJ_!d0m=FiksjjGZK62DWuJ(1aez42 zFvY$Xy`!AKCWv!^DfUm%>mk-QOmQ^B z5z3_bi48ElPn3td^fAKO#O2t5-FlZfEJFGT)SV_3lI-Y-BcYe1q8cab3xzYnWj_X9 zPtHlNCyFCndVf(K;c~cfikTC`qBstMVxTq=6_9Md2C6eH?1`xNRIabXjh?BGY}*fl z>MS$%19}~$MoVmND;+_}_AF3c#oB-r$2#=Pq%o#bqBzQ>*AnGXE_)a3zIL>=XQ2l# zfhqb5QQXX>9~R}!T=r|=cBLGJo*wOTJk%I@lCxuqpBLreeF>g9D&o>xi}Vl^!muv3dOa7k->XFvZFt!o}f@P&SZNbs8p#{h8}BF7s!O; zI;gIaVn%!6YSc&Od=m%d(fp)CL}KF2k{xlN`hb!}>nH@p!9Zbyz6EN4q%d7aC{A>y$dBiXV>&2H z2I?bF{VQdyaSLGjWRQ=6y0=nxrcy>Pk+@8mvP@9RKn!x zgeIms60uC9O%lABFo7D`05u0X42NvamR@Dk;@501G>;(%EyM%<&Ms zqgK3G5)K(lGN>_?krjhtL7H)%0#y}tYwQx!3+7;-396@P65q`q9cx(;>0OB`8ZPTL zP)rHd2b3R5v@*4h`Jl#E4rRlwj8y>1o&7s%xs_TQC_KE+E)}?K+!9 zXyfth$OgsGa0Wtl02H%jQgyp%S}rJDN#Z*DgQ%RX{S#2X6KlJs*juC6hl?g1QtXeS zH-mE)SxYC0)jJ z*R;XXWl?t#>PoeL3dSg@l!j>zl)dHX<;b4yqB^4*f}Y7cjGk!`gHvUS3}+*HJYr-* z?0-ggg0#9TP4#x7H&%x5DK>1L>`lfU)IH){uWtV6a5iQ>b<`>|XgXU8YN%-9N^yLO z9t+9jcf)N|rgZZs^ajeF6He}sz0v4RlD(tonR;#RP$4WsPx5RPcZgxVySi&>nnu8N zoA>2f4FIkh@1!O&8~SHyy}fqAhc8Yy+>k#V01+ z62LHUcF7Mh`8aqaAE%6_iIM$+-Q?l~kkXFi;^U1uaTK)t5c>g8Q1U}ec~6s+n4{iq z(Z2#Wlmlap;_y@i&-a)v`!XhcVIj#$t0C4yxEj?m}`IJQyQ(g>USS1$T2u!_a0Q?Zw2J9q( z53pMr{?Vkhn_({VZU6^((b6NX0oV(md>?=xV$!cz_<)5=f%z$wtv5ynY4ExwxEZGW z5TG;QXG@Qm0o(vEFgDb)S=i6QHG%P`)k5PB9AME6fN3A(Mu#a2u^3^NLO3uTMp*J_ zOWwkgx3uJ~Eq*&-rn;lWkF#(BFqb&R!aXg1FJK1b?u!m1xyKR=wD2HHVK^}Uv=L|+ zXr_h7Tl@(YJsFrEVs@vX!9#~;edi}J$C(K}@f=`G(5>ZLjCsJE!2$~}HWjp`!1&W1 zM?=FWEW8|;`YSEE7?>GbZP6tbz23r41LIHIXwjR1Yj6i`u^7(*^Pt-S%#`f`<^cPF zd4L`P#-H{k8w)dq?}Db@`@pm>2d)qNIWYdTzgqkXU{2s7F!e8!<_`YR5?r+eH-NeO z>tZ@I2mq!*5HJG^1?IMF0n7p00k;6Y7nu5kfvGnF7_PLD7R~@>z~g~wHyPMX!{rv^ zDPRt~37GqIJ22OLwvh6>CoXxVmF54j!$=N2`nv>YiknKydt^a#+#>@7A zY#Tp7PPS~1$Tng!IS*5^Nwd(=b zfoFJ;?_=Q>j(oB++`(c>zK_K`aY933Isnl!U(o_l{s=|c^Mb`YZXc^fqSJheA|44AE9U!%7K;l~R0dJ+ zM}0&~MbV?Q%AX%BF8hj>iL?b2#XcG=HZAb6dO}>3qVo_9Ug#q#5*rrMs$fB|urKlv z2{CXHMX3ve#f!e86+(ZEq6&zz9`ms>#7-$HSrja8TkIn$7MY7_)&H?zaoAV1S_Cek z=sH9*miSn$6|YIrw#C7s`BERzIx%f2tumGbi&MU$4I=7sis~;777HHtvDzq3NYMd^ z5|{aiHi>!5XqEkVu=tCw=o!(efTDfMlI2*hCw)XOi_9l!)n8z}d`0_3;0lVaLo{QBkJYQJ4#1iVi@O_>_<6u$cE0t+EZQm#^qe(W#iCh^Me# z#XeR?#RVxUgQ)i^AJH*Uw2D^w#aJ(2(K{k-HAS(juwJWutlkwDrRY3FgV*?oPKXU_ zXjQNp>$TQL^u8FlmZH=(STA4EDWR88Q~^;|iI3F>Vy6_9ti^h*>!ItrGS`LqiP81z zh{F1Q!n4ueza1Wlaii{m#NS5lt~Hyw)s*6PFZlcaga<)9dQ$v}F1eB0{QV2?Jss@G zU1Qtz_Kx({wL5=cKKKmyP+U6GT_k+e$bHCq6gm#fJ7GMnYKJY}35&<);=3*0dlrw+ z)cq{p`xcLnyNMR>q{ZX2@{4qe&nW=M;3M!}@X&W_r!6KQu7`n1U4}_Teh1P8K*t{f zDB}am8VKq541juko?eFz9e-r;_>fN?{hqaWd_E_SKF&!Vv^?~T&yOF8A?I3%jvx7n zIp;zKGBHfV5x|>(KLL&c-U1v0ybX8o1aQKfALsL-SaB{#|GUu7 zH+8-a`VGLHfZqY`0x-e-#Io}fn(%wrP5?gMCjgkzc7V2kS|ae{wr+l>z@+X09Kd`^ z0fzwmaw7~74rl_105k3Euc4m-@2v%QUPgz zXg~~LBVZGtIiLk#A7BgMIkEVYuzOpf*BbCgKpOyylr=dWeSX=*T4e39h6wm~63qo* ze}fmaV!`3l;B%JLt`Qd+)Nr%PSSzfVy#Uq`>xp&68e^@o=2#c61DMX&051Vb0j~nK z0`>r20lWy<4dCnzz%BseeirZ;U^9R-{R05A9`84^dKcY3D$5db|< zPXTGl=#3FA0MLeV;)NW+?Jc1f>JeyY!UWL+w{I$i}CONutsRr#P#CYiZI z&fGFI8Z!xMK&8aQ(t#R?$z;+fUje8-sG@1diIHaFX`_-%TY?!F5=13BR!q{|1Zp7i zy=So2(pCB#h#6BeRI5t)dVmUqG<~aeVt*DFs5+B8rL+b8XDl5$AgE<}&Wr!NSN{d8 z&!V9IHo#V^F9)^g?Z9s3bcdy|vzns!Aj+5+#t)ml+@sA z{i_!Z9a5jpl#ZH>8bA%|P2Z}&NUcAeaNXVnaJ^KN9`sd!bi!GC2j#uCHyfC<^tNZv z9P0zXX@Hv-){}q}fMWnU;l8cDzZuZmmYgy*qcZg00Z`6^_YJ`70M1`z+L!&KFScV-wM?%S53e~|%IL)9lje&zamuf3Y#t=6AeQCA;2<wT-dChy17?qknj%f1|(mxO;_7j{?6g4ubX~`s;r9Y2>u8o%r$L z(y!z7W#S+>D@DBur@l#S{l=;96KUul7Q@jV7c(m2!+1>yfc3Gj?f0j(IGAM9GvZ^D zV&hP@)DmYaLc_M}=8gXT_dXl5C2#j4=yZz3f4FB5f$%y=wEf1(AMVP>0rCEqH4eOX zW2;T?1VNoTB*Tvj#onh*y`(viXbnj6optBsIOKd#u zGR4ksoq7{->|1BJ`XPUWnq(P#|ch0cL+UC>Fy^Bw8i@5);+K?xh z)7>cses|N9Aqa-?idgquKcpk)QsW3@NB!>qr9~U|_Re{1vrW&p2HGy7E;;pAMam^- zn0jmASW2H1WA>(dpcQ9^Lr2cAta|3NSqEJh8OJjJF6EtK;}Y=Gn2neE>8E#Hy!3#s zFBKy%JHylu+ov4bnD09O;Xun-GVZcO(PgLptXTV_Q{OEDe|G8z#R>EWiB3N_!_@Eh zhg_&g*xDx+WyMG_2kn%2;%eij37V#U`v3dk9frO7zF!Pg;-$*h0phD4Lu^+9#GLO# z!ql4xkKW^--EDsHFzU%-(He^F-#Zi4n+p%6ev?%?xiFS;Oc+C{cNtOVu1^$cSAA2+6gh+T2b=rwO1)+; z-qvINw>Evd*nc&}N54e0{n=^zTZou^BgA$oM2!A9BusajKWcyQ-NFrh?>YH?RXH{k z9j-a`W@78l&T#bx!|s*OhM#?J)B;NjQ!%~ig<|eCr{zfpQ*U9cE`BR?+LXO@qiAeL;hD4;a3j=;=AXnNLl0MPk$kni`-Vm@4sIoV$Fs6bUn=1fBg~gR zX8-+k7x(j1?`IjA1vqU*liXroTG;eN%iHsj#wXCQl|~w#w$OYBnmDXIsn@y$+`eY@aho0=o1Dmlry|mbt)Yh` z1wtGRjTbugcfa}J=lN12J^>o)MS+VctB!ZGw?>gD+hI}0`WEm$6Sgg3`~8I04QBP3 zi`hd%+OCf>cF_AiOXKjEEB763P;YdV#sx@1VcQ|1WtYjN9T!%4zZzv+hpjEBnGxZ{ zDCz}=PUYQqYq_VKRXzqZGy3{LGq;&BH5Ay>%5j*-97!Dud>(!&6dzS z*~}<|W|Df{=>f}{BZSBka}0b@`$~)1D0XXJPb>EI1^)hOUut=3{N2Nl^EmvZNPP#8>zK`+ch^b zTEnAyNoVC;+s2d+()w8*af&|B+<2UtOPU)mK-0Fqxlu}bPjllF(np&cljDKUHaGAk zVTkSP=0+=jJy4^(|(H%etD4 znuX=dmeb0}fE&h8+DMNxI=0mlDg#nVDj=1j+}11WnDKS2jd+#sKesmWK^vuv4@@ci zM;n<{Gmbc0o3=(|&`o?Pb8Foy-EC=SOsxm~*V-8k0(|XMEX3iruI(?|8{OzZdA2-; ztGBp*9{O_p7te0)E!PouR5Ewc=Ph5l$3Cj4NF9ma8tZAtCuk@*AVAawuo2yaP zPBhjBq3V?zuSCgwThpv8K{W+0zSL8@Me*w;8Y6IEgsZph;&Qj{?RT}{q_s4-Ov01g z;A%6BPBPYncop>*Nyg~LUi5v*Mro8u%Mq>3YaR$2lZ_p1AjI{cV{lcWS+v~Y*W2JU z8PRfO2*fx)5{T7otByconpKmhNM`hsdo){XSlLp%paJ#4Jv^1{YI#gjuQ?{>gynZS)z`d(iQ_HSFQ=mV7}ZlyZ3lW^jpUU; zp5iKCm0z_y)r{pCFiJGP2{}Nr@&MN<;q8{eUoA+$q5|w!-?qJm&pn5+w zKJBge@ylaxR7t93wsOJM#Jt9m3$F^q>l)3h*sAX`H5V^?@41KHdIfJavXxqMQ#a~q zkZk`YF zeo(1F`-5-bF`u_Nr?|xO@)k*V8_ zn3{kZgGXFz#)qr7puc*lc0>K{DOy!ATGbV<-j^P?yu0n}wUK|R(oid`>dmV@cQn!C z^;?vsmzuoK`qht%zm+Ez&N`I-s~(e7d8z~HpA?4A!By?-wEgN6vigblYfnlQ*e^Vd z;m%prw7P0kzc!m!ow$FvuTpmE?EQClW?b#;SRLP~;Q#OvYaB_?d(?i)qThSaXppF< z?Rg+k?+|GmpP~EjX_Br-*D%r(^@e*okI>ud#?jGwJ!AMty@{RwBb7X3>PS7(o|%`G zlwfSb7kb9iBlTB}XNKs3#;6QE%IH2scNxnv^uHSGhw7omsv)rca)@5Xcw>lu&X_Su zf5DhKO1B$xhUgK-zM*=w5j9%xU*EQUu^_IrkEZyI@_kMl-o-ao0 HcSQUzgZAno diff --git a/package.json b/package.json index 5a58495..1b86fab 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,22 @@ { - "name": "cdn-v2-hackclub", - "version": "1.0.0", - "description": "API to upload files to S3-compatible storage with unique URLs", - "main": "index.js", - "scripts": { - "start": "bun index.js", - "dev": "bun --watch index.js" - }, - "dependencies": { - "@aws-sdk/client-s3": "^3.478.0", - "cors": "^2.8.5", - "express": "^4.21.2", - "multer": "^1.4.5-lts.1", - "node-fetch": "^2.6.1", - "p-limit": "^6.2.0", - "winston": "^3.17.0" - }, - "author": "", - "license": "MIT" + "name": "cdn-v2-hackclub", + "version": "1.0.0", + "description": "API to upload files to S3-compatible storage with unique URLs", + "main": "index.js", + "scripts": { + "start": "bun index.js", + "dev": "bun --watch index.js" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.478.0", + "@smithy/fetch-http-handler": "^5.1.0", + "cors": "^2.8.5", + "express": "^4.21.2", + "multer": "^1.4.5-lts.1", + "node-fetch": "^2.6.1", + "p-limit": "^6.2.0", + "winston": "^3.17.0" + }, + "author": "", + "license": "MIT" } diff --git a/src/api/upload.js b/src/api/upload.js index 03b3c13..b982bc1 100644 --- a/src/api/upload.js +++ b/src/api/upload.js @@ -47,7 +47,7 @@ const uploadEndpoint = async (url, downloadAuth = null) => { // Upload to S3 storage logger.debug(`Uploading: ${fileName}`); - const uploadResult = await uploadToStorage('s/v3', fileName, buffer, response.headers.get('content-type')); + const uploadResult = await uploadToStorage('s/v3', fileName, buffer, response.headers.get('content-type'), buffer.length); if (uploadResult.success === false) { throw new Error(`Storage upload failed: ${uploadResult.error}`); } diff --git a/src/storage.js b/src/storage.js index 655f76e..6196c3d 100644 --- a/src/storage.js +++ b/src/storage.js @@ -1,4 +1,5 @@ -const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3'); +const { S3Client, PutObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); +const { FetchHttpHandler } = require('@smithy/fetch-http-handler'); const crypto = require('crypto'); const logger = require('./config/logger'); const {generateFileUrl} = require('./utils'); @@ -63,6 +64,178 @@ function generateUniqueFileName(fileName) { return uniqueFileName; } +function calculatePartSize(fileSize) { + const MIN_PSIZE = 5242880; // r2 has a 5mb min part size (except last part) + const MAX_PSIZE = 100 * 1024 * 1024; // 100mb maximum per part + const MAX_PARTS = 1000; // aws limit + + let partSize = MIN_PSIZE; + + if (fileSize / MIN_PSIZE > MAX_PARTS) { + partSize = Math.ceil(fileSize / MAX_PARTS); + } + + // hardcode a bit + if (fileSize > 100 * 1024 * 1024) partSize = Math.max(partSize, 10 * 1024 * 1024); // >100mb use 10mb parts + if (fileSize > 500 * 1024 * 1024) partSize = Math.max(partSize, 25 * 1024 * 1024); // >500mb use 25mb parts + if (fileSize > 1024 * 1024 * 1024) partSize = Math.max(partSize, 50 * 1024 * 1024); // >1gb use 50mb parts + + return Math.min(Math.max(partSize, MIN_PSIZE), MAX_PSIZE); +} + +// download file using 206 partial content in chunks for slack only +async function downloadFileInChunks(url, fileSize, authHeader) { + logger.debug('Attempting chunked download', { url, fileSize, chunks: 4 }); + + // First, check if server supports range requests + try { + const headResponse = await fetch(url, { + method: 'HEAD', + headers: { Authorization: authHeader } + }); + + if (!headResponse.ok) { + throw new Error(`HEAD request failed: ${headResponse.status}`); + } + + const acceptsRanges = headResponse.headers.get('accept-ranges'); + if (acceptsRanges !== 'bytes') { + logger.warn('Server may not support range requests', { acceptsRanges }); + } + + // Verify the file size matches + const contentLength = parseInt(headResponse.headers.get('content-length') || '0'); + if (contentLength !== fileSize && contentLength > 0) { + logger.warn('File size mismatch detected', { + expectedSize: fileSize, + actualSize: contentLength + }); + // Use the actual size from the server + fileSize = contentLength; + } + + } catch (headError) { + logger.warn('HEAD request failed, proceeding with chunked download anyway', { + error: headError.message + }); + } + + const chunkSize = Math.ceil(fileSize / 4); + const chunks = []; + + try { + // Download all chunks in parallel + const chunkPromises = []; + + for (let i = 0; i < 4; i++) { + const start = i * chunkSize; + const end = Math.min(start + chunkSize - 1, fileSize - 1); + + chunkPromises.push(downloadChunk(url, start, end, authHeader, i)); + } + + const chunkResults = await Promise.all(chunkPromises); + + // Verify all chunks downloaded successfully + for (let i = 0; i < chunkResults.length; i++) { + if (!chunkResults[i]) { + throw new Error(`Chunk ${i} failed to download`); + } + chunks[i] = chunkResults[i]; + } + + // Combine all chunks into a single buffer + const totalBuffer = Buffer.concat(chunks); + + logger.debug('Chunked download successful', { + totalSize: totalBuffer.length, + expectedSize: fileSize + }); + + return totalBuffer; + + } catch (error) { + logger.error('Chunked download failed', { error: error.message }); + throw error; + } +} + +// Download a single chunk using Range header +async function downloadChunk(url, start, end, authHeader, chunkIndex, retryCount = 0) { + const maxRetries = 3; + + try { + logger.debug(`Downloading chunk ${chunkIndex} (attempt ${retryCount + 1})`, { + start, + end, + size: end - start + 1 + }); + + const response = await fetch(url, { + headers: { + 'Authorization': authHeader, + 'Range': `bytes=${start}-${end}` + } + }); + + if (!response.ok) { + throw new Error(`Chunk ${chunkIndex} download failed: ${response.status} ${response.statusText}`); + } + + // Check if server supports partial content + if (response.status !== 206) { + // If it's a 200 response, the server might be returning the whole file + if (response.status === 200) { + logger.warn(`Chunk ${chunkIndex}: Server returned full file instead of partial content`); + const fullBuffer = await response.buffer(); + + // Extract just the chunk we need from the full file + const chunkBuffer = fullBuffer.slice(start, end + 1); + + logger.debug(`Chunk ${chunkIndex} extracted from full download`, { + actualSize: chunkBuffer.length, + expectedSize: end - start + 1 + }); + + return chunkBuffer; + } else { + throw new Error(`Server doesn't support partial content, got status ${response.status}`); + } + } + + const buffer = await response.buffer(); + + // Verify chunk size + const expectedSize = end - start + 1; + if (buffer.length !== expectedSize) { + throw new Error(`Chunk ${chunkIndex} size mismatch: expected ${expectedSize}, got ${buffer.length}`); + } + + logger.debug(`Chunk ${chunkIndex} downloaded successfully`, { + actualSize: buffer.length, + expectedSize: expectedSize + }); + + return buffer; + + } catch (error) { + logger.error(`Chunk ${chunkIndex} download failed (attempt ${retryCount + 1})`, { + error: error.message + }); + + // Retry logic + if (retryCount < maxRetries) { + const delay = Math.pow(2, retryCount) * 1000; // Exponential backoff + logger.debug(`Retrying chunk ${chunkIndex} in ${delay}ms`); + + await new Promise(resolve => setTimeout(resolve, delay)); + return downloadChunk(url, start, end, authHeader, chunkIndex, retryCount + 1); + } + + throw error; + } +} + // upload files to the /s/ directory async function processFiles(fileMessage, client) { const uploadedFiles = []; @@ -98,21 +271,52 @@ async function processFiles(fileMessage, client) { url: file.url_private }); - const response = await fetch(file.url_private, { - headers: {Authorization: `Bearer ${process.env.SLACK_BOT_TOKEN}`} - }); + let uploadData; + const authHeader = `Bearer ${process.env.SLACK_BOT_TOKEN}`; + + try { + const response = await fetch(file.url_private, { + headers: { Authorization: authHeader } + }); - if (!response.ok) { - throw new Error(`Slack download failed: ${response.status} ${response.statusText}`); + if (!response.ok) { + throw new Error(`Slack download failed: ${response.status} ${response.statusText}`); + } + + uploadData = await response.buffer(); + logger.debug('File downloaded', { + fileName: file.name, + size: uploadData.length + }); + + } catch (downloadError) { + logger.warn('Regular download failed, trying chunked download', { + fileName: file.name, + error: downloadError.message + }); + + try { + uploadData = await downloadFileInChunks(file.url_private, file.size, authHeader); + logger.info('Chunked download successful as fallback', { + fileName: file.name, + size: uploadData.length + }); + } catch (chunkedError) { + logger.error('Both regular and chunked downloads failed', { + fileName: file.name, + regularError: downloadError.message, + chunkedError: chunkedError.message + }); + throw new Error(`All download methods failed. Regular: ${downloadError.message}, Chunked: ${chunkedError.message}`); + } } - const buffer = await response.buffer(); const contentType = file.mimetype || 'application/octet-stream'; const uniqueFileName = generateUniqueFileName(file.name); const userDir = `s/${fileMessage.user}`; const uploadResult = await uploadLimit(() => - uploadToStorage(userDir, uniqueFileName, buffer, contentType) + uploadToStorage(userDir, uniqueFileName, uploadData, contentType, file.size) ); if (uploadResult.success === false) { @@ -274,36 +478,128 @@ async function handleFileUpload(event, client) { const s3Client = new S3Client({ region: process.env.AWS_REGION, endpoint: process.env.AWS_ENDPOINT, + requestHandler: new FetchHttpHandler(), credentials: { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - } + }, + forcePathStyle: true, + requestTimeout: 300000, + maxAttempts: 3 }); -async function uploadToStorage(userDir, uniqueFileName, buffer, contentType = 'application/octet-stream') { +async function uploadToStorage(userDir, uniqueFileName, bodyData, contentType = 'application/octet-stream', fileSize) { try { - const params = { - Bucket: process.env.AWS_BUCKET_NAME, - Key: `${userDir}/${uniqueFileName}`, - Body: buffer, - ContentType: contentType, - CacheControl: 'public, immutable, max-age=31536000' - }; + const key = `${userDir}/${uniqueFileName}`; + + if (fileSize >= 10485760) { // 10mb threshold + return await uploadMultipart(key, bodyData, contentType); + } else { + const params = { + Bucket: process.env.AWS_BUCKET_NAME, + Key: key, + Body: bodyData, + ContentType: contentType, + CacheControl: 'public, immutable, max-age=31536000' + }; - logger.info(`Uploading: ${uniqueFileName}`); - await s3Client.send(new PutObjectCommand(params)); - return true; + logger.info(`Single part upload: ${key}`); + await s3Client.send(new PutObjectCommand(params)); + return { success: true }; + } } catch (error) { logger.error(`Upload failed: ${error.message}`, { path: `${userDir}/${uniqueFileName}`, error: error.message }); - return false; + return { success: false, error: error.message }; + } +} + +async function uploadMultipart(key, bodyData, contentType) { + let uploadId; + + try { + const createParams = { + Bucket: process.env.AWS_BUCKET_NAME, + Key: key, + ContentType: contentType, + CacheControl: 'public, immutable, max-age=31536000' + }; + + const createResult = await s3Client.send(new CreateMultipartUploadCommand(createParams)); + uploadId = createResult.UploadId; + + const partSize = calculatePartSize(bodyData.length); + const totalParts = Math.ceil(bodyData.length / partSize); + + logger.info(`multipart upload: ${key}`, { + uploadId, + fileSize: bodyData.length, + partSize, + totalParts + }); + + const uploadPromises = []; + + for (let partNumber = 1; partNumber <= totalParts; partNumber++) { + const start = (partNumber - 1) * partSize; + const end = Math.min(start + partSize, bodyData.length); // last part can be below 5mb and below but not above normal part size + const partData = bodyData.slice(start, end); + + const uploadPartParams = { + Bucket: process.env.AWS_BUCKET_NAME, + Key: key, + PartNumber: partNumber, + UploadId: uploadId, + Body: partData + }; + + const uploadPromise = s3Client.send(new UploadPartCommand(uploadPartParams)) + .then(result => ({ + PartNumber: partNumber, + ETag: result.ETag + })); + + uploadPromises.push(uploadPromise); + } + + const parts = await Promise.all(uploadPromises); + parts.sort((a, b) => a.PartNumber - b.PartNumber); + + const completeParams = { + Bucket: process.env.AWS_BUCKET_NAME, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: parts } + }; + + await s3Client.send(new CompleteMultipartUploadCommand(completeParams)); + logger.info(`multipart upload completed: ${key}`); + + return { success: true }; + + } catch (error) { + if (uploadId) { + try { + await s3Client.send(new AbortMultipartUploadCommand({ + Bucket: process.env.AWS_BUCKET_NAME, + Key: key, + UploadId: uploadId + })); + logger.info(`aborted multipart upload: ${key}`); + } catch (abortError) { + logger.error(`failed to abort multipart upload: ${abortError.message}`); + } + } + throw error; } } module.exports = { handleFileUpload, initialize, - uploadToStorage + uploadToStorage, + downloadFileInChunks, + downloadChunk };