From e50b63c1117a871159c4870d28bf1019172f7e35 Mon Sep 17 00:00:00 2001 From: thomasabishop Date: Fri, 23 May 2025 11:06:13 +0100 Subject: [PATCH] complete script --- data/wakapi.db-journal | Bin 0 -> 53864 bytes wakapi_import.py | 112 ++++++++++++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 data/wakapi.db-journal diff --git a/data/wakapi.db-journal b/data/wakapi.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..766181b71f982637f83bc632a584b863aaf0d5aa GIT binary patch literal 53864 zcmeI52YeLO{^)n7Y?*BoA<`F6AsE8U?re*NP(mjJLJ6@Xv%9kp2uVzWv%IP!b zoH=W;sDEvFTfVkf9yw&OP+;t)26Sov#VPOz%TfQdEV8l|?_=J}y*1wP-a(!p-DBMY zT;IF4xgK&|>`J;uyZUn9aGSaNxr;c7JC@6He&u}Kd5`ma=gH2P^DxI3j!ljg*6ST% zM|b_NNNcF4BQcB}1AwmCN5=3)<6o7vmhv)BqY$a}ZHsyxM>!5+%} zfqR{Mh5I7+0_(fhHB>+A&(wUXh}up)L0wCoPB+jq=>Xl+bjBUQ3>!%|FRWi8HA(TB zriJwliTWiCOPl4!s-{GZT$hm9${A*yCYW(5G2v8EW5#LTKxWureB*VCmM)N*>l>|> z`#TQ4ujAkcI}U!J-B(Nnc?Z)Ir(k_~_Lg2S3qq@Z%i^ zukyZa@#JpJL=mW*WyWcy8K*KcPE*Y|O)=v%*^JX9GforDI2D_58g0hucoR+)znO9R z)r`|(Gfs;-E><_0ImbdXP7BO9rOY_VW}K2{oDyc7;%1yAGfvfJoT|(?oovSGBr{GY znsGY8j8mlBORVN{suFe=P+7!_tZjCpnwu`54l#_50=rw`3I?Kk7}ff=Xw z%{aYh#;Mhe)4q=Tr+YgNzNh2hyE_iPtK;Am9S7gpaqt}-2jAXt@NFFjw{#qQYsbO2 zbR2xMnS9%8#_3%%PVbm;+GECPw;88h@DP1u?m_zZ6ccBvG}9YYn&}NH&9uCgW?J4# zGc9lBcr&pZ*OvG8F&4|pPdMgd%Wu>atM4`6^S(!YD}2}a{^mQ|cZx6Ro9CP2JI=@Z zhWdK@oZg?kUwGg5Zuf5VKI47Jdz<%a?_a!s@-FdKd&|8OyfJTqcaXP-m-T$_`NXr= zv&FN{v)XeXyt^;=obNf+Q{y?&Gs8316Y>mqU+O-`-RPFx74E6-FbN%9K zbA902;d;gOtm|Rd?XGKFe|7!YRp*jivt1KiMXr%9zpJOq#{Iy3%I)K}a_hM#VPxVK z?(f_M+-ck*?j){^8^?vYe6By|<0$7h&V$Z9&NrMdI9ECEcHZE;%(=|jFh|H5d|nMx z>tZd;H@0hSm)g#?ooZWbtA!L{%?U)xW9gWvaSdRj*Lh z%T@I%RlQPG|E{Xnsp_?=`VUpTMpds?)mv2cW>vjORc}<)8&vgrRlQ49SE%Zps(Od2 z-ma>*scMU=-m0pPsOm~peOOf=Qq>1l^#N7AUsdl@)q7R-9#y?tRi9VYHLCiYsy?f# zCts``|wKB=m!RrLu~eOy&nsp?~@`Y4oKSGGe?t(#T#O;vqERbN-t*HraYRo$ej zuc+!qRo$SfFRSW$Rb8j5FRAK_s``Seu62BA*_VyQf1}W}0hP10P_BhpT9~P(&Y4vl4a|R63-4&*0xg`c z1yKt@Ed;b6Xo1(lC@mCfVWb8se$&FQT3Dln=d^IW7OvI8Vl6DvLX8#{YGHvEQd*F; zko4`cobU(L9PM?gv~aQ(PSV1OS~x)qm0Fmug$gaq)52UW%+Z3jE>&pjQiZlIRm{{L zlkV7+PP{_biI-|av^A?jTeB*(HLF5fvnsSTt3q3|Dzr6gp0+5@J5O`bdFN_jnHJ8` z!r5BTmfm^V(mU@6t=kYS4Az2Q3xl*UPz#4^Ax{gvwa`lohiRdw7J6u*yB4}>!KVeU z7Cc&TYr&-jP76*gI5aT#C@u8Sf;QT7wb7odjrLq^wC8H0U4Bq=w(V>L$fvTRbs^_WdxvIKMRnJk?vsLvh?Y(`dh5cIiKnw3{;XN(1YGI!i_G&@f z;FoI~{PI0ox7}LUrG>Y(uu}^=w6I+ZZ)sth7Pe|(ixxI(;Y}^Pp@rAA@R}B0)xst% zyrPATTG*h4m$k573+uG-k``Xn!V6khtA*#autp2o-mLstZOAiPcv=fjY2isNtk%L4 zT6kOwtF-W#79Q2YBhbeUGttkrs+4xU>{{)*$928y5_tQrb)DoY zg>?YDYp|<_i{}2xeaP+LHo%+rz1$7l-?%fmCEUr}46c|HxFK9mj)AxC1J0e!jn1c> z_c?ELUP}AnZI+_GruI|Y9Zx&%ciiN-%yBlnWmh|9I>tJJjw2ks95(xR_Jj6a_D%L@ z;0^m``{nj?>}537v^(jpS!@_55^~caDC+3 z?)%*Lj&GCiNn5M$Vc%W8n|)WqXvulLGkguc1-_Ggb9~c$<9tOv!8hDD2u4jjKF0f# z_bcxqZ>x8w_YLoQ@3XMp;yxHXxz2lq_ag6E-e&J&Z`?cITjrhUJ>DCFQIw(HzTWO$ zhvzrXx1P^D`(XvdR?jP*wJ@6UkY|PGCeKx#OFZX#PWRM%Ql67MvprK`R3+x&J;OW$ zJ-s|`5AFUDMjH;g_qlhtUw5ygN6};GD2yabq3dXwK9RnXzL;J{pGH3qV+s$_chWb~ z-_W1ZAJDt$E%Zj_I7VcSW`@AnLN~@v|H|CUv@q8)mopc_XhIXSh>@5IW(G?$KQdo3 z2Vu-%2lG0!j(LW8G+orZ^ZY}q8zIz$0*8JMH! zIxOF<$6b1STaP>SxI>TI_4t+^x9M@K9=GUmvmW2n;~RQ>U5~Hn@l`!;(&H<7+^EM5 zdVE=r>-D%!k1y%*MLoWt$F+KVUXN?^_?#Y})#EdId|HoB>G4TDuGZrddVE}utGd%e z=^61wa-zBBRC(k$tA8D`ST@h>Mh~W^*VNUNCmL%Snwi%=LWU=sk)fhqZ=g@y0UgLx zGrh10A6*P8^$&EF`Ukp7edAr3dkBgs&j?{f2`P%GC_zODD2kvcyrPU!ltM)rsVD`C za%i}zhcQ7-CLo~FIZ)x9SxJ|`*4iHdT9 zqA1VuUJ6xc>(s5-;@{ZLvZ0;j<+i-H*1~rnJ2+37WruaM1=id19>1rX$Kq~tx5CWz zTK7tKi+j0ynY+PV?Jjc{yLq?Y-OX)*dHYt^X4hKRN>__(xoerL!By=la}~p=f#225 zW#QVmR``lwEw_?u;g)mDxCX8o=IV<%p7V3vIE%B*+3MWvT}D7j zSjo1q%h_dY16$3OvBfOU`q^$UJKn~$GMkyT%u09{Ud}9I8klOPj46h17X0v*Y@yre zR(dnNmR<=n^vmJhx`D2y%jja7htZF2v;}7NTdB>|T52WLLM^A3Q4LfzRYnz4Jmsgl zQ5I{Pwbi=W*XC>WZT7A8t@O3{miv~$h(fim%vTKSFZ{l4K8v>v#t=4p*LqiaTfED? z%e)P+@}kUJ?B%`w%u9CAM-QUw>YGO{uFG&B%@zx?h7pGli---+s=11Q{CUKq5EmjI ziMRmq(TGPN&PO~P@i4?kAwClE5r~H(9)frF@u;!Od+-+M)hs}Z~E~0 zzastx@z02VLi{7*9}s_!_&dbkBK`*PKM{Y8_$$O;B5p(c1>(;Ue}?!|#GfGk81W&* z2N8dS_yFP$5${L*0pj-&zlXRL@jk?R5x`JL0zxZ$rEl z@fO6J5xhUWNEE#E&9=1o2A54sTKzu*q`w-uY_#VV}Bfbmq z3dDCJz60^?h;KvOg7{X%w;;Y5@lA+tM0^9{>k(gv_*%sOKzt42s}Wy?_)5foN4y;I z6^JiKd>P_P5&sSGC5SIZ{8z+(L3|P73lU#{_u5n z9O7dUAA`6EaSU-3aRk-_w8A*RR?gLC*#hGLUGMh)ci-*3-Mki0o2S*Y8P))-^t8bD z6w5pfo@!5-GD4Gk<3AGR__BWsY(vZp_oDRp9Pwv}KSlfr;*Sv@LVOVMM~Dv~{t)qg#2+AjAMtyLTM_R=ychAih~GiH z2k~yiyAZ#Pxa+-FHyS+uWt2Ya5wAo165DBe#i_&8p;<1RwATCBc8u9Unk3)Pc;$sjOA&w!AB90&qBMu=J z5eE?m5DSQT#G?=wA|8pj0P)d?Mb#7Dt6z#A|Qu!XaBv(#JKEHD%8hgs-W zn1ODA+2?AQdG^DbX)C;yw!j-{HN1`bVXZ(btP^N~H3HSJKEMxa16pBSKntu1sD||b zepn083hMw`U=2Vu%>Vmg?!Oi0{aawpzuH-BX`@C^ob_YtChL9HORS5mQ>-JL0~|lW z+Wf~ISHd>~(;WE@7krKXru`B43ctxd+b+OfgP+(o+E&=kvsKxSwe@AcW8Z@H{g<4O86{; z@JLZ8qJ+;h4$s|rnPGU4SB{@y7#;{I;ic`vgV7>2e7Zq+xG0dj|7nBi5(?5dG?Eua z3PHah6p4W%UMP%2$t5hm4WTDu51}X!E)<3AU08lKbm3!#;b1nRmS2or#KLGWn^u;e zf6qnW!`XZJsr}tTY=ux^IGdrCAKRZK7%39@!ibV5mLCjVf_z~poJ|(X_l7P)tS}l- zlI1&Nmq=lRSK|Dwp$i`_j6{^%{l>s0Tm<(L6_q&u)4(NEB!&tD;cO|jd~N6w3_#*4 zxBHcWi&zv27Y0Mhg}yX&fus`x%7xktU4qfVU@qytFm?$QilTBapBv<&n2vNLmd)qS z3{Mh>7KUTWt$b?iA{Gjwa+XgFT!KZxbS^4!{@CDFz$IF!v@MoHh9?O?*%p<2K4|E| zr=zUg%0~t+0ca))qfw>!A249Oj&nr0+Yby+0#ObG zlv4S=u}h>-%;nH~1}@Na#0q1gQi)oPT|$Miu#)`y3|#o40ACmulzZ9R-X#DHTnv&$ ziP5{pE+VvH%Duc}>=G;t^GeF?8Ay*t?HnH{5|lISM!`JvB2W#L+uUUk3}$sCl>Nb7hu_G z;1VfHm)@9CVRsm~gwst>Bv-(<8@mLdty41lEn^pW5dYK!4X((QXtsYsg*dPOxe*-Fimhvs=h%=OBX4Ubn6ovoZ?7#_$y6-+b?7qaQi zR!%Su&z2^(vcw=K=E^U%sDPbJ01 z;n|YFR*r5TE<%ycg&*HOJOHg&?*5N63|Crpw(?kAxab$)c@gd}mww0KaEM>5NGXPF zWsyO6y4oo5iy4H&Q(rFsqK4sW{zMGJ1G)4I8-%BOYvuYOLdKR{FWGCST?Zctd237K%@xYun@)pF63Z<&`Din@%`ZY%J+$Hzwce&4&R%;jlLIr zPx~J8Jpelg-r~C!R<2(R-^2ge*XOZlpN6~39Ui{LolvA&S6&^HXm5c>Ff_+0Q^ z{4d^bVMoD(-uJw_z1zI6!ME{iysKe#`@P=Vy*I+{f|tSf@#lC?_nzWi~? zdM7dyn9)p>5ttFoP^LfA3wFS>!wUQ#=&$Hc=>7D&^bYz>dL!(Y_cZ+&{Q$j!zJhB7qgXl6~2Cdj(LK47A-~dyD%S_;rIT+-JBG?&0Zh`#}8(}|&iLQXFKkS3p z#_i_TaSw6Vap!Th+*}w>8NnUq{1tXR-0FPRc^7<1@h4}}S?Y{BhdAAiZ{Zt?O^#KL zTVa>OCdWyRi4MWh&%wgC6T9u}><`(mv!7?LgI{kLYcGKP7A&@pZExAu*zU1iX*<)F zvdx6u6^^v|*dN#r*w@+B>}~92>}jmTPGv)EM?32O+l#P`w$N_aWoMlh{z{mI1$5$IEsiPi8z9Yp+pQJVlWYYA_fsLkca_9^e3Vp z5q*j1L&V`k_pgzV2NOe zfK4dVX-yGfCBl*s^M51aS0a8P;%6d$BH~9Pejws|BEBQy+l;6zBVq;-r9@08Vj2-s ziI_sfWFjUJF_DM~M3fLQo``Woj3r_W5yeD|CgOM^jw9k&B90-Vh=>>wQ6eHlgoy|d zArcYnMUSKecsY*np+Jm}MM9}0Z(pxig`g1R6Onj85~A+@#41L`KvWbXA}`8v%*|%4 z5nhsnlo*am5rKDoN36ks2yd8tG9KlFk&xw3PdXp2mVV0<0tq=7NhaBwh$qiWVloxw z6G`|#==wTi4W)!+l#fYbG!+Q)vVA78%JHx)r=rQA6qWgy<@2mHoQ#J=Q4(W;cvQ4c zA=YFt7#756JQxY^$q02eu_jWyASObx5R+v|V2;RI1F2*xCWM2DNGupiiUztOV%2Y!eKce#X=z|oS?r@tf5di zEGA;fgeWH*cN1$UB%~lx(O^snCZgP~tTiYH!l951Sr!k)xgl9=K$gKSCZoY*ILI!} zT6r-lq{0!njZi3MpFpf)NRGw>kc)9K84uXcB32j> zV-2RF2_Y1e6RBh%kdUmTm5eO&vpbR~x`9}s z$i-7qSdzq0C?0l^_62NDUHxBMex4Zww=qR7#h zD5v59*G6K6#v=vk8kOO0g(Q7z)|yNTu^?2yR4f#auq%i)9tfvI3I55@5+t~BSt}1M zhRg?2qL4@ht)x~3Bq&~yL^z&|iOGOqeVzD1$$=&#o`^&7jRmY>VvR;(Au$z}Ad$kc zxc#-Pl}{uBi2$GAQ!y#(7Kk+*2!!}}JRn4qF*!~-h*jWakq@SVah~U+5t@{A0gg*a zkT=l6KB#A;LjO3uuuX*Arhc zDW?P(DrOj3bjk5T)*6pP50navLQDz<>|4n#h++t`Kn}wdBXKL~Qv~>v;OKw=&8`&T z?ja|D*|G#wt584|!+|98Qq~IH5_D=pBFabNadvRl3YiuPB*Wo^6qh9Ky^NJl36U^V z;3!;76lJ$Zu?C>&hnWCKxTt8okyxQ@MUv4#GRDh7IO=#YYn4KAAudD%K_M9y-DePM zG6fx5N{;Y}ICQ&oFl$XhHz)GYH$xvS(Y>=)F)j*W$c;F3uOjOw)_5WsfV0ISCIGp`Zl)T0rJVqr*dwD#v3`FOw3K3uY5J zZy1^z=vG5X=uJa$W(To`q(G7nLtRbq(2-iQu@L1XvMVPNMk4y8vA>yAzWQ|`D`jUt?BEBHvb0R(?;!`3%A>v~q4iRyXh>wUk zK*Wbc>?h&_BHkzBJtA6(*hj=(BHkt99U}G+v73lpM7&MJP9k;?v7Lywh}cHNRwA|# zv6+ZBiFkvE*NJ$Ih*yc&M8qpZY$Rd>5ib+5o``isyhOx{M7%)6S|Xk&Vhs_`5%DY$ z&k*r65l<2EBoV9I@^(A~;{e+@%U;V&%SwyyPT#4%Nxr_`PvO0NIqYF4dTpM!VaNY7 zJ*6JM`z!dJzJItEx{KUy*Iw8e;5^vH|0wQzZWDJCR|oH^J)HZUPdNW-S!o?-?d6>3 z90qTw>m64+5{`hwYTs_Z7k*1{ru_)pceYKo>tTn#h|R`sXYXX2VCVWC%zN-H{y9u3 z%t(AjKTH3OuB1m$KTsRtHvm&qh+?eUthZZFfg6Rtk<75s`d`W;^Zd$aJIv8*=Gc8( zEs_6?msS%OQHO+QoAJ&Ku$J@8u!)rJHwtsJJDEvzm~<9&bHOGK(9aBadH|j$?(_gW zYi!~G{Ty-(!;C{S(IzgSpO-dsfNqw$(*y9F zb*E84GuS3Bpr6k+ae#hyyVC>kT(_A6bTi*34$#kocbZ6OR@}q|^mF894$#e*cX|Mx zKR0uLZZ^Hs1Mu8>X9u*KX*Y9#ZrTd7d6hQoxnW<`eyZ_WA+Gv2w`%#u#Mb!$I5^VY8f`Io%M zcI=H;iFE3%T@z4j#ydBIKe{t-{d$tly!ER~On7V7nsnf8u=1p1Z@dnrQ*Yg>luo^M zYg9~lYgVk7@Ybwf>C{`dn#GKF?$t{)t0NFtE(IYGv2v18+@nU+S)J* z-ggVlzw^R40PGk2h?8@?Wx2unl*M<*x7GIyj0Ri=;{jDLA|Uws`e+yx*bd_Ycf-iQ z889|56-EaJ!1%xyFhcNx=RVjyFlH<8T<`g-=QP;GevW6nC+HdC=?>oyeBs^;yVgJJ zzTbU=`(pR$?giGT;Cp~Q)?cij!hU&!C@+I6PyQzOr7gEjC&(tR{SN{t2B;7>E zVP3zO9z_qNJzU89PrBp2Vk^fo&7QU?e@RJ-h=h>)7xbHoS@rt=O;+8}?$uyV&p!HtfNM z-Po`T8{Wo-o!GDg8@6M^SJ?0+Hnd^G7ufJQHhhK+pJKx&*zhqn9Kwc!*pS)lS=$&K zz#boB!+vb|02|)NhWD`HCS-(bT(vEgfrg_g}f33YjPr9f8-{L3lOn;r?zpGz8+!jDH+H6?1~ zx`Yf%I*rZcxQ;%I4G&?%gV^u@Hr$U5_hG}m*l-Uv+>H%)VZ#b+xDy-hz=qqg;Wlh& z!G>G0;TCMT**Ilq8JlMso6C&NGmOop#^&k9=4r;}smA6h#^%Y!=1IopiN@v$#^w@Z z^LS(PIAilzWAhkebFr~`w6XbkLvzJ%#^zs*&5MoAi;T@R@Lj-PU>sl{d>3GOnQpQC zpX^`X`5t#LGSJrbD+FeLh2URqle-T1uT258uSeH@t-5`YD!cZr(Wl3M{Ju4)ydR19 z2*g7X4?#Q_u^;gu!~+ozK-?d3Kg2M0uKN|2K8O!ToQJq;pHqGMu_%3D6Chps(ugU< zR>T%PckOeE(xYpiQ(gI*|1L_OcM$JEyc_W@#BU?siFgO%?TFt(ybbYI#9I(=M*Jq? zHxR#$_%+0@BHo1f6~r46Z$R9&&#AurJ&w|672?MbKZ^Jf#48a$jQAnM4_frW_1BdGT`AC&0$nNa|9uLad+If-X7{xWJ2%p+&&WOl9D@c8vW$jQ zDWMz=5W0o%qvkZpjZLG7End^Ku)bkbL!;bK-!ux(AdMO?HN&^1O>)zyiH%aj!kN=X zjcKT_t*y_Bs&cvURJpNmNitPaD<|rcHFXQ{IhIIGFef!#YFwOz-#p3v$Z30}-4|ra z$6f6Mf`xo`g`nm0bC+JTs@y2*);~7t<7#1&t@)3NdUm;F`=|$UQBR!bx@Og^bj+py zp-i9C1o>MfH7;1X1ft$lby`jH!YWcAs{YXAP`)K>mmECogqSS})|c{O48z z^>YIh_4G>MT-2cw)L*lzEFJaye{WHk-6%CJ95t>{mYOqJUzKeJs-Ow^qbfc`E&I_# z`#6WxIRBQ;?iv4W*{ys5(k|*jL5+H*%_}vE`hRYl7s+jU-7e~|SG7{f6Y17|`hQi- zHJ|3_W1bE`F`xF|6?647BNTH8P#g9{x}r}tiur$RyRO;%9>pBKrP8! zdVFgvjPFK^Z!fF}co>wfzpfPMN`bBv=t_aE6zEEUt`z7>fvyzjN`bBv=t_aE6zEEU z|H%~S!O+8~rlm`kNR4My)ildXnk;KQG(E^lY&CMzS#Uf4j&7-&6+RhUC`*maaTsWA zg70>`6g`MaE^UNi*ZR69uhf>e1-_%RSj%TlgG=~R^^HrUW`EGq&0@7$iv4~IJ1YR| z1rBMhG6Zx+!&Z%MtftRVcu_tJp;q5dX`g<+pIldSTU|4q_d4e}eskQ&wy|fjM=)!d za)yCl0+8*)S-v$s;lImz-k9F@yg`Gk7tDm&s9HIfusJ1f+^mu@vrGK5$Bdm; z;;+uxtNp{PYm(Len!09rf!ydXn>Bg*m|64vQ%mOi$IO{MV{$1RIK8BFc0u)07}2gu z`!>s`H~ULx%=VYgnKrGUI@MUeq^h}Qi3}m|e^axxq(KwhT;D-p-O?phYPRNXqhQ>O z((>7}#!N1q?XONPuFBnRRr)5X{S#))Dw#a7G;^23b2panpH(uUWL8P(xRP>z#@bY! ziCw;bMyY>%$+QwkfN^8W$Bh|ZlD5n#8=p>+tR;VOFMHm|!Pawo)zl^B(`%BaXD(LN z&{)4nPBg1`**lw*lS{{!RQMtF^p4f(Q&+3*a0^Px6;x+WR-Ipdm@RMcU@J@)Wv>-) zlR;pXDR9Yabr#-CwX%#1f6X$~R%X^^v zm6J8Fje~mPp6yReTntW|4alF{-Ih0Wu=TXu{VtIb3v22$x7(xrS+hO{r&U7or+2gE z4H#@)oI7v2T&lO)z5Pi^@okrJ8FT((pFJ-K1;dv+pL(vUg)sf2Vz=z6=af#KIj6+0 zDI%&*<`9G1PoJqeKkl{X4TFn0a~DhBM5ZXJr|=oaAX_`yMMEurBtmmNDBatYac}zz zC`Gq@@m50e8{PK25Zrse+`ZQ~$mtdu=0uy$fJyEpY7{)}uYjEm(x{zF{)NL`_H<3U zq+h1xQ!k)u?(C(qZJ!$OZ)p5rmba$)jCO5e^BE24<`5dng6f0>O>2E))=6yxvcXEN zCeFDUmkq6fIpjv@(5n5=RLYGtiGpgST9E_VrAYQbr7~nOQv{$TsB5nFpDHyb7D|o7 z1EElUK{YA*>Hjk=_wXRtGDR`xCq%$h-&nJtrVbxWteIXxf7V2LLR3yxNpL;=xoe?< zH>oXsb(*W{>NS?c(#EFx#wu8Dp)qFMl5(w_>F;rXl!WFe{UD$T$QH}0#`^l^szm+L zbkwum9PVG#BJoFdFmcYFC-k$P%jT+wR9ho8Riz6{)nfUK49QKd_BBKAN9sk^8@iWF zVJoPHGgRl#b=vas`)Qg~a%ejHvj=d$clxvsKDE+vYc}E{@olhPwPN{_=2{YWM zEKPq!-K0mSuDYSX;dRmyxd4vLKkq2JJ+EIs>jgtI=gi(+RycI$sLopdh5AY=pz>6s zKCCDa%2W%@Q%XAVuRZY!}=_WOE ze@*QSHhtQ)3s0wFF1iKPnphTSpZ}7+*(jAHPocl| zT{mjt3aV>iHCI(_{elJ1`qk9w8^D6dYhCqjct0E5+s7&YYw~bIqyHaaN@}B50R?C#KN* zNy#PYZs&I#prgtp=I=V3CM||cga;^Hwii_E>`NPKl}v7wo8;!I=K95QF5lA&{~FV4 zxny~9EtH$zxs`^c@!Fb1RW1CsjNBNnm(m60cU+s6#+wq2HE}2*&@0uXFRpaIkh>(X zAzk#U8s$3qGzmFC?S~E{v9zhV9v%sDuVe+)HFc1taMF10SwlWOQM)t=kB@bW>tJo6 z@-m}l5A^l9x9ZG$TK&>yXxJ0Et}<8n!aR@K2dGo|x}J1tT}`GTgj`c9S@uDAc&^&z n=jYS5yyNnIuR+Pw*s4HAF!@yHu0qC@CU4SxdbZ{5T=%~KEE*4i literal 0 HcmV?d00001 diff --git a/wakapi_import.py b/wakapi_import.py index cfb0c9d..8930fef 100644 --- a/wakapi_import.py +++ b/wakapi_import.py @@ -1,29 +1,105 @@ import json +import sqlite3 +import time +from datetime import datetime -# def parse_individual_heartbeat(heartbeat): -# processed_heartbeat = { -# "id": heartbeat.get("id", ""), -# "branch": heartbeat.get("branch", null), -# } +from rich.progress import track +def upload_heartbeat(heartbeat): + conn = sqlite3.connect("./data/wakapi.db") + cursor = conn.cursor() + cursor.execute( + "INSERT INTO heartbeats (user_id, entity, type, category, project, branch, language, is_write, editor, operating_system, machine, user_agent, time, hash, origin, origin_id, created_at, lines, line_no, cursor_pos, line_deletions, line_additions, project_root_count) VALUES (:user_id, :entity, :type, :category, :project, :branch, :language, :is_write, :editor, :operating_system, :machine, :user_agent, :time, :hash, :origin, :origin_id, :created_at, :lines, :line_no, :cursor_pos, :line_deletions, :line_additions, :project_root_count)", + heartbeat, + ) + conn.commit() + conn.close() + + +def convert_utc(timestamp): + dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") + return dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-4] + "+00:00" + + +def convert_unix_time(unix_stamp): + dt = datetime.fromtimestamp(unix_stamp) + formatted_date = dt.strftime("%Y-%m-%d %H:%M:%S%z") + return f"{formatted_date}+00:00" + + +def map_json_heartbeat(heartbeat): + mapped = { + "user_id": "thomasabishop", + "entity": heartbeat.get("entity", "no entity"), + "type": heartbeat.get("type", None), + "category": heartbeat.get("category", None), + "project": heartbeat.get("project", None), + "branch": heartbeat.get("branch", "not-available"), + "language": heartbeat.get("language", None), + "is_write": heartbeat.get("is_write", None), + "editor": heartbeat.get("editor", "not-known"), + "operating_system": heartbeat.get("operating_system", "not-known"), + "machine": heartbeat.get("machine", "not-known"), + "user_agent": heartbeat.get("user_agent", None), + "time": convert_unix_time(heartbeat.get("time", 0)), + "hash": heartbeat.get("hash", None), + "origin": heartbeat.get("origin", None), + "origin_id": heartbeat.get("origin_id", None), + "created_at": convert_utc(heartbeat.get("created_at", 0)), + "lines": heartbeat.get("lines", 0), + "line_no": heartbeat.get("line_no", 0), + "cursor_pos": heartbeat.get("cursor_pos", 0), + "line_deletions": heartbeat.get("line_deletions", 0), + "line_additions": heartbeat.get("line_additions", 0), + "project_root_count": heartbeat.get("project_root_count", 0), + } + return mapped + + +deadletter = [] +print("----") +print("INFO Collating heartbeats data...") + f = open("heartbeats.json") data = json.load(f) - days = data["days"] -first_day = days[0]["heartbeats"] -sample_heartbeat = days[0]["heartbeats"][0] -formatted = json.dumps(sample_heartbeat, indent=4) +total_days = len(days) -test_batch = first_day[:10] -dead_letter = [] -print(len(test_batch)) +print(f"INFO Total days = {total_days}") +print("----") +for i in range(0, total_days): + day_count = i + 1 + print("----") + print(f"INFO Processing day {day_count} ") + hbeats = days[i]["heartbeats"] + hbeats_count = len(days[i]["heartbeats"]) + print(f"INFO Day {day_count} has {hbeats_count} heartbeats") + if hbeats_count == 0: + print("INFO Nothing to upload") + else: + for j in track( + range(0, hbeats_count), + description=f"INFO Uploading Day {day_count} heartbeats...", + ): + try: + mapped_hb = map_json_heartbeat(hbeats[j]) + upload_heartbeat(mapped_hb) + except sqlite3.Error as e: + print( + "ERROR Heartbeat could not be uploaded. Sending to dead letter..." + ) + deadletter.append( + {"sqlite_error_code": e.sqlite_errorname, "heartbeat": hbeats[j]} + ) + continue + print("----") -# for hb in first_day: +time.sleep(0.25) - -print(formatted) - - -f.close() +if len(deadletter): + print("----") + print("INFO Some heartbeats could not be uploaded") + for dl in deadletter: + print(json.dumps(dl))