From 091032766683c9a0135c10fb28ba7c5759051afb Mon Sep 17 00:00:00 2001
From: Jakub Dylag <jjd1c23@soton.ac.uk>
Date: Fri, 7 Feb 2025 00:56:58 +0000
Subject: [PATCH] Remove Old Files
---
.gitignore | 8 +-
img/nihr-logo-1200-375.jpg | Bin 17028 -> 0 bytes
medcoder/publish.py | 31 -
medcoder/report.py | 175 -----
process_codes_WP.ipynb | 1288 ------------------------------------
run.sh | 44 --
6 files changed, 2 insertions(+), 1544 deletions(-)
delete mode 100644 img/nihr-logo-1200-375.jpg
delete mode 100644 medcoder/publish.py
delete mode 100644 medcoder/report.py
delete mode 100644 process_codes_WP.ipynb
delete mode 100644 run.sh
diff --git a/.gitignore b/.gitignore
index 69fafb5..d125b2c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,12 +13,8 @@ __pycache__
~$*
# Build
-output/
-concepts-output/
-archive/
-maps/*
-concepts-new/
-codes/
+medcoder/resources/maps/*
+medcoder/resources/codes/*
# temporary
script
diff --git a/img/nihr-logo-1200-375.jpg b/img/nihr-logo-1200-375.jpg
deleted file mode 100644
index 8a29828ca03036c03f0c9045ba9576d33e324bb0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 17028
zcmex=<NpH&0WUXCHwH#VMur0n2N)RsKV(pHt;kGcVDR;2&|+X<;9y{66k=dzU|?Wm
zU}OM;|F;>O8JHLuL5K+gm|2-YfQ^NPnU$T5ogIWZIJr4FIJh|2*|~VRxVU+EczD=3
z`S^Kx_&{usA&g8+Ow7#8Y|PATJRIyCAQhzH{|6WZIT!<(#F!bC7?=bZnFSgDA7PMZ
zU|?ir1Ou>pL2hSaW?^Mx=iubx{(ppFs{jKNBQrA-3o|P#3kw4SV=W^SGXskttB|6h
zBb#twBD+$dh*9Ijg&fLG8xM*GUHqV8oK)1r$t5N(At|M*rmmr>WnyY(ZeeNV?BeR?
z?&0Yb91<E99uXOpoRXTBo{^bVTvA$AUQt=q+|t_C-qG1LY4Vh*)27duIcw44B}<nr
zU$Ju4rp;TnZri?N=dMGCj~qRA{KUyqmo8tqdhPm+o3|c5di><+v*#~fzWVs-^OvvR
zzW@073*;|G24=9YKm^2RjHvz+WME=sVqsxsVF&q(k*S=4k%?K5g;mjzO~^5jJ+V+&
z$*7S-#A)KfjR!fEje|ajCKX-e5>qjGsQMA)HL%Z!^H>vEK7)G<;jdc^Jj{#?OoGgU
z4E79v|19d#U>L=tVKABohGrUAa&PDF!=IO~Z>fK<=Rd=?{YUI?6n`uE(BEMv^Xl3!
zyOnDydNQ{MolO;eP{F(AQ-V%IL;0~ejh9U?wj4Sy7-f=}z3-XC9G8qsrKy|j-rh03
z)_eHc{yC2S8Q6L2t}@i0c)Pfky*;PmKf?)=`XkQqTiyTi{j&cjd`<pO*1qn)erN1=
zfB(;Lz$X9eWcFwE?x|O+CfIFRxSglZ@kMR+&(DA7|7Upc_<r9#p&v&V-mh`GYVw~!
zv_j9iVb`(w`AedsV{CUb$u6q6YG2X(JZ6qzY)gZu{NDLIcB(Jp#rMg*_^1Bi-pcGX
z8BtGH#odf^y07zbzryDewp|lA6q*=W!Y9vs@?#Fml<c*?*7iFy&kQ`OeaVW;*um0-
zZ|c*<7q~PQHE?B4bpObIZf?Bxe};oe`!nOk|DBZIP~K%z_@Men$$7!d?b8Gu3w1Ze
zXz6S(bo<Ys+_d~@@|CbF8mG2;T<UwfBsFf+Qr|1JqUTqBU%z7M-S~4N^&bk(zYXYM
z|D$%xUjD1zjsFZEXT|^M$p5zd{GYyCmj4-cmL31k&>Mcae&vpTAAjHfDX0GHWXtW~
zUsgsd>?`Z7incvGZu;lJ{2vzS-<DaP?=2~lz5Gw`wc>ooi4wMb9xo@xUkq=`e|FVI
ze(Bv9v!CyBm8Q8d=sinenZsu_>r=7MbiszF-@aY@T`7A*=G-*Rt^P~*3kOXuI3MU8
zFZG|{U_pI$zU;rF^EaN|-eV{CDj?J9>G6)WCfDa>Tdcc&a+2n0hDl93PwCr7eg5zz
zHEPjP+1nfMMy<&_cxBp&-`WLV7n}Zh*#E;K|J!ore(8({W%3{TZhWwRe^&87Lvvew
zR?UBg6K@y)t$!%H@jt`k&@cC2<=Ow<U;l^s@PCF#gSmS3?zvj)icT3`pH*PF{yF1+
zh6fMpZ_WA7aOUM=vy1l+Z&NP6F34njxydSR|KigHhgk1k7k!d8(@vS;(dygLZ=dFW
zw&meV^%gJpUb<iKOOvb0pTkdIpZALT(f*Hn_wS@1zrV@<o&3l8(fRhOrQ0U2&5G08
zb}4;UwZ^(Dw@;>4E34fMos#;X=Lw(L@9S*R{8nF+-K4iCuiBdG#aDDwK0j}9X}v}L
zM}hNieQzI^vev7x|IioF)BpRc`yUt6-@zOIGfcei{-5CxB>pb_XIQzjb^2j&>bCzn
zn=|`Wy?gcX^)*4wv;LL*k^j%Y&iwC+)bSpZ+1~w<FFr+|FBb_~@!{{J>23v*j)F`^
z`~g+hzXay)aM-=?o8@|Clan7M*hS-R-OtySo5yarv}^a$O)87Jv`<?f>eA#qxPZk-
z!@eTuo!r*A)bnT0KaD?Me~kZ!{{FX?56*Yb)4KTOKZE2;!-$tvNAE7(W#0ZKVoBkd
zR|_VK`7y}M+xqh9PT!i6IT_L3c?HRvr6NMbgIE6L$(P%bA0PFv(EgF)`CE(swb#o3
zXW+ao|IgI@kBjl|kopU67xUNuaL=j#akc6{!>c+-OrHDC5LqxU{Yyz|;2-awmp8|A
z)E{KG&#mv6zuo@L%SZni1Zq64=9bn5dwf*9{B~*6DQ$-h-5DweBM!({IiF+yx_;MV
zxhVY&M{C|0a|iQAg=ppVXY+UTW(BV1TYRRA|DOC@>Emm|W_sKSss8n!VgC88_161z
z|1&f<?a$JcKRRpa((Sz_9dF;}t97Z^$!|<rwa$TM#^yPUv6XSb@p4aAM0@8yD|)<H
zE5a{6``Ys_;d`ns?y0)|x${4RAYa{8`7hQl{xkgW1Lew982*mi|7(8#d-=Wp874me
z&#-2Ozws~Yd$t+(Y_oqJ`OnbNZlBfvOZH3sA9YZW$bS@g{x<MGgY&-r{|qf}`~NdY
zE&l6&{Coes{|qPWulcCY{44ujZcD!0)<2Kxk2ueV6^z^L|77+5=qUcS;y=SeM0yJR
zxBT(%<L~RA$iG$-KJ%~ddw*2y)jtpaGoWc*`%?ZxVftIwWpO&2D;6K#)ztWJ+2h0C
zPD{%dOD(Q*3H-M_`S<bn{}~z}=Jb^F1c?V&^`F}>xR15|;N^Wrm)0-re-wVWo<BRR
zrgFNS+`MS+j&`4pXPHxP->%{>ytLojKI#4sE&hY|Zy!2w{6W5u){i!c?{{8UP2%<q
z5@7QboZK10BlOXD#R-<?2{wE3JbMqV%i3^x*HyVTag)Q>F9}`wWo;6YajHxFsmPb#
zTc6Bp*b?Z<Ad=RVX_3Xi(`DaLe0f&vgLzxS(r!ze-GBIR>d(%P+5fmaKV~1U7q#(T
zcJuYMjggtVwq9MkOVrj*aC#VSG-%#*UuoY|aZA>mfKt+s#l!#KX<g{)pvHt1;{
zjQwm~x;?FK(KLpj=85sT{}~RN?9X^FUU$)s=cDNRuJsG|@3{PaaT}X=Nqg25mM2R6
z*Y+)HNK%Zoun93VNsgO$=9BT7Mf<{g=dlatRph6aZmM7LN@LNkejSZPT?{a+{ps~T
zzJ>N5JLLK6-xSLKaxIQuvGr&1e+Kq|`jZR(*#2kOc%=TN=-Fx+`~M8eQgO>?&#B*e
z{=y$!S&aoHi}q*iuAV>ZxNozqwSfD>7mi<FzwP>~@}HrJt^Q!he};!MYHIlp*~`9~
z=YDv<NUY$}xH~7e+4INm|M1*ich4QR=Nv!kr%3YF<z8r6{O+`#Rc_Lf#_JPLyq?&%
zW1Y-lW`A+V3$wm_eOH?BG5GM+P{q&o8yyZbz7ArZ_2<BUh9<`U3}8zZ)v#uNh(Ero
z@4Z|_(&f_^_0v*&x8CNM&SbOlaEwDUOKZ@@b?;{^>5X4rVBis67}Lkc#{M+E`|*b2
zLt!OZy5WzX)lF_W>MZCP+8*R{(LG9g%1f>v#sBz}{xeJot9bnHkW~1;>CW+m*AMUi
zP$>Rq(fZ@^-vlo0|FTQ!%clPCx8fhg&pbHuKf|u`7w4bzu6_1vmPY=b`@ybtcXemm
z8-4oB_x$DG?B5JO%KtN{{AY0f&#<1u{!7&LkN+9&{Iy?OAOD|$Z~v3|FY3QOWdGM4
z^+V3;Kg0C!&wu&XJj{JnrGMq-`g8YxSnl}GaBNpj{hPuK{~0cdhyGHHpF96UarvLc
z>yH0B&dvW<xHMjQ(tn2iLH`+q?7EHqGnCC=_~&n+tLX2vU;CCnGpsHX`OhH#^G)_|
z(f)T_|E&5y{%4qXZ2!iT-_0-M=Y2eX)GhYP{%@CG?+DMDuYV=#&*lFN%}f6?JUIX=
z?^Bsh|7Y;MHMMS9__O(cWDWl_sIfn||64<>{#V|W)lWCsfBzN#$bIkZ`d$7P{s==7
zMpm`i`7>{~R7uP8?SHZ`!~UD#_WukQSH@reC9`7v6Xu84|4h+k|98mjKf^Cm?O*@8
z--SP)Q~Jj%{@eE#we4K{%a))1D(fl|esF$-$o1ELOX_z!2wMET9sbGwx66NqlLvmh
z{&!5)`9FhK%KFzWU$XwW@6VY3(?VXh{>I@I^)G#<ey`zO>b0-?tDS$BNw0O~oLrH>
z_{F8+cCKc1uj&u3vHz42FKGX!F#oU8*7aBPAFe-r|A%SwKVR-g>VNxm|1&HJ+xn~Z
zkLZu}0`(`3tzY@e%=h1O?ui?IN>qn`;{M40&;4)tzeDEozf4#E`dz=PZt9<=8t4BE
z6aO>B{%6?Gd;DK-)~`6$AFFO}sSiH>s8_@`{zdXdSJ7YUh9Cc&(O7IL!aOl_g{Hxx
zE<>;P8)FW-O8(sT{mX(6b~DdEFsfHPs~hhaqxHS*kN=7a>my=T+a}l6E`3#%bJBL}
z-8pAPkMJ#-nv&pljpd1e8(-CvX1A`H8YfLv7c8vUnk)JH=zj(l_x}t{7ta2U>W*)<
z7tvc7-*Ii*n_F?~%nr>{?i88k7TBOS(b-NUx7001Gk*4$Q+KC0?XY#1`MUn`_k(o?
zSM6kf9QMBc#n$Pu&oz_m6YJOLEwE16I!}$K&-gey?|%lycLn@kRsS=vy#KrQZGG<k
zo7@NQ3oQDSwC~E=J-t!)lt10Nf23yTWm%``L8s>k^|db){aG%Wvhh&JucNH*yzA9!
zuQ|{3+7qiaU+7PN$yv!|`iA{^7R&osYbJhi^et}mTVh`kz+%q)G(Yj@^+*24>woIm
z8UJV4@b>$^DW37$g==rx-?cxrU+|yxe}<EVAD91a&)@%Lk=KuTCjWTt!|MJ%5dZy8
z`;XzD^E>JvbnpM<yC?lWL!!j~F9GYfi#!QGGXI0?{hw<0O#U-$=!yTO6Mj2+X?2bD
ze+H#rb@Nx-+kUkE=hYLx>OaG!fIwH#xAWOVKxI4>i()Ak>lTMUJ^x46@jruF^M~*M
zjtl;0SQ2jfrr<xrpB~Msd1;0Jo(CgJOM5dejq@VRvlwpuDF4VD@x%Y|T<s(G<gVy#
znOp3<yYJJncXv;nWI0i;&7U`uz4^e`^)K`Pai;%gU=_Vs|6o!5<~3Gt^7E4)^K!VS
zK8n>c`?+SFNA9KtuM%fyrr0gKKB>3=nbzLL-?;v6tMUKMQg<=Rzi*G#dD}ge+pc~&
zHf2rJ-pQh$74n^rJxSs_9&jVU!`}W#q)p+K$_0CWy!{t%eC<@Rc<bExhyF7#|61F*
z<3Gde%g4PQ)~Dz9|C9W?{2y<Ioz;ilPxmwI6t)T<_)|H-+`Y?z^R|+K#_4&I9_yak
zy{&W9Dr%XdzVnw#%2H8{H@cvJWf!bJrS+%qKSLs~_J0QNh3j{$et7?fMfbnJ@W=hX
zIWpsa#qw53y8nKw|55+U<1_p3roTA<Ty@{)T3-?Kd-*FD?a!Nb=Fc(fKZkezs*n1&
z|DSCAN&5@`8D1aa|0}ljPeZxA|Lgec{}~Ri|Ec~<{%?ije}+ZZ_Hp;^e|r7=7rV`^
z+gE-v%l^FiA79>ohAE{t{xjTonE0PzQQqn=e*3NMA3S^i(>=ETZ=aa_FSS?uoK^ob
z)Ghs|QY)7F@2$P_&tHqWw7=c>VsH4O)&BUuYN>xmC%@xgS}&{q<3Gds<oFw#Y7c%{
zf4<`JznCiT{|vcyyIadve){XW{;d5EtK@$^{0H}c3()z`@a68x@RzA|ng1CMUY7rq
zP{;e9VN+xJe}-jmz4nDG|7WOQ`cI{L=l0)A{_-C*ywt_+rTuOD#-H2ccii|<`kz6S
z+YwZ#?Ely6^>6>8-&6lI{IUAaFsXIHe})@w_y04vEv=8gSbOTv=f3^j-v91_DvtQJ
zdi^V<B7yO2wFN=;cb9^T&w0J^C%(TG{?DM!{CNFuiBkT*nv?dg6TZ0mr`+E~^-mtu
zl>TQ(=3DrmVX0C0-4FI3wSMWJov(gWp4WcnF;L;9`AP83-z)zanj`F=T-c{nf8){p
zU#{8lE8-8vPv8F`oBhu$@gx4fWupHxc&yF-%Kbz8W1hl)hKarL!Cz`-|2-$yEdOr)
z(t3sY0{<DF)L*ZEqg?o(VR6*{_1k|he;I#5{fPZf`LFkXbKI$aac%w52k~9&7yb2@
z?=ov&{g<c8{`{pb?Jpkhiywc#@YnA@LlgghhJ*V5`2OAb&#-xY&mQNG@`tbAI<|Jn
zkKQe>GZ$Q4mvhMZTXgxsh`IHhx7XHQ*#9B#`<qQ6^+$!*)IW5tJ^s&Sy)33rGQVek
z^7(9Z^ss!8;EGHBJHOwI<7@TNQMFjU@Wb~%&bxo>)L)pq=0C$j*V_LK$2H_X<h_5h
zx&Nn7Y5yOM_4|K3tv%{mS%2%s)vb}W&lmok-TKe<yFAb7{|szz|D6s#s8au-^>yIS
zhKAFRq%Vd1dT_$6Zb8>i`)@b?Gc+gZezE7ipIy)V$9GkHme|Ih^FQ=uzZW)rA!(hk
zvo|obfp=?a5T~+{no)h~sx4=G-o0$Hns_w7{rtu&%a*>C{*+c))T8V1>61#m+^=PG
zOBZKN7c0N^`R4toQx{^Q&$X&Hxmu*8+O!36D172rVLa=%@;~m|9}fHdNd3>iYX9+5
zX29Xx{B=v0ZBz-ExK(Z5O`bRI>6Ol>>R+#`5{#E_J}zrrtNry4pJMnzuKx^YCjVzJ
z{=COk-D}OtvIIWc)!*{}@rM1lZu`8YPOHXx$JAN({r7D7$l4`5XXe$l@Ax;!`EHk6
zGVkk;lkv;$f5^65tP;QNexJ@Y_KZ9mVUzXd)=4S*H*vLZi?EjYb|d@5qbje;T$f*w
z|1{p8%qn`U_<Z5-y#lwo-}?JaIr@95!pr#nulDYrxtzbZ>MH&|)Sx{3-N}D1|AqeO
z|9JnOcYm+_8|&Kt3`>Q=-yXSN=KtOQ)A}Rzeg7Fg#nfc}XGpyL{;$TQed@jTjs92X
z@2_C}_n)EbKLby_@jLw=O7H(n&X=mc;imsrO#AJ|OZ!0WSiS!YC(S-C|J#!EpW$NC
zzT2|ew;$gBDSr9O-^icOJMVuAN&3%lWdD~bB7v^e>4#WF0v+JEF6KYO!7K8gGVBp8
z>#JC55J=0KD{lGBIrTfQe6i<t_tKEncpJY(es=u<-3%$&=bc+m+Zy|CpY=KLihAF*
zYxhL-RW5nl4h?nJIl6&CW|G6I&xcjFTyj^*mzlWH-R<|O!%h5`_9}lfeEjY35C5%Q
z{~3-ZFPT(6_4uz6mU))vPucGHb^S&A9~t%k3@l6kGqh%|kNs2mfo<061=+&-e^htX
z820Wey}W_xWB7uuu!K!~-OBGIKl`267oHWou(M5S*__wz&&mT2oLedx+JCdY@uwGG
z#<AMtOBtTupLk_b<;yvr=dU->JaPW<{HU+;4+>spFMjv_hm!aoQN8>d@sBk=%<tA;
z@}HqK{IT}bz8@#N^;%2ZuI`Qh#(!#3;~&N7NwcGWDP3NlxNk#SMQUG0TENE>J(4NG
zClvZPl;>Hjf4qOC<#Wqdbst{1UuNl%+iANoe)Ief`TBoEgZ~N5wLAZS{|K+2`;|41
zv>&{6dirx`YskjScQ;S^xFW6ns_0}(m-`=TA}8MaaY{S%V^z7`>h{mx(Y%4b>L>pC
zIqOF`q{$!p1K#A%JF1t+b344}>-vuU4<70NbdB$`f1|jx{$+{j$4Y3G^~>k{-)nU}
z|B9mDn_s!;3~K*tfBTtp{!H@zGuHE$e_j9jKf}Z8e<uIs|F=isKf}fB_!I2KU;b_V
zTMsVA>lglKxZWcFOE>$$j@SQUYyLAhzqW|3ny;LOQ9bQ<d5x_idtwy-{Z;%U_ls%q
z?=F9_pTA0D(f-_i=g)E-!GG$XZ(RQ-((b{p@Q;`3`~Q41{oAd6N8ant_dffl$Csct
z_?Ou~Iq*mNKSL_hd3^2tB71Oqe>X~dKf9ju=k{n&bN`&3;*0oi0^99>ifaG**V}e5
z{&?NYm3K0J_3pj9nd6=AvdD)?=ByLzUJLkf$hCyuI{BZ0o#)?`i|ej@+}5}HM*Sh{
zqyl@1>?c3wtFG^xqv~`kzTR5ynfuJ3Lz-H9b3j8;Yo(8`t+$wuWt`%llFZ*#_AkmW
z{%80w7d%q;>He!a`#p8v|GDq|w<*!){EPP$Uza3LUH@DhLv09bnD>_bKVi^lM(e>J
zzi;3Deg7x>uahmG?H8xzu&sYm|E=>s!{o+~*Z=ldwf~!PGCouuJff*QUpnLY{y*K*
zY<JhMe*gMtP4c6B!Mi^WADPv*c7=!PzN>mEyLS}qcJ@>jPi{Q$WFBj+JI|9Db6A#Z
zeY<wM<~+aEnU_gZLhV|mG#2gK@j-d#@0I@<*f!LkTv+4spJC(C{a?DS?GM=Bv|qpe
z!8QJ$Ci)%ozgcYmH!<q^EAbEEkM1~vdguF=zxZkXdyXxe{at&ne~xyF{~1pFXIKYn
zC_Mhp;I_3sKL5M_rTr89JL*5#zux~%aL0d!i)-V(59}9>cl|ZbUL==4{FlU+K!?YF
zzOKKRytjXX`@7|Tqz)f{)BchF-*K`3441T{{yMoyyPN*m`tQX355f0);y2uHGS%n*
zWATx-Z{pHd)6cHB<7_uQx9pPt(^Whs9Y=VSz9{X@`Ok3haQ#*X`%g9*_kZ}^`1qe;
z!s^d+uD5R7`muODcjUhR3?KN{o!)%LR7!BE^^1D<=;LdpuU`qrHN3SSWmqq+96r2t
z``@NStC)X%JzdsaPyQLz#C<q_puc_B_O3tiYa(7q@pUuK`}FITzizVQMK`;&G8?UR
zkuw?=Ok%CKjcWfRBmX-z?v_pV<G*%SXa9>j{P3>*8UOFjPT`*=*U#CX!GG)hqw=?*
ztBzKF6h1De^fB4NXTt5fcjj-7-YB7_;V5u)NA5{icJ24E5pmo{E!XF&<}1(JFB};7
z_fG8D`#&`F-+J8NR{pm2@%m0C&b4d4zP`Inz>Dc5?;JrLv4;&0Ec;B-n19T;-n-$j
z+@G%^b8RgiD<ss<|9sp_`%9Kcve%qU=Zqy2KYrEnDlS#M+VK3(e}?&i@oVBgWG!!4
zvg^m*kNHP)C8Kk--8Nln7N{%RG0W4>uIc*x@6)gCSNC{)*@9tyOi$t4>z~%YIQ@@<
z{kKL;-I@Cz{Qt`RX9(Le>2}<Y&XRv>yE?CbpKwq~DO%CCvQ^A?`MWJ|ti#V2mT3kZ
zu&PZ>{q?VX?S#NmLoe?)uWFC~v5%^F=^Hq8Ib&n|B=<9aIW+n2iEL?q&$VE<*2(K{
zDc|{Xd-<Iz_cuxQj}7dE|1;cpxTOANN$B60iq(g{*%^I!`fd8Eh}*BC;#TH-+g>(Z
zu)Xb-deE9a4$Gh=O@=4b_D`9AWA{IQ?uYh&`>gE$O?QpImQria7(b_Q@6{^b{PZ6Y
zzx5=qi3;C(@h{^)!-MtlAAH_A|Koi8@$YoswnvX2Ztq$$>G9$1>&m4AC!J5%E;Ty7
z+tS8rYToXTpEPT>$3=X6|F7Qa-}?PI_iw5ny1(uGo4@O8>_4XNoOC?2yLRs-wU=K`
zFJ(Wuvt?di%jAQSJg4fy6#g?9%5QCdv;EQf8}}by+xy}B(Q8e-(eCqeiWRi^7BdSz
zjd8j0bMZts$H$qhei8z<_ax6<)0@6=<;V5M&wMIgv+?08s|i2iA5<PI(z@2ZHm)K&
zdh?Ynzq?mkzqKmaxosZ%O_iX<Y~f)i4hF4q1l8iJ7QgFXR@{?*U+zoz6)*FcO*R&%
z&RYC^{-yIj154q5hNd}@{}~QC?qmKtJ!;3+?LX!|v{=0T`r0W!l>_{DP5-FyVpF|D
z;P0JA`%Bgp|7Upc@Z<J>oWYM9d-uQm6)!jc<%R1H=D(TyJLt#iZ^l2~FRF3)P<s95
zS6?}Yw8rjAs}`0=-jxo&FYfx_a9Z=Q9>*Q~bo&RZ{xfWy-&&tr|3k5D=i~PrXLlVB
z+jHsKvukeg_P#s4U$AVu)OhjY#y6)<9o9PK|J>rt#0Q_PLN(VH1;y)@*&XMdxyFrc
ziI#n(`sWLOq`^afJax%mA9ejzing9Fdxf+{K*dmQ1mxJ8O#jI$!Z=Ig8UGoLMO~K7
zGrw_ZEb3k+zTaU{Q{m6FzA07jY^z#D0v+6ozX$*0P593++382;e}?4l`j?_>>lW6t
z*2Vp2IJib0b%5@{`lCCV|GAmg`d|E%|J|aB2m7GgzkT<y^)dHNZ>-bW*ZtL|PUDu`
z`Y(nr7Ipm<oG&L55Y7C;sBYdC!7f{S!PnUf7WFNEeN?2<Fz%tNNMOS4gat))TVKXA
z>`&g`6~9&W)$tGi84lQ~e+;QBpYT1b%t+cIe%(zyz8k?S?5+FueC+yb@VB}C!OdE)
zT))Fd=F5GJP~6(F=KhDS7uThi&20?c_Ux^J0RIe;<}&9(t;g)Aj;BlbsKjrbws+T`
zb9dBY;{~oOmA$LE{-1#*{69le@GJZ6{>P-_#VZ#3SJVmph<ci`Z(50>>CXx6VxE_a
z<bP(JzQ}fud!_uv*w!UwWrl}&vUv<wxVQ$a>at*9Jn(pHQU&|NYuz9A9}1IVZ!ei^
z?|QLhhU?a;y>pnPt1}Kf3>0BlJg>m|x+CWq@A+qX()MeeGtPYc`nPAsv=hJo?AGrS
zo3C|$NBL2Ul0{vg{2gFrJ40XoZuQ^Ge}+G@2an8w#?jsq9+_iB9hut?8JPq1$RzfE
ziCDkwxYv)#|2TI4XHdC)Z2mXF?*9y~UB7cm*5|j^KZ*Ch{5AdgJoWmMf!+TZgy2JS
z93s$RIS@_x&|I|q?2_O8UVppyKl!i^RI)!_{GS0-w5MtQwCDQg^PeH<{ZAYHmj4Xj
zEMEL)_!TQ!AGz{7`=$MskNY17eW_pgEk^&tm9I{M&9i>&|HrNSpMmAj=Kl;$Eq{2-
zrF6uu{bvx`qs_A^qdZ8|b#G74$y9AQGtd61vs;Sz@9NLrvPzTfxoutihxI*&9;G|2
zTJ2v_tESYF?RcnK{`j`ti|+l%j^2CutG4%QUU&Z;)!DnVXY>?D`}Bl9p6IXj>OdR&
zoaT91YwOO}A3PlY!K?m<r%t||eU05^@2qKt&5wEQ?&<vep!A>Nglnj~PS*l2!&4gd
z)3c}jNj%0by=`90SNp$Je`jl+dHFa)Ys39jZ|ZXU4xZ)?JrrH?h5z$;ueUamAHN^D
z$MM7Vu{^`g`{sX?=2@BbuDr4Tto+PeBVUn^&)<3Mj^zJ7&F!4ur9H8tFT4cj&JOon
z{@Gn6H8*+LwBH9kwK!iLR_oGORnNUcP4@idU)R6>XZY~?pXa~!{|s>g^)I%rf5MDy
zfbT<0>3;^jef0~!#^~<#pST%w+GA1a&zt`l*v;!tS-q)$Q?TJb!$tGZPm=n-zv_Qf
zez95qd*(0s^H)Uz<F_4O__N<d^Pm0a8`r<-%rB_b{_zrNRIjf3xcsNbm*RhCe!Zi;
z)qnacU91Ck`@OY4`~Tyc@t<Kb^N0MuQ@HK_nrDT6+A070ul`5nyJz>`Rew>(**7J?
zDlmTQ{+2(t|1lOHzUB0z{fL-VwbkKouUoboRXX|W7ase&Cr{bppt{}N?605ff0*2z
z%lMz6_2YKi{SP_M*?+3ruM8UMbh=RgMmhgK!{VU*>tF19T9YoGc4LjMS2n+0^zz$F
z`xia@BP!#Q`Zn}!{*gaL{~0=bqyKeZ{}(C$_rCidr>y&5;`V<!@t@({ufP3^_BsDe
zJM*96`Rm`)f6BZ5ZEyY;f8<XAiXM;p2ba_2KZ7*M|Ct}ZNu>XKX8oF()1%hEi+>`2
zqr(2Fy>#?HUh9h6{r%Pr&;Pm1sdIn(Tz=yEllmQXLjM_Bn2-N@*8Hu1o2+=ijt#~?
z<HF}9O`Ndr&Nk!i7AIfFU%&a{>Yx6)i}s%i?5zGXY-|ts&v0p`_Uj$~CP$N=2PUbW
z%`W?FS8jLU;^AX@leWGq+!E+oyYchH0ULI-zUW%lv3=(!j^FeDaVGy~V2RHEp_zV&
zU$A0rol5;7kz0J5|4#lE{P6u-MQsk{DW`Y8H58nv5ySuP`W4Op3~aN_|B5f%z5h?3
z{NMX=GqH5%F2B3}bmy_!zQw&O{pNbB{OR9zx@e7Qj83}6tMf<K%lv0JxN_IF_6L8%
z-n;&|$hLX)?OP{J`{tXs{5viB=ETxTYE$m*=rLwky0L*@;p_U7we?Q>J}lc_yX;lj
zcB{ID%g(EAy}R|)lv%eTZY*BBx@gTB4u!`o$;?aZ6a5zJ&YSBmoBdJvAD`QQhNjZ`
zL%Tn|-G02E>*m{we*z0-r*|cF)_GfhU}$^Jyy(n=)xr(y8a}0&zAU&e7`D3O*EYTI
zXE*mf*DIWKdAa|s3(@&6>VgHE`yM>_9l&qWRp<Jve*UM*rElXG<gT{!{%Y@N|KR?A
zh94U3e~hNTmHf}Z_4&y4wfqhLRASt7)+#^JH4aT$nO|V_HqxT=JoAj19T6wjzi_&H
z#r=D8)%B0p{A?o&=kQ;DRcquf`Bn6<c>O`{{|pZnseilmcZTwXKkhZ#<pu5JRiesP
zrPf}~D*K^y@3Z<eodrwUf4?~Lz+>yr`x2>U)30ad{8N5oQEvFXZf)S}3%?i01W#}D
z37#VI)Kc_=$FGAKi@Fj-+TU|UIe%|ua5uj@`S0aFt3S#=-v4L1yj1;-vfuw1yj0fT
z?%elg{=4~4<2(K{$o^+|vg60)e}}i<{}rfO<1bauG=H^y-3RW!{~1L8N&HK{6OTF7
zRO9)dfhG1o!=&8D{=Yen{%2Tpw04_S`1Zs5Kj|-j`D^p%^TPX|tT_6gp|k#_hQ^|<
zZ|7UUi@87xx<L5P&i@R}E9{?KsFVB8aO08vm#$~~16O}~{*Q0Le}>79KdS#TBzJ&%
z>9f9B)c<o6+3Rm?@t>i1<%>EoNRhNfxBu*}+T%;>j^6)N5zqOb;Z5Pi{|sMtY5m(A
z44T?di{DuOXP)?>`rk4o^?&uc_OH1DnjxO^@%YD;RrVKt>xuu!s#=)2BzVR8R=X9;
zSKK(h;+AiIA=jPg^OBxPC47@sN@zJsC^R|BKaM~5pW%<#`+uCt_HXn*_Wp@|Xlv*H
zBec$Fmwi&ir(l=emT4O&uYY$g@{y(W1S0{S%K8_ZcG%yNe!$;WpZUH;>OVtcP2zH1
z@5B3)w`OOrIM1`EX<a*e_F{%t)kZ6y?d_RtwsQI-QP&T9FFsF;;CQq$`mJvNlm0wD
z%P-IOtJFJR?y{2l)_?o)H`R~kZ#aLuzPiWuQEBGF{=eEDSNH7Qn03v6hkL<-boQrQ
zFFduCcASh6e)J@{cWL3VOGV+gFZazUjNkb1qrh7KTbIgx{ybVM88myj?S(*B(Iwl`
zvJ2kT&A+yP{eOn`_)q>{?0<b|{?FjLwq5~y&DQjI{m;_Rpm~|xS2ew}{=E6m&^&wp
zQ`s{6Zx#{tFJ`a$E4=>P`5%_M{{>!8{?Cxy+WxO|*ZK)3>+i3u@053)S-;!=!oRu|
zi@N@9v*kZirRa}5cYoXysY3gF&*DGBbh%0Y8E*Tpdl<W_W`AHdmfGr9Wc;l94<*O{
z%#d%cf3qO+Kf}x2LF=cU_|I_vWqqgIyWD>-?VUgPlq_nz)b)4mo4VZV@67CE>rbLp
zM)eBzXmk5t;*0j%JYN6k(tn2S`<-7~gg=}Qo5qg;)g_to?=G;UFAo1?OYAH@{~xb?
z-JjD>esn+V{|r<g)OFRgl-rluV;KZ{`u>m9!T$_utsjLEV^#mS{zP}!KS>V1@>gE%
zKSOc*g5y}`n_j>D_wrZxCr*^9{g)Nuk5<3fAAI~#@4{FA8Jy+1G}d3_pV0RlG$w{_
zkxSgvpXJz#c(lPe#$&F9^JVI|U(|2WKQy=gpq<9XSm(q4)HY2z_Dzks?3x5mOY_#N
z8(!V8NM&zgEt_)b>Jv?ieHriiy#j;!b+*}0et$dso8R^QN%gnlkId)$<1qW##dr1>
z@_ka1udJ1S*7VT+zViu|w*{xKdaTzv_x?XaT=p-%{|wCy_7CPUf7|ePj>^aPZEwW#
z<!j8hPx=^U`uExDWerh}Zufbs*<5*lc)>I0`1768e&%A9%<8L!k3TD@suTTf<~&QM
zOnT;@>_=biEzWMx=Z-s{zKZAimbwG-+uSz(=KOa>PX6P!X_x&oGyXHAUeigo|0Hzz
ziSk>4i4&HzGG4A!@K|x<)Po;;W!dHRE#;2O*6p>rzA~;yJGkWenz-k4{g$je8`AbV
z`PcU!@gKEae(eA7+U28rd;Y>JTWssjX}^|U(SO<`NF&wmI7^D0;|U9;T`ZPy+8?X`
z@oxCf(9~I<l4U-B^M8h}^-Z^9D`&Mgr_9|Ku|##&qodUf_f1w9v4sT~#pu?ZKlJ#t
z%=Je*m&MzNsfDP2s=fFtFTT4^scXgd*RSlljAv<|w|1KF_o25(-G>Dn&wg*J-&H@g
zPWeyye}+kqAN&92yI=p(W$N!jT-8J8WB&)iU(?^;W31o*X#ZjPGrzmBb!s~8F}pwC
zew6=n{qom7tgiA=`Jbs0`#<XcRb^#gDiXe}Mp0(fnYw^A^|xfWw7fK;xDL9C1Q>u8
zt*<{l|A$rcKVSBT@Bj8G|7Y-6m;F_0zux`_FaI;l{^t0jf<69+&y+f~W6$MJT;F*3
zKf@=(rLQV0&;Oq5@BCo{BzrH6_;Y(^bRE_jA>gU_%lMPi-$a<#-Kgg>JpU)m%Hlu6
z$rr1CooRWdsyxBXwXXQM{IR7k{xdkg`4RtM#h>GWuA(27Uu89&Z1PHj`RV;1TJ3*?
zS9|_vXzHpvKV6FZ?`;2%ZEbSj4@+%cZ&t37_^61jrn704h2#^r-=4k2e=gRz{HR=8
zqx$jM+K2N6Gq+3G?OYh^nOd7(usdb@ROxKZH<j-!ELjR%>*I@<-fo{$Yhn7c|33rk
zpZ^R^A@gtPA2gSjv|(O0{p0sG-QqPjqt#MAvbH3@Z>alvL~Fmk)5mwQM^<tD<`X|J
zC%63SYWpI8f3=!>wNqDZD1BMuU2puMr$<VInSu2M12c1j?cNpZujnmZbmK>D=sN5r
z`x6G9cmEkuQhpbMmmn;j-;P*<z}{E3wBG*0uXExTuk6RuaWON<|Dhh+)jaJ#L(8GV
zjsF>9^j1Iql>hxz+=^c^m;YX?YybS!mElQX{PvtTbt&6?xuEknia*&e?eCj>%^%dA
z(H8%;^;h`wJ;nKV^!9!KcdNFX>uvDUUwz<}2nqigjxqi)|96UY{=eB<wf^dy1@~+&
zo#B7O`QYQ9_zxR${xf{Jy)yjNllu2x_8;l5HoeXXs{s1G{!X84ZxZAii?NdHOEPEt
z3I4<De`;9C|F&w~X?K6G*T4OXEgif5yzjHK@?XFA=i=8tBCEIBIj>mM^~3lz)5M)3
zTjqK$;5g5Ptsn5~-i$|P>z}lvPq368xPJsx4nO&^_+e|GS(k>)^%pPg+K;b2f7c&l
z)mDpT)Sum;sr~*xN#OMe5%o`|uB}_RT>19eI^84kpA1(0Ie*Eh)?h-Mb3C@eqW$Za
zCvM%Z_0O>mI(t%o!|B0)h9_?8mpq8?TlJrT|5e4EfWNoOKL6#Z(pXR&zi7!Xd4=s*
zsw64-X`sw_P5D2A1f))q|EZcCzhdH?1xLY7F+@0J<r+i3MSF^`Xr1Su^{{Na`Z}w^
z>wo?;^wwL|e~9(}p`QHj$bW_{Pwpvx_}hQ;pInX0r0BcGC#91vz1xx9Z6w|ra86R>
zVa>K%_E)#oyWM?#_&-Dbe})^E-T&^kPwN+~vA^0<_C0)s-j=zJmkTewf6Q{w;J5|L
z!}gMi$BX;Z70=4C3oZL6zW?XEwCu)+S-Pds^%MWB-f-BA?c(HPhW$B#{ogi5&llb&
z^*WxdPT_T&=tsHvYE#!$l)ZhUzcz(MBGpaHL4DPn6mvHg!3o(sALBP2^75W%Z5#OW
z$fHL;<NeQE`jFo9)xSN?;L;z*h4=1c?mE|9F8jCl3GcS(%!^YeUVjlFBgZehIsWqV
zKf>1E9$wm?KmVq;^T)O8Z>}v}@p1X<Up2n-H^d(k@i4re+_Xv2{?OqI*FQauPx#Mp
zQ0za$gN44o+5TNMTfHvMQ~vOIX^AiE1>KH+FPoX9ut(uP!+Eu|yZt}YJ|&2mey;JI
z@;96BmGh&%)t6V<_VgS4+MCK*V|3x))a0B+mmAMIryJdgI<91NVjZ*g$`c7y0$Cw1
z<0pOHZ0~eGFq1#f+gG{pPX^=UIokgjc4b5R2a^98;2nhMMJzg=FP2<?6o0%``{U_j
z4SnIU`_BJk@r>s`w*I@w-^kzJ8={go`u=CosW*NnkGAMwtEK2i`9FfC8$W=C!Gs{g
zV1;h1)qAY--~1|j`)mHZA9MdRsJiL@@&3=?>F5fo9wY)?MHpZ>e8#`a@|)}r-@o;r
z;m|cJ`6K;2w|^}6yqwSab?u(Z-U4m8J&gx@Cwy3U_k7fsS6T}#oxBRlvNsxcO{q-X
z_$li@!$JQ445@SKPbywK{&)ZV`j4;dA0GeDu<86?AzAtV49xHM|M*&)>RPhqu*m+W
zZ}xpF{_EHNNq@)B{6qIYxXr)0{J=e(OJC2bKlGcPzUoDJVf>9dx?f+{&a0SX<-s_k
zr!An8ZQr6!FE_@w?`M^Ywt8L7io7~`>h9gMgO_sDfGW1Iv_@Bv0LD-~rA&G2@*IT*
z_fPiApWnE;ON068e8KvI;``I$B`TUehh6z6^up+s)YK!>%l$=<-r@Lk;^>@?O9?Bu
zg`8F}l!*F3eKPSsgGsa7e};pt-~ThntzWZZ-TL)OVOLeZZ=U(!h3$iVkDo06&(I|I
z@3i%%I)Rtc`&;+juex2=aK^PZddbU=FRfC<HXMI?IIqd#vdY^412#W@rZs_)*N%j)
z{u60gwR=|ZVn5y;D(i2uR=$5L`rEKV`rF12I&~61S|8r77tSz@UHa+fd6kI^W{OUY
zH{7(%q3r#=y<A<Vj0$TmmeskQ((*d<)3p9U`~6Ml?$rsr_|MS$_xpba`GxzzLiSH4
zWZaM64vK?cr@CJMyKuDr{K?ukhhLSLJ&7;;<g@>S^M8h$xwG^CXvr#0C}I2eqd%^#
zrmk(@=IwW%tb4cb(0teAXu+_xb0aOIckf(lW%BYr1D|G<!uz=F-(Gd6?lJ#ew?{hr
z;r)(#(%V1Ubtc!$n!9(+rPET!izc3wI>D3Eu9!NZ^m+Z(s!-86sTwD}^v;TEtkgDs
za`it$lYHH2MfX2Zvhw}E>i7L;czi|vhvI*RH=F1GRI##uSPxnrwfeP|_UhlxqW>A}
zRljeZ|H{OEV(syhx9a~0>VJ#BY|5dh_``jP%C${vF6`L1SIsPV!kyE7{K>O^ZMz}J
z^d#Z&@}-_&=da%DY5wY6u3xW(`ALzz0)eigoeP5GG&2@($QYhy&ySn>>D$)-3@p)q
zXV&<xxtA5yeyCRI(SL@3x$pAQ%(t&fop@ujAyZC~r%l!Mt0G}jvac@r&rqr^@1ATm
zCDFcD{Z-BE{|tQgf2Y;Im;l;9ArEr;3i}Vv?{B6w&u_7^{Lirc_xpdg@sWYi{~0op
z*5+sG@1612ul4<jeUH!7r~PNRDgN;No92h!cljTlZ+*K*f5|KJ`n(<6n<CR9)ozGa
z8d=M{4V9@1U#j)S?{C$WDd8s1HTPv&Z(bxA(63XbeX3>J&**0l9{2n+{dD#FA3^hP
z@mJob?cDLhbMeW-LS=8|zuUf+Zx-@AFk5Ewt$k}3t?RAYvcFYb-aSbxV7YbilE40w
z>oGDY`#+A_-#QbDKeW5sfBVk>@)9UE^561=u6?WeeY5_`3wwWx->K|>bN{!}AKSkZ
zYBZ}C{;~Y9cYEuXxo+>@UKjh|t#exCHq(1G_vE>&7WaF7IdMHHX|8LrU(w`elXqJw
zib{Msp0KD(%ZsZoMCbnVKmQqit@6?|EwHQn@%ldl%ibUSzrAX_c71H`-6#Fh>UpQW
za-Ze<Th$#YJ2)09>nWA+91PKW#GM;p+`w9t{hxuw_V2Vn5&jk{d**-j&?2VvxNK+F
z(_YWW%FXj4ta$GptUZ2m|A)u#Z>IcbII9R+;9>XsKSO^=?tcar`@hq^fBeww@p@kV
z`@%WN|ElZc=ho)B@*5nrwOzCCoBx#+^-8~c^xOBbT&|P&q3!kIUuVv@U2dJ3qP;!*
zCX*`#Tox+(eloVs4Pc9yd^B&pm&UsA)w`{>g*kfm$RC^5#n5=U;-CKPKI`z_*Y`j8
z|7U3Vr*!qbsq}sJo9Am+_<a{X{rt?nl)j@gH%Rb#s`p2I+odfvqpDOWRB7(+xif5z
zi?&RjCbeW??a%1&-+j^#`48xG|6#kFx3)%o@v_PJmFlx?3QxRKNig6)b8L4-cuGRO
zSE=YytHPELg*6jfb6le?mu7pO@-O>ZUlHgk>gEV4AxCil<7gO+h5?ajfc^hX0FBLa
AYybcN
diff --git a/medcoder/publish.py b/medcoder/publish.py
deleted file mode 100644
index f39a231..0000000
--- a/medcoder/publish.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import os
-import pandas as pd
-import argparse
-
-def main(config):
- #Load Output Concepts CSV File
- if config["concepts"].endswith(".csv"):
- df = pd.read_csv(config["concepts"], dtype=str)
- else:
- raise Exception("Concepts file must be '.csv' filetype")
-
- for name, concept in df.groupby("CONCEPT_SET"):
- concept = concept.sort_values(by="CONCEPT") #sort rows
- concept = concept.dropna(how='all', axis=1) #remove empty cols
- concept = concept.reindex(sorted(concept.columns), axis=1) #sort cols alphabetically
-
- concept.to_csv(os.path.join(config["output"], str(name)+".csv"), #save to csv
- index=False )
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description="Script divides single CSV file into one CSV per concept",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
- parser.add_argument("concepts", help="Output Concepts CSV file")
- parser.add_argument("output", help="Output Folder")
- # parser.add_argument("version", help="Version of output")
-
- args = parser.parse_args()
- config = vars(args)
- main(config)
diff --git a/medcoder/report.py b/medcoder/report.py
deleted file mode 100644
index d4069c3..0000000
--- a/medcoder/report.py
+++ /dev/null
@@ -1,175 +0,0 @@
-import json
-import os
-import pandas as pd
-import numpy as np
-import argparse
-
-#Get all Files in JSON
-def get_json_files(folders):
- out = []
- for folder in folders:
- if "files" in folder:
- for file in folder["files"]:
- file_path = folder["folder"]+"/"+file["file"]
- if "concept_set" in file:
- for concept in file["concept_set"]:
- out.append({"json_concept":concept, "filepath":file_path, "json_code_types":list(file["columns"].keys())})
- elif "concept_set_categories" in file:
- for code, concept in file["concept_set_categories"].items():
- out.append({"json_concept":concept[0], "filepath":file_path, "json_code_types":list(file["columns"].keys())})
- else:
- out.append({"json_concept":None, "filepath":file_path})
-
- out = pd.DataFrame(out)
- out["filepath"] = out["filepath"].astype(str)
- return out
-
-
-#Get all Files Excel Summary
-def get_excel_files(out2):
- out2 = out2[["CONCEPT NAME ", "CODING LIST", "AGREED", "FUNCTION"]].loc[1:] #select relevant columns
-
- #Filter Concepts in use
- out2 = out2[out2["AGREED"] == "USE"] #remove deprecated concepts
- out2 = out2[out2["FUNCTION"] == "QUERY BY CODING LIST"] #remove deprecated concepts
- out2 = out2.drop(['AGREED', 'FUNCTION'], axis=1)
-
- #Get filepaths
- out2["CODING LIST"] = out2["CODING LIST"].str.split(",") #split by ,
- out2 = out2.explode("CODING LIST") #one row per file
- out2["CODING LIST"] = out2["CODING LIST"].str.strip()
- out2["CODING LIST"] = out2["CODING LIST"].str.replace("https://git.soton.ac.uk/meld/meldb-external/phenotype/-/tree/main/", "")
- out2["CODING LIST"] = out2["CODING LIST"].str.replace("%20", " ")
-
- out2 = out2.rename(columns={"CONCEPT NAME ":"excel_concept", "CODING LIST":"filepath"})
- return out2
-
-
-#Get all Files in /codes
-def get_code_files(path_codes):
- all_files = []
- for root, dirs, files in os.walk(path_codes, topdown=False):
- for name in files:
- if ".ipynb_checkpoint" not in root: #exclude notebook checkpoints
- if name.endswith(".csv") or name.endswith(".xlsx") or name.endswith(".dta"): #exclude non-data files
- all_files.append(os.path.join(root, name))
- all_files = pd.DataFrame(all_files)
- all_files = all_files.rename(columns={0:"filepath"})
- all_files["filepath"] = all_files["filepath"].astype(str)
- return all_files
-
-
-
-def test_concept_def(config, report, folders, summary):
- report.write("## Check Concept Definitions")
- out = get_json_files(folders)
- out2 = get_excel_files(summary)
- all_files = get_code_files(config["codes"])
-
- #Merge all into single table
- outs = pd.merge(all_files, out, how="outer", on="filepath")
- outs = pd.merge(outs, out2, how="outer", on="filepath")
-
- report.write("\n\nCode source files:\n")
- report.write("- {} total files\n\n".format(len(all_files)))
-
- report.write("\n\nJSON concepts:\n")
- report.write("- {} unique concepts\n".format(len(out["filepath"].unique())))
- missing = outs[outs["json_concept"].isna() & outs["excel_concept"].notna()]
- if len(missing) > 0:
- report.write("- Missing from JSON\n")
- for id, row in missing.iterrows():
- report.write("\t - ❌ {} {}\n\n".format(row["filepath"], row["excel_concept"]))
-
- report.write("\n\nEXCEL concepts:\n")
- report.write("- {} unique concepts\n".format(len(out2["filepath"].unique())))
- missing = outs[outs["json_concept"].notna() & outs["excel_concept"].isna()]
- if len(missing) > 0:
- report.write("- Missing from EXCEL\n")
- for id, row in missing.iterrows():
- report.write("\t - ❌ {} {} {}\n\n".format(row["filepath"], row["json_concept"], row["json_code_types"]))
-
-def get_output_files(version):
- output_files = [f"output/{version}_MELD_concepts_readv2.csv",
- f"output/{version}_MELD_snomed_no_translate.csv",
- f"output/{version}_MELD_icd10_no_translate.csv",
- # f"output/{version}_MELD_med_no_translate.csv",
- f"output/{version}_MELD_atc_no_translate.csv"
- ]
- error_file = f"output/{version}_MELD_errors.csv"
- return output_files, error_file
-
-def test_concept_changes(config, report):
- version_1 = config["previous"]
- version_2 = config["version"]
- output1, err1 = get_output_files(version_1)
- output2, err2 = get_output_files(version_2)
-
- report.write(f"\n\n## Compare Concepts {version_1} to {version_2}\n\n")
-
- for out1, out2 in zip(output1, output2):
- report.write(f"`{out1}` to `{out2}`\n")
-
- df1 = pd.read_csv(out1)
- df1 = df1[["CONCEPT","CONCEPT_SET"]].groupby("CONCEPT_SET").count()
- df2 = pd.read_csv(out2)
- df2 = df2[["CONCEPT","CONCEPT_SET"]].groupby("CONCEPT_SET").count()
-
- #Added/Removed Concepts
- report.write("- Removed Concepts {}\n".format(list(set(df1.index) - set(df2.index))))
- report.write("- Added Concepts {}\n".format(list(set(df2.index) - set(df1.index))))
-
- #Changed Concepts
- diff = df2 - df1 #diff in counts
- diff = diff[(~(diff["CONCEPT"] == 0.0)) & diff["CONCEPT"].notna()] #get non-zero counts
- s = "\n"
- for concept, row in diff.iterrows():
- s += "\t - {} {}\n".format(concept, row["CONCEPT"])
- report.write("- Changed Concepts {}\n\n".format(s))
-
-# ✅ ❌
-
-def main(config):
- #Load Report
- if config["report"].endswith(".md"):
- report = open(config["report"], 'a')
- else:
- raise Exception("Unsupported filetype provided for source file")
-
- #Load Mapping File
- if config["map"].endswith(".json"):
- folders = json.load(open(config["map"],'rb'))
- else:
- raise Exception("Unsupported filetype provided for source file")
-
- #Load Excel Summary File
- if config["summary"].endswith(".xlsx"):
- summary = pd.read_excel(config["summary"], sheet_name="CONCEPT_TRACKING", dtype=str)
- else:
- raise Exception("Unsupported filetype provided for summary file")
-
- #Title with version
- report.write("\n\n# Report {} \n\n".format(config["version"]))
-
- #Compare JSON Mapping with Excel Summary
- test_concept_def(config, report, folders, summary)
-
- #Changes in Concept Codes between versions
- test_concept_changes(config, report)
-
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description="Script performs testing and generates report for output files",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
- parser.add_argument("map", help="Concept/Phenotype Assignment File (json)")
- parser.add_argument("summary", help="Summary working excel document")
- parser.add_argument("codes", help="Folder containing all code source files")
- parser.add_argument("report", help="Output Markdown file containing report")
- parser.add_argument("version", help="Version of output")
- parser.add_argument("previous", help="Previous version of output")
-
- args = parser.parse_args()
- config = vars(args)
- main(config)
\ No newline at end of file
diff --git a/process_codes_WP.ipynb b/process_codes_WP.ipynb
deleted file mode 100644
index 68e26be..0000000
--- a/process_codes_WP.ipynb
+++ /dev/null
@@ -1,1288 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8c8f4cdf-04a5-4762-895e-6555781a1f05",
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import numpy as np\n",
- "import json"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c5786d78-7dc2-4f02-ad21-cee95e473823",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Ho generate JSON"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "0292dc90-e31a-4724-8536-d0b55533aaef",
- "metadata": {},
- "outputs": [],
- "source": [
- "#List v4 to json\n",
- "\n",
- "df = pd.read_excel(\"PHEN_code_lists_sources_V4.xlsx\", sheet_name=\"ho\", dtype=str)\n",
- "# df = df.sort_values(by=\"mapped_condition\")\n",
- "\n",
- "def json_file_template(file, cons, types, metadata):\n",
- " concepts = \"\"\n",
- " for concept in cons:\n",
- " concepts += f'\"{concept}\", '\n",
- " concepts = concepts[:-2] #remove last ,\n",
- " \n",
- " type_str = \"\"\n",
- " for k, v in types.items():\n",
- " type_str += f'\"{k}\":\"{v}\", '\n",
- " type_str = type_str[:-2]\n",
- " \n",
- " meta_str = '\"metadata\":['\n",
- " for v in metadata:\n",
- " meta_str += f'\"{v}\", '\n",
- " meta_str = meta_str[:-2]\n",
- " meta_str = meta_str + \"]\"\n",
- " \n",
- " return '''\n",
- " { \n",
- " \\\"file\\\":\\\"'''+file+'''\",\n",
- " \\\"columns\\\":{\n",
- " '''+type_str+''',\n",
- " '''+meta_str+'''\n",
- " },\n",
- " \\\"meldb_phenotypes\\\":['''+concepts+''']\n",
- " },'''\n",
- "\n",
- "out = '\"files\":['\n",
- "folder = \"codes/GitHub_TG_repository/\"\n",
- "for file, grp in df.groupby(\"mapped_condition\"):\n",
- " file = file.replace(\"%20\", \" \") \n",
- " \n",
- " for ext in [\"_CPRD_GOLD.csv\", \"_CPRD_AURUM.csv\", \"_IMRD.csv\"]:\n",
- " path = file+\"/\"+file+ext\n",
- " if os.path.isfile(folder+path):\n",
- " out+= json_file_template(path, grp[\"meldb_condition\"],\n",
- " types={\n",
- " \"read2_code\":\"READ_CODE\",\n",
- " \"snomed_code\":\"SNOMED_CT_CODE\",\n",
- " # \"med_code\":\"MEDICAL_CODE_ID\",\n",
- " },\n",
- " metadata = [\"DESCRIPTION\"]\n",
- " )\n",
- " else:\n",
- " print(\"NOT FILE\", folder+path)\n",
- " for ext in [\"_ICD10.csv\"]:\n",
- " path = file+\"/\"+file+ext\n",
- " if os.path.isfile(folder+path):\n",
- " out+= json_file_template(path, grp[\"meldb_condition\"],\n",
- " types={\n",
- " \"icd10_code\":\"READ_CODE\",\n",
- " \"snomed_code\":\"SNOMED_CT_CODE\",\n",
- " # \"icd10_code\":\"MEDICAL_CODE_ID\",\n",
- " },\n",
- " metadata = [\"DESCRIPTION\"]\n",
- " )\n",
- " else:\n",
- " print(\"NOT FILE\", folder+path)\n",
- " \n",
- " \n",
- " \n",
- " # out+= json_file_template(file+\"/\"+file+\"_CPRD_AURUM.csv\", grp[\"meldb_condition\"])\n",
- " # out+= json_file_template(file+\"/\"+file+\"_ICD10.csv\", grp[\"meldb_condition\"])\n",
- " # out+= json_file_template(file+\"/\"+file+\"_IMRD.csv\", grp[\"meldb_condition\"])\n",
- "\n",
- " # out += f' \"{file}/{file}_CPRD_GOLD.csv\":[{conds}],\\n'\n",
- " # out += f' \"{file}/{file}_CPRD_AURUM.csv\":[{conds}],\\n'\n",
- " # out += f' \"{file}/{file}_ICD10.csv\":[{conds}],\\n'\n",
- " # out += f' \"{file}/{file}_IMRD.csv\":[{conds}],\\n'\n",
- " \n",
- "out = out[:-1] #remove last ,\n",
- "out += \"\\n]\"\n",
- "out = out.replace(\"%20\", \" \") \n",
- "print(out)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f155b635-b459-4aff-81b2-e065fc223858",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d040eda5-4028-4047-834c-7315e307e415",
- "metadata": {},
- "outputs": [],
- "source": [
- "df = pd.read_parquet(\"maps/processed/icd10_code.parquet\")\n",
- "df\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "e0228ac9-8852-4818-b7f0-98429ca5229c",
- "metadata": {},
- "outputs": [],
- "source": [
- "code = [\"A00.0\", \"*00.0\"]\n",
- "code = pd.Series(code)\n",
- "print(code.isin(df[\"icd10_code\"]))\n",
- "print(code.isin(df[\"icd10_alt_code\"]))\n",
- "# print( )\n",
- "~(\n",
- " ~code.isin(df[\"icd10_code\"]) \n",
- " &\n",
- " ~code.isin(df[\"icd10_alt_code\"])\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "18efcacd-45f0-4341-86cc-d8e2e584350c",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Analyse the JSON file"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "85dc197b-451e-4fa9-a53b-e6770c132123",
- "metadata": {},
- "outputs": [],
- "source": [
- "import json\n",
- "import os\n",
- "\n",
- "path_json = \"../concepts/PHEN_assign_v3.json\"\n",
- "\n",
- "#Load JSON Concept Definitions\n",
- "mapping = json.load(open(path_json,'rb'))\n",
- "summary_config = mapping[\"concept_sets\"][\"concept_set\"]\n",
- "summary_df = pd.DataFrame(summary_config) #change to dataframe\n",
- "\n",
- "summary_df = summary_df.join(pd.json_normalize(summary_df[\"metadata\"])) #metadata to columns\n",
- "summary_df = summary_df.drop(columns=[\"metadata\"])\n",
- "summary_df = summary_df.rename(columns={\"concept_set_name\":\"CONCEPT_SET\"})\n",
- "summary_df = summary_df.drop_duplicates() #remove duplicates\n",
- " \n",
- "summary_df\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4c9b6b3f-08aa-4f61-b9b2-44a24b5d00a0",
- "metadata": {},
- "outputs": [],
- "source": [
- "import json\n",
- "import os\n",
- "\n",
- "path_json = \"PHEN_assign_v3.json\"\n",
- "path_excel = \"PHEN_summary_working.xlsx\"\n",
- "path_codes = \"codes/\"\n",
- "\n",
- "#Get all Files in JSON\n",
- "def get_json_files(path_json):\n",
- " folders = json.load(open(path_json,'rb'))\n",
- " out = []\n",
- " for folder in folders:\n",
- " if \"files\" in folder:\n",
- " for file in folder[\"files\"]:\n",
- " file_path = folder[\"folder\"]+\"/\"+file[\"file\"]\n",
- " if \"meldb_phenotypes\" in file:\n",
- " for concept in file[\"meldb_phenotypes\"]:\n",
- " out.append({\"json_concept\":concept, \"filepath\":file_path, \"json_code_types\":list(file[\"columns\"].keys())})\n",
- " elif \"meldb_phenotypes_categories\" in file:\n",
- " for code, concept in file[\"meldb_phenotypes_categories\"].items():\n",
- " out.append({\"json_concept\":concept[0], \"filepath\":file_path, \"json_code_types\":list(file[\"columns\"].keys())})\n",
- " else:\n",
- " out.append({\"json_concept\":None, \"filepath\":file_path})\n",
- "\n",
- " out = pd.DataFrame(out)\n",
- " out[\"filepath\"] = out[\"filepath\"].astype(str)\n",
- " return out\n",
- "out = get_json_files(path_json)\n",
- "\n",
- "#Get all Files Excel Summary\n",
- "def get_excel_files(path_excel):\n",
- " path_excel = \"PHEN_summary_working.xlsx\"\n",
- " out2 = pd.read_excel(path_excel)\n",
- " out2 = out2[[\"CONCEPT NAME \", \"CODING LIST\", \"AGREED\", \"FUNCTION\"]].loc[1:] #select relevant columns\n",
- "\n",
- " #Filter Concepts in use\n",
- " out2 = out2[out2[\"AGREED\"] == \"USE\"] #remove deprecated concepts\n",
- " out2 = out2[out2[\"FUNCTION\"] == \"QUERY BY CODING LIST\"] #remove deprecated concepts\n",
- " out2 = out2.drop(['AGREED', 'FUNCTION'], axis=1)\n",
- "\n",
- " #Get filepaths\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.split(\",\") #split by ,\n",
- " out2 = out2.explode(\"CODING LIST\") #one row per file\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.strip()\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.replace(\"https://git.soton.ac.uk/meld/meldb-external/phenotype/-/tree/main/\", \"\")\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.replace(\"%20\", \" \")\n",
- "\n",
- " out2 = out2.rename(columns={\"CONCEPT NAME \":\"excel_concept\", \"CODING LIST\":\"filepath\"})\n",
- " return out2\n",
- "out2 = get_excel_files(path_excel)\n",
- "\n",
- "#Get all Files in /codes\n",
- "def get_code_files(path_codes):\n",
- " all_files = []\n",
- " for root, dirs, files in os.walk(path_codes, topdown=False):\n",
- " for name in files:\n",
- " if \".ipynb_checkpoint\" not in root: #exclude notebook checkpoints\n",
- " if name.endswith(\".csv\") or name.endswith(\".xlsx\") or name.endswith(\".dta\"): #exclude non-data files\n",
- " all_files.append(os.path.join(root, name)) \n",
- " all_files = pd.DataFrame(all_files)\n",
- " all_files = all_files.rename(columns={0:\"filepath\"})\n",
- " all_files[\"filepath\"] = all_files[\"filepath\"].astype(str)\n",
- " return all_files\n",
- "all_files = get_code_files(path_codes)\n",
- "\n",
- "\n",
- "print(\"ALL FILES\", len(all_files), len(all_files[\"filepath\"].unique()))\n",
- "print(\"JSON CONCEPTS\", len(out), len(out[\"filepath\"].unique()))\n",
- "print(\"EXCEL CONCEPTS\", len(out2), len(out2[\"filepath\"].unique()))\n",
- "\n",
- "outs = pd.merge(all_files, out, how=\"outer\", on=\"filepath\")\n",
- "outs = pd.merge(outs, out2, how=\"outer\", on=\"filepath\")\n",
- "print(len(outs), len(outs[\"filepath\"].unique()))\n",
- "outs.to_csv(\"output/MELD_file_to_concept.csv\", index=False)\n",
- "\n",
- "# display(outs[ outs[\"concept\"].isna()])\n",
- "\n",
- "# display(out ) "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f8e70c33-c869-46f8-953e-f6b52992cfbb",
- "metadata": {},
- "outputs": [],
- "source": [
- "display(\"JSON MISSING\", outs[outs[\"json_concept\"].isna() & outs[\"excel_concept\"].notna()])\n",
- "display(\"EXCEL MISSING\", outs[outs[\"json_concept\"].notna() & outs[\"excel_concept\"].isna()])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "9d84465f-f064-4df2-b0e4-2dfb217aea21",
- "metadata": {},
- "outputs": [],
- "source": [
- "f = open('concepts-output/MELD-report.md', 'a') as f:\n",
- " f.write(\n",
- " \"\"\"\n",
- "# Report\n",
- "- One thing\n",
- "- Two thing\n",
- "- Three thing\n",
- " \"\"\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7f7fc771-e406-42c7-8a09-16a20b5298f5",
- "metadata": {},
- "outputs": [],
- "source": [
- "total_length = 0\n",
- "for file in all_files[\"filepath\"]:\n",
- " if file.endswith(\".csv\"):\n",
- " df_file = pd.read_csv(file)\n",
- " total_length += len(df_file)\n",
- " elif file.endswith(\".xlsx\"):\n",
- " df_file = pd.read_excel(file)\n",
- " total_length += len(df_file)\n",
- " elif file.endswith(\".dta\"):\n",
- " df_file = pd.read_stata(file)\n",
- " total_length += len(df_file)\n",
- "total_length\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08a9c565-28d6-46ee-9fa8-6fa0ee28a4d5",
- "metadata": {},
- "outputs": [],
- "source": [
- "#turn filepaths into gitlab links\n",
- "outs2 = outs.copy()\n",
- "outs2[\"filepath\"] = \"https://git.soton.ac.uk/meld/meldb-external/phenotype/-/tree/main/\"+outs2[\"filepath\"].str.replace(\" \", \"%20\")\n",
- "\n",
- "#Groupby concepts and concat filepaths\n",
- "outs2 = outs2.groupby(\"concept\")[\"filepath\"].apply(', '.join).reset_index()\n",
- "outs2 = outs2.sort_values(by=[\"concept\"])\n",
- "outs2\n",
- "outs2.to_csv(\"output/MELD_GitLab_link_to_concept.csv\", index=False)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "357bb84c-90c2-4b5f-95c0-443191783a7f",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Analyse Output Files"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7d3f9cb7-be86-4f1f-92f6-991094eb7bb7",
- "metadata": {},
- "outputs": [],
- "source": [
- "version = \"V2_2_2\"\n",
- "output_files = [f\"output/{version}_MELD_concepts_readv2.csv\",\n",
- " f\"output/{version}_MELD_snomed_no_translate.csv\",\n",
- " f\"output/{version}_MELD_icd10_no_translate.csv\",\n",
- " # f\"output/{version}_MELD_med_no_translate.csv\",\n",
- " f\"output/{version}_MELD_atc_no_translate.csv\"\n",
- " ]\n",
- "error_file = f\"output/{version}_MELD_errors.csv\"\n",
- "\n",
- "for output_file in output_files:\n",
- " print(\"---\"*3,output_file,\"---\"*3,)\n",
- " df = pd.read_csv(output_file)\n",
- " # df[\"MELDB_concept\"].loc[df[\"CONCEPT TYPE\"].isna()]\n",
- " print(\"MELDB missing concepts \", len(df[df[\"CONCEPT TYPE\"].isna()]))\n",
- " if df[\"code\"].dtype == \"object\":\n",
- " print(\"Chars present:\", np.sort(df[\"code\"].apply(lambda x : set(x)).explode().unique()))\n",
- " \n",
- "# len(df[\"MELDB_concept\"].unique())\n",
- "\n",
- "print(\"---\"*3,error_file,\"---\"*3,)\n",
- "df = pd.read_csv(error_file)\n",
- "df = df.drop_duplicates()\n",
- "df[\"CODE_TYPE\"].value_counts()\n",
- "# for i, row in df.drop_duplicates().iterrows():\n",
- "# print(row[\"CODE\"], row[\"CODE_TYPE\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08e0ecc1-9271-48c3-9c5b-094800072906",
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_output_files(version):\n",
- " output_files = [f\"output/{version}_MELD_concepts_readv2.csv\",\n",
- " f\"output/{version}_MELD_snomed_no_translate.csv\",\n",
- " f\"output/{version}_MELD_icd10_no_translate.csv\",\n",
- " # f\"output/{version}_MELD_med_no_translate.csv\",\n",
- " f\"output/{version}_MELD_atc_no_translate.csv\"\n",
- " ]\n",
- " error_file = f\"output/{version}_MELD_errors.csv\"\n",
- " return output_files, error_file\n",
- "\n",
- "# version_1 = \"V1_0_0\"\n",
- "version_1 = \"V2_1_4\"\n",
- "version_2 = \"V2_2_3\"\n",
- "output1, err1 = get_output_files(version_1)\n",
- "output2, err2 = get_output_files(version_2)\n",
- "\n",
- "print(\"## Compare Concepts\", version_1, \"to\", version_2)\n",
- "\n",
- "for out1, out2 in zip(output1, output2):\n",
- " print(out1, out2 )\n",
- " df1 = pd.read_csv(out1)\n",
- " df1 = df1[[\"code\",\"MELDB_concept\"]].groupby(\"MELDB_concept\").count()\n",
- " df2 = pd.read_csv(out2)\n",
- " df2 = df2[[\"code\",\"MELDB_concept\"]].groupby(\"MELDB_concept\").count()\n",
- " \n",
- " #Added/Removed Concepts\n",
- " print(\"- Removed Concepts\", list(set(df1.index) - set(df2.index)))\n",
- " print(\"- Added Concepts\", list(set(df2.index) - set(df1.index)))\n",
- " \n",
- " #Changed Concepts\n",
- " diff = df2 - df1 #diff in counts \n",
- " diff = diff[(~(diff[\"code\"] == 0.0)) & diff[\"code\"].notna()] #get non-zero counts\n",
- " s = \"\\n\"\n",
- " for concept, row in diff.iterrows():\n",
- " s += \"\\t - {} {}\\n\".format(concept, row[\"code\"])\n",
- " print(\"- Changed Concepts\", s)\n",
- "\n",
- "\n",
- "# for output_file in output_files:\n",
- "# print(\"---\"*3,output_file,\"---\"*3,)\n",
- "# df = pd.read_csv(output_file)\n",
- "# # df[\"MELDB_concept\"].loc[df[\"CONCEPT TYPE\"].isna()]\n",
- "# print(\"MELDB missing concepts \", len(df[df[\"CONCEPT TYPE\"].isna()]))\n",
- "# if df[\"code\"].dtype == \"object\":\n",
- "# print(\"Chars present:\", np.sort(df[\"code\"].apply(lambda x : set(x)).explode().unique()))\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "cc60c137-5a85-4155-af6b-6796f8c05980",
- "metadata": {},
- "outputs": [],
- "source": [
- "import glob\n",
- "import os\n",
- "import pandas as pd\n",
- "\n",
- "df = pd.read_csv(\"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/PHEN_summary_working.csv\")\n",
- "df = df.set_index(\"#\")\n",
- "\n",
- "for vocab in [\"atc\", \"icd10\", \"readv2\", \"snomed\"]:\n",
- " df[vocab.upper()] = \"\"\n",
- "\n",
- " for file in glob.glob(f\"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/{vocab}/*.csv\"):\n",
- " concept_set = os.path.basename(file)[:-4]\n",
- " row_index = df[df[\"CONCEPT NAME \"] == concept_set].index[0]\n",
- "\n",
- " df.loc[row_index, vocab.upper()] = \"YES\"\n",
- "\n",
- "df = df.drop(columns=[\"READv2_CODE\", \"ICD10_CODE\"])\n",
- "df.to_csv(\"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/PHEN_summary_working_labelled.csv\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e5c4291f-847b-4c82-976e-bd5b3a7b6bcc",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Mappings"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08e34750-413c-469e-bcb8-e71bb188ff42",
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "#NHS Read Browser\n",
- "import simpledbf\n",
- "import pandas as pd\n",
- "\n",
- "\n",
- "#r2 only\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/ANCESTOR.DBF').to_dataframe()\n",
- "df = pd.concat([df['READCODE'], df['DESCENDANT']])\n",
- "df = pd.DataFrame(df.drop_duplicates())\n",
- "df = df.rename(columns={0:\"read2_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> atc\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/ATC.DBF').to_dataframe()\n",
- "df = df[[\"READCODE\", \"ATC\"]]\n",
- "df = df.rename(columns={\"READCODE\":\"read2_code\", \"ATC\":\"atc_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code_to_atc_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> icd10\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/ICD10.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read2_code\", \"TARG_CODE\":\"icd10_code\"})\n",
- "df = df[~df[\"icd10_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read2_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read2_code_to_icd10_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> opcs4\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/OPCS4V3.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read2_code\", \"TARG_CODE\":\"opcs4_code\"})\n",
- "df = df[~df[\"opcs4_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read2_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read2_code_to_opcs4_code.parquet\", index=False)\n",
- "\n",
- "#r3 only\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/ANCESTOR.DBF').to_dataframe()\n",
- "df = pd.concat([df['READCODE'], df['DESCENDANT']])\n",
- "df = pd.DataFrame(df.drop_duplicates())\n",
- "df = df.rename(columns={0:\"read3_code\"})\n",
- "df.to_parquet(\"maps/processed/read3_code.parquet\", index=False)\n",
- "\n",
- "#r3 -> icd10\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/ICD10.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read3_code\", \"TARG_CODE\":\"icd10_code\"})\n",
- "df = df[~df[\"icd10_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read3_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_icd10_code.parquet\", index=False)\n",
- "\n",
- "#r3 -> icd9\n",
- "# dbf = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/ICD9V3.DBF')\n",
- "\n",
- "#r3 -> opcs4\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/OPCS4V3.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read3_code\", \"TARG_CODE\":\"opcs4_code\"})\n",
- "df = df[~df[\"opcs4_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read3_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_opcs4_code.parquet\", index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5fe95638-1f25-45f3-803c-2fff74a2a4fd",
- "metadata": {},
- "outputs": [],
- "source": [
- "#NHS Data Migrations\n",
- "\n",
- "#r2 only\n",
- "# df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/rctcremap_uk_20200401000001.txt', sep='\\t')\n",
- "\n",
- "#r3 only\n",
- "# df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/ctv3cremap_uk_20200401000001.txt', sep='\\t')\n",
- "\n",
- "#snomed only\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/sctcremap_uk_20200401000001.txt', sep='\\t')\n",
- "df = df[[\"SCT_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"SCT_CONCEPTID\":\"snomed_code\"})\n",
- "df = df.drop_duplicates()\n",
- "df = df.astype(str)\n",
- "df.to_parquet(\"maps/processed/snomed_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> r3\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/rctctv3map_uk_20200401000001.txt', sep='\\t')\n",
- "df = df[[\"V2_CONCEPTID\", \"CTV3_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"V2_CONCEPTID\":\"read2_code\",\n",
- " \"CTV3_CONCEPTID\":\"read3_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code_to_read3_code.parquet\", index=False)\n",
- "\n",
- "#r3->r2\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/ctv3rctmap_uk_20200401000002.txt', sep='\\t')\n",
- "df = df[[\"CTV3_CONCEPTID\", \"V2_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"CTV3_CONCEPTID\":\"read3_code\", \n",
- " \"V2_CONCEPTID\":\"read2_code\"})\n",
- "df = df.drop_duplicates()\n",
- "df = df[~df[\"read2_code\"].str.match(\"^.*_.*$\")] #remove r2 codes with '_'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_read2_code.parquet\", index=False)\n",
- "\n",
- "\n",
- "#r2 -> snomed\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/rcsctmap2_uk_20200401000001.txt', sep='\\t', dtype=str)\n",
- "df = df[[\"ReadCode\", \"ConceptId\"]]\n",
- "df = df.rename(columns={\"ReadCode\":\"read2_code\",\n",
- " \"ConceptId\":\"snomed_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code_to_snomed_code.parquet\", index=False)\n",
- "\n",
- "\n",
- "#r3->snomed\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/ctv3sctmap2_uk_20200401000001.txt', sep='\\t')\n",
- "df = df[[\"CTV3_TERMID\", \"SCT_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"CTV3_TERMID\":\"read3_code\",\n",
- " \"SCT_CONCEPTID\":\"snomed_code\"})\n",
- "df[\"snomed_code\"] = df[\"snomed_code\"].astype(str)\n",
- "df = df[~df[\"snomed_code\"].str.match(\"^.*_.*$\")] #remove snomed codes with '_'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_snomed_code.parquet\", index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "267fa1cc-5159-48c4-9eee-19af5039d627",
- "metadata": {},
- "outputs": [],
- "source": [
- "#OPCS410 Data Files\n",
- "df = pd.read_csv(\"maps/OPCS410 Data files txt/OPCS410 CodesAndTitles Nov 2022 V1.0.txt\", sep='\\t', dtype=str, header=None)\n",
- "df = df.rename(columns={0:\"opcs4_code\", 1:\"description\"})\n",
- "df.to_parquet(\"maps/processed/opcs4_code.parquet\", index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "01d046fd-69af-44f3-acad-5d0edef3f745",
- "metadata": {},
- "outputs": [],
- "source": [
- "#ICD10_edition5\n",
- "df = pd.read_xml(\"maps/ICD10_Edition5_XML_20160401/Content/ICD10_Edition5_CodesAndTitlesAndMetadata_GB_20160401.xml\",)\n",
- "df = df[[\"CODE\", \"ALT_CODE\", \"DESCRIPTION\"]]\n",
- "df = df.rename(columns={\"CODE\":\"icd10_code\",\n",
- " \"ALT_CODE\":\"icd10_alt_code\",\n",
- " \"DESCRIPTION\":\"description\"\n",
- " })\n",
- "df.to_parquet(\"maps/processed/icd10_code.parquet\", index=False)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "36630e24-f56c-48e1-8ecf-4ccd2b41eaea",
- "metadata": {},
- "outputs": [],
- "source": [
- "code1=\"read2_code\"\n",
- "code2=\"icd10_code\"\n",
- "df_map = pd.read_parquet(f\"maps/processed/{code1}_to_{code2}.parquet\")\n",
- "\n",
- "codes=df_map[\"read2_code\"].iloc[:5]\n",
- "\n",
- "pd.merge(codes, df_map, how='left')[code2]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "9787adeb-8507-488b-9a91-b8df3fbbe21e",
- "metadata": {},
- "outputs": [],
- "source": [
- "#CPRD Code Browser\n",
- "df = pd.read_csv('maps/CPRD_CodeBrowser_202211_Aurum/CPRDAurumMedical.txt', sep='\\t')\n",
- "df = df[[\"MedCodeId\", \"CleansedReadCode\", \"SnomedCTConceptId\"]]\n",
- "df = df.rename(columns={\"MedCodeId\":\"med_code\",\n",
- " \"CleansedReadCode\":\"read2_code\",\n",
- " \"SnomedCTConceptId\":\"snomed_code\"})\n",
- "\n",
- "# df = pd.read_csv('maps/CPRD_CodeBrowser_202211_Aurum/CPRDAurumProduct.txt', sep='\\t', dtype=str)\n",
- "\n",
- "# df = pd.read_csv('maps/CPRD_CodeBrowser_202211_GOLD/medical.txt', sep='\\t')\n",
- "# df = df.reset_index().iloc[:,[1,6]]\n",
- "# df = df.rename(columns={\"level_1\":\"read2_code\", \"20220523\":\"description\"})\n",
- "# df = pd.read_csv('maps/CPRD_CodeBrowser_202211_GOLD/product.txt', sep='\\t', dtype=str) #CANNOT OPEN\n",
- "\n",
- "df"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a968ffb1-4337-456b-8d20-419888b4044f",
- "metadata": {},
- "outputs": [],
- "source": [
- "#BNF\n",
- "\n",
- "df = pd.read_excel(\"maps/BNF Snomed Mapping data 20231215.xlsx\")\n",
- "df = df.astype(str)\n",
- "df = df.rename(columns={\"BNF Code\":\"bnf_code\",\n",
- " \"SNOMED Code\":\"snomed_code\"})\n",
- "df[[\"bnf_code\", \"snomed_code\"]].to_parquet(\"maps/processed/bnf_code_to_snomed_code.parquet\", index=False)\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c70b1ce2-0f41-4d02-ad17-6fc44bc3c6bf",
- "metadata": {},
- "outputs": [],
- "source": [
- "#BNF to Readv2 Merge\n",
- "df1 = pd.read_parquet(\"maps/processed/bnf_code_to_snomed_code.parquet\").astype(str)\n",
- "df2 = pd.read_parquet(\"maps/processed/read2_code_to_snomed_code.parquet\").astype(str)\n",
- "# df1.merge(df2, how=\"inner\", on=\"snomed_code\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d5d34237-02d4-4dea-8c20-5adaf337f6b5",
- "metadata": {},
- "outputs": [],
- "source": [
- "df1.merge(df2, how='inner', on='snomed_code')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b3166cf0-e4a5-43e0-aeac-78827427422e",
- "metadata": {},
- "outputs": [],
- "source": [
- ".astype(str).dtypes"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c0a766f9-7959-4a10-b58f-cd946a878b60",
- "metadata": {},
- "outputs": [],
- "source": [
- "df = pd.read_csv(\"../concepts/PHEN_summary_working.csv\")\n",
- "cols = list(df.columns)\n",
- "cols.remove('CONCEPT NAME ')\n",
- "cols.remove('AGREED')\n",
- "df = df.applymap(lambda x: str(x) if isinstance(x, (int, float)) else x) #change to int\n",
- "\n",
- "df_copy = df.rename(columns={\n",
- " \"CONCEPT NAME \":\"concept_set_name\",\n",
- " \"AGREED\":\"concept_set_status\"\n",
- "})\n",
- "df_copy[\"concept_set_status\"] = df_copy[\"concept_set_status\"].replace(\"USE\", \"AGREED\")\n",
- "df_copy = df_copy[[\"concept_set_name\", \"concept_set_status\"]]\n",
- "outs = df_copy.to_dict(orient='records')\n",
- "\n",
- "for i, out in enumerate(outs):\n",
- " out[\"metadata\"] = dict(df[cols].iloc[i])\n",
- " \n",
- "json.dumps(outs)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8a204a95-dc4c-4183-9ea7-f5c5e95e9087",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5ce1ab58-50b4-4c22-b72b-c698de6830f7",
- "metadata": {},
- "outputs": [],
- "source": [
- "import json"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f1ea81c6-d1db-408f-9d3a-b96f44efe21f",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "markdown",
- "id": "5eb544a3-9dd1-41e8-88c2-a808646c6112",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### OMOP Database"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c9e58e62-9e44-4d0c-9d8d-35c175c07e6c",
- "metadata": {},
- "outputs": [],
- "source": [
- "import sqlite3\n",
- "import csv\n",
- "import pandas as pd\n",
- "import os"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4f67c9a1-373f-4799-8a85-72767662d912",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d0ecdf69-ee90-42c1-ad25-d8357b603d1b",
- "metadata": {},
- "outputs": [],
- "source": [
- "#IMPORT OMOP VOCABS\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\") # change to 'sqlite:///your_filename.db'\n",
- "folder_path = \"codes/vocabulary_download_v5_{9424944c-2b76-4127-8f05-f535e0f15e2a}_1731661390540\"\n",
- "\n",
- "# Check if the folder exists\n",
- "if not os.path.isdir(folder_path):\n",
- " raise Exception(f\"Error: The folder '{folder_path}' does not exist.\") \n",
- "\n",
- "# Iterate through files in the folder\n",
- "for filename in os.listdir(folder_path):\n",
- " if filename.endswith(\".csv\"): # Check if the file is a CSV\n",
- " file_path = os.path.join(folder_path, filename)\n",
- " try:\n",
- " print(f\"Reading file: {file_path}\")\n",
- " # Read the CSV file with the specified delimiter\n",
- " df = pd.read_csv(file_path, delimiter=\"\\t\", low_memory=False)\n",
- " table_name = os.path.splitext(os.path.basename(file_path))[0] #Get name of file\n",
- " \n",
- " #Export Table to sqlite db\n",
- " df.to_sql(table_name, conn, if_exists='replace', index=False)\n",
- " \n",
- " except Exception as e:\n",
- " raise Exception(f\"Error reading file {file_path}: {e}\")\n",
- "\n",
- "conn.commit()\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b9cafd0c-a3bd-408b-bca8-b0de2acde1cd",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Create a SQL connection to our SQLite database\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\")\n",
- "cur = conn.cursor()\n",
- "\n",
- "#Print ALL Columns in Table\n",
- "# table=\"CONCEPT_SET\"\n",
- "# cur.execute(f\"PRAGMA table_info({table});\")\n",
- "# print(pd.DataFrame(cur.fetchall()))\n",
- "\n",
- "#Print ALL TABLE NAMES\n",
- "# cur.execute(\"SELECT name FROM sqlite_master WHERE type='table' AND name=? ;\", (\"VOCABULARY\",))\n",
- "# print(cur.fetchone())\n",
- " \n",
- "cur.execute(\"SELECT vocabulary_id FROM VOCABULARY WHERE vocabulary_id=? ;\", (\"MELDB\",))\n",
- "print(cur.fetchone())\n",
- "\n",
- " \n",
- " \n",
- "#Print WHOLE TABLE\n",
- "# cur.execute('SELECT * FROM CONCEPT;')\n",
- "# cur.execute('SELECT * FROM CONCEPT WHERE standard_concept = \"C\";')\n",
- "# cur.execute('SELECT * FROM CONCEPT WHERE concept_code = \"119768002\" LIMIT 1;')\n",
- "# cur.execute('SELECT * FROM CONCEPT WHERE concept_code IN (\"119768002\", \"5905001\");')\n",
- "# cur.execute('SELECT DISTINCT VOCABULARY_ID FROM CONCEPT;')\n",
- "# df = pd.DataFrame(cur.fetchall())\n",
- "# print(list(df[0]))\n",
- "# display(df)\n",
- "# for row in :\n",
- " # print(row)\n",
- "\n",
- "\n",
- "\n",
- "#Get Header of Table\n",
- "# table=\"CONCEPT_CLASS\"\n",
- "# cur.execute(f\"SELECT * FROM {table} LIMIT 3;\")\n",
- "# print(cur.fetchall())\n",
- "\n",
- "#create meldb VOCABULARY\n",
- "# meldb_version='v3.2.10'\n",
- "# meldb_description = 'Multidisciplinary Ecosystem to study Lifecourse Determinants and Prevention of Early-onset Burdensome Multimorbidity'\n",
- "# meldb_reference = 'https://www.it-innovation.soton.ac.uk/projects/meldb'\n",
- "# df_test = pd.DataFrame([{\n",
- "# \"vocabulary_id\": 'MELDB',\n",
- "# \"vocabulary_name\": meldb_description,\n",
- "# \"vocabulary_reference\": meldb_reference,\n",
- "# \"vocabulary_version\": meldb_version,\n",
- "# # \"vocabulary_concept_id\": 0,\n",
- "# }])\n",
- "# df_test.to_sql(\"VOCABULARY\", conn, if_exists='append', index=False)\n",
- "\n",
- "\n",
- "# cur.execute(\"\"\"\n",
- "# CREATE TABLE CONCEPT_SET (\n",
- "# concept_set_id INTEGER PRIMARY KEY AUTOINCREMENT, -- Unique identifier for each concept set\n",
- "# atlas_id INTEGER, -- Unique identifier generated by ATLAS\n",
- "# concept_set_name TEXT, -- Optional name for the concept set\n",
- "# concept_set_description TEXT, -- Optional description for the concept set\n",
- "# vocabulary_id TEXT NOT NULL, -- Foreign key to VOCABULARY table\n",
- "# FOREIGN KEY (vocabulary_id) REFERENCES VOCABULARY(vocabulary_id)\n",
- "# );\"\"\")\n",
- "# cur.execute(\"DROP TABLE CONCEPT_SET;\")\n",
- "\n",
- "# cur.execute(\"\"\"\n",
- "# CREATE TABLE CONCEPT_SET_ITEM (\n",
- "# concept_set_item_id INTEGER PRIMARY KEY AUTOINCREMENT, -- Unique identifier for each mapping\n",
- "# concept_set_id INTEGER NOT NULL, -- Foreign key to CONCEPT_SET table\n",
- "# concept_id INTEGER NOT NULL, -- Foreign key to CONCEPT table\n",
- "# FOREIGN KEY (concept_set_id) REFERENCES CONCEPT_SET(concept_set_id),\n",
- "# FOREIGN KEY (concept_id) REFERENCES CONCEPT(concept_id)\n",
- "# );\"\"\")\n",
- "# cur.execute(\"DROP TABLE CONCEPT_SET_ITEM;\")\n",
- "\n",
- "# Be sure to close the connection\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d03b75f3-902f-42d7-b52f-dac7e79ecb11",
- "metadata": {},
- "outputs": [],
- "source": [
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\") # change to 'sqlite:///your_filename.db'\n",
- "cur = conn.cursor()\n",
- "\n",
- "file_path = \"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/snomed/HEART_VALVE_DISORDERS.csv\"\n",
- "df = pd.read_csv(file_path, low_memory=False)\n",
- "df = df.set_index(\"code\")\n",
- "\n",
- "df.to_sql(name='test', con=conn, if_exists='replace')\n",
- "\n",
- "conn.commit()\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d96c3511-3831-400e-ba40-0a36abcc60d3",
- "metadata": {},
- "outputs": [],
- "source": [
- "#DISPLAY SQL TABLE\n",
- "table=\"CONCEPT_SET_ITEM\"\n",
- "\n",
- "# Create a SQL connection to our SQLite database\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\")\n",
- "cur = conn.cursor()\n",
- "\n",
- "#Print ALL Columns in Table\n",
- "cur.execute(f\"PRAGMA table_info({table});\")\n",
- "df_cols = pd.DataFrame(cur.fetchall())\n",
- "print(df_cols)\n",
- "df_cols = df_cols[1]\n",
- "\n",
- "#Print TABLE\n",
- "cur.execute(f\"SELECT * FROM {table};\")\n",
- "df = pd.DataFrame(cur.fetchall())\n",
- "df = df.rename(columns={i:s for i, s in enumerate(df_cols)})\n",
- "display(df)\n",
- "\n",
- "conn.close()\n",
- "\n",
- "\n",
- "# a+s = 13364 \n",
- "# a+s+i = 13591\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "42d49a00-9646-4ba4-afb6-12297289b7a7",
- "metadata": {},
- "outputs": [],
- "source": [
- "def sql_row_exist(conn, table, column, value):\n",
- "\t# Execute and check if a result exists\n",
- "\tcur = conn.cursor()\n",
- "\tquery = f\"SELECT 1 FROM {table} WHERE {column} = ? LIMIT 1;\"\n",
- "\tcur.execute(query, (value,))\n",
- "\texists = cur.fetchone() is not None\n",
- "\t\n",
- "\treturn exists"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f7b51bcd-6ee1-4023-8d36-7f419ce4120d",
- "metadata": {},
- "outputs": [],
- "source": [
- "#EXPORT MELDB CSV OUTPUT\n",
- "\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\") # change to 'sqlite:///your_filename.db'\n",
- "cur = conn.cursor()\n",
- "\n",
- "vocab_output = \"MELDB\"\n",
- "vocab_type = \"SNOMED\"\n",
- "file_path = \"/home/jjd1c23/ssd/meldb/jjd1c23/phenotype/output/V3_2_10_MELD_snomed_no_translate.csv\"\n",
- "# file_path = \"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/snomed/HEART_VALVE_DISORDERS.csv\"\n",
- "\n",
- "# Read the CSV file with the specified delimiter\n",
- "out = pd.read_csv(file_path, low_memory=False)\n",
- "print(df.columns)\n",
- "\n",
- "for concept_set_name, grp in out.groupby(\"MELDB_concept\"):\n",
- " # display(concept_set_name, grp[[\"code\", \"MELDB_concept\"]])\n",
- " \n",
- " #Create Concept_Set\n",
- " if not sql_row_exist(conn, \"CONCEPT_SET\", \"concept_set_name\", concept_set_name):\n",
- " cur.execute(f\"INSERT INTO CONCEPT_SET (concept_set_name, vocabulary_id) VALUES ('{concept_set_name}', 'MELDB');\")\n",
- " else:\n",
- " print(\"concept_set\", concept_set_name, \"already exists\")\n",
- " #TODO: ask to remove old concept_set?\n",
- " \n",
- " #Get Concept_set_Id\n",
- " query = \"SELECT concept_set_id FROM CONCEPT_SET WHERE concept_set_name = ? AND vocabulary_id = ?;\"\n",
- " cur.execute(query, (concept_set_name, vocab_output, )) \n",
- " concept_set_id = cur.fetchone()[0]\n",
- " \n",
- " #Get corresponing Concept_id (OMOP) for each Concept_code (e.g. SNOMED)\n",
- " concept_codes = \"'\"+\"', '\".join(list(grp[\"code\"].astype(str)))+\"'\"\n",
- " query = f\"SELECT concept_id FROM CONCEPT WHERE vocabulary_id = ? AND concept_code IN ({concept_codes});\"\n",
- " print(query)\n",
- " cur.execute(query, (vocab_type, ))\n",
- " df_out = pd.DataFrame(cur.fetchall(), columns=[\"concept_id\"])\n",
- " \n",
- " if not len(grp) == len(df_out):\n",
- " print(\"ERROR: Some\", vocab_type, \"Codes do not exist in OMOP Database\")\n",
- " \n",
- " #Create Concept_set_item\n",
- " df_out[\"concept_set_id\"] = concept_set_id\n",
- " df_out.to_sql(\"CONCEPT_SET_ITEM\", conn, if_exists='append', index=False)\n",
- " \n",
- " display(df_out)\n",
- " \n",
- " \n",
- " \n",
- " # break\n",
- " \n",
- " \n",
- "\n",
- "# #Create New CONCEPT_SET\n",
- "# table_name = os.path.splitext(os.path.basename(file_path))[0] #Get name of file\n",
- "# cur.execute(f\"INSERT INTO CONCEPT_SET (concept_class_name) VALUES ('{table_name}');\")\n",
- " \n",
- " \n",
- " \n",
- "\n",
- " \n",
- " \n",
- "\n",
- "\n",
- "conn.commit()\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "85007741-e34c-4112-a63c-9fb302b76958",
- "metadata": {},
- "outputs": [],
- "source": [
- "\"'\"+\"', '\".join(list(grp[\"code\"].astype(str)))+\"'\""
- ]
- },
- {
- "cell_type": "markdown",
- "id": "423e7c21-f3bd-439d-9dcb-c17cc2cc6854",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### ATLAS"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c6b45e4d-c7d2-42e7-9b4a-0e9c1c86d34b",
- "metadata": {},
- "outputs": [],
- "source": [
- "#Create ATLAS Concept Set\n",
- "\n",
- "def atlas_create_concept(name, description=\"\", items=[]):\n",
- " data={\n",
- " \"id\": 0,\n",
- " \"name\": name,\n",
- " \"description\": description,\n",
- " \"expression\": {\n",
- " \"items\":items \n",
- " }\n",
- " }\n",
- "\n",
- " try:\n",
- " # Sending the POST request\n",
- " response = requests.post(url, json=data, headers=headers)\n",
- "\n",
- " # Check the response status\n",
- " if response.status_code == 200 or response.status_code == 201:\n",
- " print(\"POST request successful:\")\n",
- " print(response.json()) # Assuming the response is JSON\n",
- " return response[\"id\"]\n",
- " else:\n",
- " print(f\"POST request failed. HTTP Status Code: {response.status_code}\")\n",
- " print(\"Response content:\")\n",
- " print(response.text)\n",
- " return None\n",
- "\n",
- " except requests.exceptions.RequestException as e:\n",
- " print(f\"An error occurred: {e}\")\n",
- "\n",
- "# Heart Test 1 - 1885487\n",
- "# Heart Test 2 - 1885488\n",
- "# Heart Valve Disorders - 1885449\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "45497623-1da0-4f74-b21e-da8811c89b04",
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_omop_concepts(cur, codes, vocab_id): \n",
- " #Create List for SQL\n",
- " mask = \"\"\n",
- " for c in codes:\n",
- " mask+=f'\"{c}\", '\n",
- " mask = mask[:-2] #remove last comma\n",
- " \n",
- " #Execute SQL\n",
- " cur.execute(f'SELECT * FROM CONCEPT WHERE concept_code IN ({mask}) AND VOCABULARY_ID = \"{vocab_id}\";')\n",
- " df = pd.DataFrame(cur.fetchall()) #convert to pandas df\n",
- " \n",
- " print(\"Identified\", len(df[0]) ,\"OMOP Concepts:\", list(df[0]))\n",
- " \n",
- " return df\n",
- " \n",
- "def omop_concepts_to_atlas_json(df):\n",
- " json = []\n",
- " for i, row in df.iterrows():\n",
- " #template for atlas api\n",
- " out = { \n",
- " \"concept\": {\n",
- " 'CONCEPT_ID': row[0],\n",
- " 'CONCEPT_NAME': row[1],\n",
- " 'STANDARD_CONCEPT': 'S',\n",
- " 'STANDARD_CONCEPT_CAPTION': 'Standard',\n",
- " 'INVALID_REASON': 'V',\n",
- " 'INVALID_REASON_CAPTION': 'Valid',\n",
- " 'CONCEPT_CODE': row[6],\n",
- " 'DOMAIN_ID': row[2],\n",
- " 'VOCABULARY_ID': row[3],\n",
- " 'CONCEPT_CLASS_ID': row[4],\n",
- " 'VALID_START_DATE': int(row[7]),\n",
- " 'VALID_END_DATE': int(row[8])\n",
- " },\n",
- " 'isExcluded': False,\n",
- " 'includeDescendants': False,\n",
- " 'includeMapped': False\n",
- " }\n",
- " json.append(out)\n",
- " return json \n",
- "\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\")\n",
- "cur = conn.cursor()\n",
- "\n",
- "vocab_id=\"SNOMED\" #SNOMED, ATC, ICD10CM, ICD9CM, Read\n",
- "csv_output = \"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/snomed/ANGER.csv\"\n",
- "\n",
- "#Load CSV Output File\n",
- "df_in = pd.read_csv(csv_output)\n",
- "print(len(df_in))\n",
- "\n",
- "# df = get_omop_concepts(cur, [\"119768002\", \"5905001\"], \"SNOMED\")\n",
- "df = get_omop_concepts(cur, list(df_in[\"code\"]), vocab_id)\n",
- "json = omop_concepts_to_atlas_json(df)\n",
- "# display(json)\n",
- "\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ea759907-c085-472a-82e2-07b6b19e2c8f",
- "metadata": {},
- "outputs": [],
- "source": [
- "#ATLAS GET CONCEPT SET\n",
- "import requests\n",
- "\n",
- "def request_get(url):\n",
- " try:\n",
- " # Sending the GET request\n",
- " response = requests.get(url)\n",
- "\n",
- " # Check if the response status code is 200 (OK)\n",
- " if response.status_code == 200:\n",
- " print(\"Response data:\")\n",
- " # print(response.json()) # Assuming the response is in JSON format\n",
- " return response.json()\n",
- " else:\n",
- " print(f\"Failed to fetch data. HTTP Status Code: {response.status_code}\")\n",
- " print(\"Response content:\")\n",
- " print(response.text)\n",
- " return None\n",
- "\n",
- " except requests.exceptions.RequestException as e:\n",
- " print(f\"An error occurred: {e}\")\n",
- "\n",
- "\n",
- "#GET SET INFO\n",
- "set_id = \"1885449\"\n",
- "url = f\"https://atlas-demo.ohdsi.org/WebAPI/conceptset/{set_id}\"\n",
- "request_get(url)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5a70e636-6051-4930-bf1b-30d093fd0552",
- "metadata": {},
- "outputs": [],
- "source": [
- "#GET SET ITEMS (Concepts)\n",
- "set_id = \"1885449\"\n",
- "url = f\"https://atlas-demo.ohdsi.org/WebAPI/conceptset/{set_id}/expression/ATLASPROD\"\n",
- "response = request_get(url)\n",
- "display(response)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "96bfcd9c-27e8-4be4-a680-7553d908790e",
- "metadata": {},
- "outputs": [],
- "source": [
- "#ATLAS CREATE CONCEPT SET\n"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.9.7"
- },
- "toc-showtags": false
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/run.sh b/run.sh
deleted file mode 100644
index b64772d..0000000
--- a/run.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#! /usr/bin/bash
-
-version="V3_2_10"
-previous="V3_2_9"
-
-python main.py -r2 PHEN_assign_v3.json CONC_summary_working.xlsx
-mv output/MELD_concepts_read.csv output/${version}_MELD_concepts_readv2.csv
-
-python main.py -i PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-mv output/MELD_concepts_read.csv output/${version}_MELD_icd10_no_translate.csv
-
-python main.py -s PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-mv output/MELD_concepts_read.csv output/${version}_MELD_snomed_no_translate.csv
-
-# python main.py -o PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-# mv output/MELD_concepts_read.csv output/${version}_MELD_opcs4_no_translate.csv
-
-python main.py -a PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-mv output/MELD_concepts_read.csv output/${version}_MELD_atc_no_translate.csv
-
-# python main.py -m PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-# mv output/MELD_concepts_read.csv output/${version}_MELD_med_no_translate.csv
-
-mv output/MELD_errors.csv output/${version}_MELD_errors.csv
-
-
-#Generate Report
-rm concepts-output/MELD-report.md
-python report.py PHEN_assign_v3.json CONC_summary_working.xlsx codes/ concepts-output/MELD-report.md ${version} ${previous}
-
-#Divide Concepts to Output Repo
-rm -rf concepts-output/readv2/*
-rm -rf concepts-output/icd10/*
-rm -rf concepts-output/snomed/*
-rm -rf concepts-output/atc/*
-python publish.py output/${version}_MELD_concepts_readv2.csv concepts-output/readv2/
-python publish.py output/${version}_MELD_icd10_no_translate.csv concepts-output/icd10/
-python publish.py output/${version}_MELD_snomed_no_translate.csv concepts-output/snomed/
-python publish.py output/${version}_MELD_atc_no_translate.csv concepts-output/atc/
-cp output/${version}_MELD_errors.csv concepts-output/${version}_MELD_errors.csv
-
-# Show Changes in Output repo (should be same as report)
-cd concepts-output
-git diff --stat
\ No newline at end of file
--
GitLab