diff --git a/.gitignore b/.gitignore
index 69fafb5db55832f139fe4cec5d3941698d4630c0..d125b2c8e9304f21350e42fe47a4c785f8d2208c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,12 +13,8 @@ __pycache__
~$*
# Build
-output/
-concepts-output/
-archive/
-maps/*
-concepts-new/
-codes/
+medcoder/resources/maps/*
+medcoder/resources/codes/*
# temporary
script
diff --git a/img/nihr-logo-1200-375.jpg b/img/nihr-logo-1200-375.jpg
deleted file mode 100644
index 8a29828ca03036c03f0c9045ba9576d33e324bb0..0000000000000000000000000000000000000000
Binary files a/img/nihr-logo-1200-375.jpg and /dev/null differ
diff --git a/medcoder/publish.py b/medcoder/publish.py
deleted file mode 100644
index f39a2316fa845dd0f112abdee16a2fa6fbf22da9..0000000000000000000000000000000000000000
--- a/medcoder/publish.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import os
-import pandas as pd
-import argparse
-
-def main(config):
- #Load Output Concepts CSV File
- if config["concepts"].endswith(".csv"):
- df = pd.read_csv(config["concepts"], dtype=str)
- else:
- raise Exception("Concepts file must be '.csv' filetype")
-
- for name, concept in df.groupby("CONCEPT_SET"):
- concept = concept.sort_values(by="CONCEPT") #sort rows
- concept = concept.dropna(how='all', axis=1) #remove empty cols
- concept = concept.reindex(sorted(concept.columns), axis=1) #sort cols alphabetically
-
- concept.to_csv(os.path.join(config["output"], str(name)+".csv"), #save to csv
- index=False )
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description="Script divides single CSV file into one CSV per concept",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
- parser.add_argument("concepts", help="Output Concepts CSV file")
- parser.add_argument("output", help="Output Folder")
- # parser.add_argument("version", help="Version of output")
-
- args = parser.parse_args()
- config = vars(args)
- main(config)
diff --git a/medcoder/report.py b/medcoder/report.py
deleted file mode 100644
index d4069c33b12e04ae0ddb91fb22f4478bb027a736..0000000000000000000000000000000000000000
--- a/medcoder/report.py
+++ /dev/null
@@ -1,175 +0,0 @@
-import json
-import os
-import pandas as pd
-import numpy as np
-import argparse
-
-#Get all Files in JSON
-def get_json_files(folders):
- out = []
- for folder in folders:
- if "files" in folder:
- for file in folder["files"]:
- file_path = folder["folder"]+"/"+file["file"]
- if "concept_set" in file:
- for concept in file["concept_set"]:
- out.append({"json_concept":concept, "filepath":file_path, "json_code_types":list(file["columns"].keys())})
- elif "concept_set_categories" in file:
- for code, concept in file["concept_set_categories"].items():
- out.append({"json_concept":concept[0], "filepath":file_path, "json_code_types":list(file["columns"].keys())})
- else:
- out.append({"json_concept":None, "filepath":file_path})
-
- out = pd.DataFrame(out)
- out["filepath"] = out["filepath"].astype(str)
- return out
-
-
-#Get all Files Excel Summary
-def get_excel_files(out2):
- out2 = out2[["CONCEPT NAME ", "CODING LIST", "AGREED", "FUNCTION"]].loc[1:] #select relevant columns
-
- #Filter Concepts in use
- out2 = out2[out2["AGREED"] == "USE"] #remove deprecated concepts
- out2 = out2[out2["FUNCTION"] == "QUERY BY CODING LIST"] #remove deprecated concepts
- out2 = out2.drop(['AGREED', 'FUNCTION'], axis=1)
-
- #Get filepaths
- out2["CODING LIST"] = out2["CODING LIST"].str.split(",") #split by ,
- out2 = out2.explode("CODING LIST") #one row per file
- out2["CODING LIST"] = out2["CODING LIST"].str.strip()
- out2["CODING LIST"] = out2["CODING LIST"].str.replace("https://git.soton.ac.uk/meld/meldb-external/phenotype/-/tree/main/", "")
- out2["CODING LIST"] = out2["CODING LIST"].str.replace("%20", " ")
-
- out2 = out2.rename(columns={"CONCEPT NAME ":"excel_concept", "CODING LIST":"filepath"})
- return out2
-
-
-#Get all Files in /codes
-def get_code_files(path_codes):
- all_files = []
- for root, dirs, files in os.walk(path_codes, topdown=False):
- for name in files:
- if ".ipynb_checkpoint" not in root: #exclude notebook checkpoints
- if name.endswith(".csv") or name.endswith(".xlsx") or name.endswith(".dta"): #exclude non-data files
- all_files.append(os.path.join(root, name))
- all_files = pd.DataFrame(all_files)
- all_files = all_files.rename(columns={0:"filepath"})
- all_files["filepath"] = all_files["filepath"].astype(str)
- return all_files
-
-
-
-def test_concept_def(config, report, folders, summary):
- report.write("## Check Concept Definitions")
- out = get_json_files(folders)
- out2 = get_excel_files(summary)
- all_files = get_code_files(config["codes"])
-
- #Merge all into single table
- outs = pd.merge(all_files, out, how="outer", on="filepath")
- outs = pd.merge(outs, out2, how="outer", on="filepath")
-
- report.write("\n\nCode source files:\n")
- report.write("- {} total files\n\n".format(len(all_files)))
-
- report.write("\n\nJSON concepts:\n")
- report.write("- {} unique concepts\n".format(len(out["filepath"].unique())))
- missing = outs[outs["json_concept"].isna() & outs["excel_concept"].notna()]
- if len(missing) > 0:
- report.write("- Missing from JSON\n")
- for id, row in missing.iterrows():
- report.write("\t - ❌ {} {}\n\n".format(row["filepath"], row["excel_concept"]))
-
- report.write("\n\nEXCEL concepts:\n")
- report.write("- {} unique concepts\n".format(len(out2["filepath"].unique())))
- missing = outs[outs["json_concept"].notna() & outs["excel_concept"].isna()]
- if len(missing) > 0:
- report.write("- Missing from EXCEL\n")
- for id, row in missing.iterrows():
- report.write("\t - ❌ {} {} {}\n\n".format(row["filepath"], row["json_concept"], row["json_code_types"]))
-
-def get_output_files(version):
- output_files = [f"output/{version}_MELD_concepts_readv2.csv",
- f"output/{version}_MELD_snomed_no_translate.csv",
- f"output/{version}_MELD_icd10_no_translate.csv",
- # f"output/{version}_MELD_med_no_translate.csv",
- f"output/{version}_MELD_atc_no_translate.csv"
- ]
- error_file = f"output/{version}_MELD_errors.csv"
- return output_files, error_file
-
-def test_concept_changes(config, report):
- version_1 = config["previous"]
- version_2 = config["version"]
- output1, err1 = get_output_files(version_1)
- output2, err2 = get_output_files(version_2)
-
- report.write(f"\n\n## Compare Concepts {version_1} to {version_2}\n\n")
-
- for out1, out2 in zip(output1, output2):
- report.write(f"`{out1}` to `{out2}`\n")
-
- df1 = pd.read_csv(out1)
- df1 = df1[["CONCEPT","CONCEPT_SET"]].groupby("CONCEPT_SET").count()
- df2 = pd.read_csv(out2)
- df2 = df2[["CONCEPT","CONCEPT_SET"]].groupby("CONCEPT_SET").count()
-
- #Added/Removed Concepts
- report.write("- Removed Concepts {}\n".format(list(set(df1.index) - set(df2.index))))
- report.write("- Added Concepts {}\n".format(list(set(df2.index) - set(df1.index))))
-
- #Changed Concepts
- diff = df2 - df1 #diff in counts
- diff = diff[(~(diff["CONCEPT"] == 0.0)) & diff["CONCEPT"].notna()] #get non-zero counts
- s = "\n"
- for concept, row in diff.iterrows():
- s += "\t - {} {}\n".format(concept, row["CONCEPT"])
- report.write("- Changed Concepts {}\n\n".format(s))
-
-# ✅ ❌
-
-def main(config):
- #Load Report
- if config["report"].endswith(".md"):
- report = open(config["report"], 'a')
- else:
- raise Exception("Unsupported filetype provided for source file")
-
- #Load Mapping File
- if config["map"].endswith(".json"):
- folders = json.load(open(config["map"],'rb'))
- else:
- raise Exception("Unsupported filetype provided for source file")
-
- #Load Excel Summary File
- if config["summary"].endswith(".xlsx"):
- summary = pd.read_excel(config["summary"], sheet_name="CONCEPT_TRACKING", dtype=str)
- else:
- raise Exception("Unsupported filetype provided for summary file")
-
- #Title with version
- report.write("\n\n# Report {} \n\n".format(config["version"]))
-
- #Compare JSON Mapping with Excel Summary
- test_concept_def(config, report, folders, summary)
-
- #Changes in Concept Codes between versions
- test_concept_changes(config, report)
-
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(description="Script performs testing and generates report for output files",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
- parser.add_argument("map", help="Concept/Phenotype Assignment File (json)")
- parser.add_argument("summary", help="Summary working excel document")
- parser.add_argument("codes", help="Folder containing all code source files")
- parser.add_argument("report", help="Output Markdown file containing report")
- parser.add_argument("version", help="Version of output")
- parser.add_argument("previous", help="Previous version of output")
-
- args = parser.parse_args()
- config = vars(args)
- main(config)
\ No newline at end of file
diff --git a/process_codes_WP.ipynb b/process_codes_WP.ipynb
deleted file mode 100644
index 68e26be260fd3eece0d460dc54adf53d712bb8cc..0000000000000000000000000000000000000000
--- a/process_codes_WP.ipynb
+++ /dev/null
@@ -1,1288 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8c8f4cdf-04a5-4762-895e-6555781a1f05",
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import numpy as np\n",
- "import json"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c5786d78-7dc2-4f02-ad21-cee95e473823",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Ho generate JSON"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "0292dc90-e31a-4724-8536-d0b55533aaef",
- "metadata": {},
- "outputs": [],
- "source": [
- "#List v4 to json\n",
- "\n",
- "df = pd.read_excel(\"PHEN_code_lists_sources_V4.xlsx\", sheet_name=\"ho\", dtype=str)\n",
- "# df = df.sort_values(by=\"mapped_condition\")\n",
- "\n",
- "def json_file_template(file, cons, types, metadata):\n",
- " concepts = \"\"\n",
- " for concept in cons:\n",
- " concepts += f'\"{concept}\", '\n",
- " concepts = concepts[:-2] #remove last ,\n",
- " \n",
- " type_str = \"\"\n",
- " for k, v in types.items():\n",
- " type_str += f'\"{k}\":\"{v}\", '\n",
- " type_str = type_str[:-2]\n",
- " \n",
- " meta_str = '\"metadata\":['\n",
- " for v in metadata:\n",
- " meta_str += f'\"{v}\", '\n",
- " meta_str = meta_str[:-2]\n",
- " meta_str = meta_str + \"]\"\n",
- " \n",
- " return '''\n",
- " { \n",
- " \\\"file\\\":\\\"'''+file+'''\",\n",
- " \\\"columns\\\":{\n",
- " '''+type_str+''',\n",
- " '''+meta_str+'''\n",
- " },\n",
- " \\\"meldb_phenotypes\\\":['''+concepts+''']\n",
- " },'''\n",
- "\n",
- "out = '\"files\":['\n",
- "folder = \"codes/GitHub_TG_repository/\"\n",
- "for file, grp in df.groupby(\"mapped_condition\"):\n",
- " file = file.replace(\"%20\", \" \") \n",
- " \n",
- " for ext in [\"_CPRD_GOLD.csv\", \"_CPRD_AURUM.csv\", \"_IMRD.csv\"]:\n",
- " path = file+\"/\"+file+ext\n",
- " if os.path.isfile(folder+path):\n",
- " out+= json_file_template(path, grp[\"meldb_condition\"],\n",
- " types={\n",
- " \"read2_code\":\"READ_CODE\",\n",
- " \"snomed_code\":\"SNOMED_CT_CODE\",\n",
- " # \"med_code\":\"MEDICAL_CODE_ID\",\n",
- " },\n",
- " metadata = [\"DESCRIPTION\"]\n",
- " )\n",
- " else:\n",
- " print(\"NOT FILE\", folder+path)\n",
- " for ext in [\"_ICD10.csv\"]:\n",
- " path = file+\"/\"+file+ext\n",
- " if os.path.isfile(folder+path):\n",
- " out+= json_file_template(path, grp[\"meldb_condition\"],\n",
- " types={\n",
- " \"icd10_code\":\"READ_CODE\",\n",
- " \"snomed_code\":\"SNOMED_CT_CODE\",\n",
- " # \"icd10_code\":\"MEDICAL_CODE_ID\",\n",
- " },\n",
- " metadata = [\"DESCRIPTION\"]\n",
- " )\n",
- " else:\n",
- " print(\"NOT FILE\", folder+path)\n",
- " \n",
- " \n",
- " \n",
- " # out+= json_file_template(file+\"/\"+file+\"_CPRD_AURUM.csv\", grp[\"meldb_condition\"])\n",
- " # out+= json_file_template(file+\"/\"+file+\"_ICD10.csv\", grp[\"meldb_condition\"])\n",
- " # out+= json_file_template(file+\"/\"+file+\"_IMRD.csv\", grp[\"meldb_condition\"])\n",
- "\n",
- " # out += f' \"{file}/{file}_CPRD_GOLD.csv\":[{conds}],\\n'\n",
- " # out += f' \"{file}/{file}_CPRD_AURUM.csv\":[{conds}],\\n'\n",
- " # out += f' \"{file}/{file}_ICD10.csv\":[{conds}],\\n'\n",
- " # out += f' \"{file}/{file}_IMRD.csv\":[{conds}],\\n'\n",
- " \n",
- "out = out[:-1] #remove last ,\n",
- "out += \"\\n]\"\n",
- "out = out.replace(\"%20\", \" \") \n",
- "print(out)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f155b635-b459-4aff-81b2-e065fc223858",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d040eda5-4028-4047-834c-7315e307e415",
- "metadata": {},
- "outputs": [],
- "source": [
- "df = pd.read_parquet(\"maps/processed/icd10_code.parquet\")\n",
- "df\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "e0228ac9-8852-4818-b7f0-98429ca5229c",
- "metadata": {},
- "outputs": [],
- "source": [
- "code = [\"A00.0\", \"*00.0\"]\n",
- "code = pd.Series(code)\n",
- "print(code.isin(df[\"icd10_code\"]))\n",
- "print(code.isin(df[\"icd10_alt_code\"]))\n",
- "# print( )\n",
- "~(\n",
- " ~code.isin(df[\"icd10_code\"]) \n",
- " &\n",
- " ~code.isin(df[\"icd10_alt_code\"])\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "18efcacd-45f0-4341-86cc-d8e2e584350c",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Analyse the JSON file"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "85dc197b-451e-4fa9-a53b-e6770c132123",
- "metadata": {},
- "outputs": [],
- "source": [
- "import json\n",
- "import os\n",
- "\n",
- "path_json = \"../concepts/PHEN_assign_v3.json\"\n",
- "\n",
- "#Load JSON Concept Definitions\n",
- "mapping = json.load(open(path_json,'rb'))\n",
- "summary_config = mapping[\"concept_sets\"][\"concept_set\"]\n",
- "summary_df = pd.DataFrame(summary_config) #change to dataframe\n",
- "\n",
- "summary_df = summary_df.join(pd.json_normalize(summary_df[\"metadata\"])) #metadata to columns\n",
- "summary_df = summary_df.drop(columns=[\"metadata\"])\n",
- "summary_df = summary_df.rename(columns={\"concept_set_name\":\"CONCEPT_SET\"})\n",
- "summary_df = summary_df.drop_duplicates() #remove duplicates\n",
- " \n",
- "summary_df\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4c9b6b3f-08aa-4f61-b9b2-44a24b5d00a0",
- "metadata": {},
- "outputs": [],
- "source": [
- "import json\n",
- "import os\n",
- "\n",
- "path_json = \"PHEN_assign_v3.json\"\n",
- "path_excel = \"PHEN_summary_working.xlsx\"\n",
- "path_codes = \"codes/\"\n",
- "\n",
- "#Get all Files in JSON\n",
- "def get_json_files(path_json):\n",
- " folders = json.load(open(path_json,'rb'))\n",
- " out = []\n",
- " for folder in folders:\n",
- " if \"files\" in folder:\n",
- " for file in folder[\"files\"]:\n",
- " file_path = folder[\"folder\"]+\"/\"+file[\"file\"]\n",
- " if \"meldb_phenotypes\" in file:\n",
- " for concept in file[\"meldb_phenotypes\"]:\n",
- " out.append({\"json_concept\":concept, \"filepath\":file_path, \"json_code_types\":list(file[\"columns\"].keys())})\n",
- " elif \"meldb_phenotypes_categories\" in file:\n",
- " for code, concept in file[\"meldb_phenotypes_categories\"].items():\n",
- " out.append({\"json_concept\":concept[0], \"filepath\":file_path, \"json_code_types\":list(file[\"columns\"].keys())})\n",
- " else:\n",
- " out.append({\"json_concept\":None, \"filepath\":file_path})\n",
- "\n",
- " out = pd.DataFrame(out)\n",
- " out[\"filepath\"] = out[\"filepath\"].astype(str)\n",
- " return out\n",
- "out = get_json_files(path_json)\n",
- "\n",
- "#Get all Files Excel Summary\n",
- "def get_excel_files(path_excel):\n",
- " path_excel = \"PHEN_summary_working.xlsx\"\n",
- " out2 = pd.read_excel(path_excel)\n",
- " out2 = out2[[\"CONCEPT NAME \", \"CODING LIST\", \"AGREED\", \"FUNCTION\"]].loc[1:] #select relevant columns\n",
- "\n",
- " #Filter Concepts in use\n",
- " out2 = out2[out2[\"AGREED\"] == \"USE\"] #remove deprecated concepts\n",
- " out2 = out2[out2[\"FUNCTION\"] == \"QUERY BY CODING LIST\"] #remove deprecated concepts\n",
- " out2 = out2.drop(['AGREED', 'FUNCTION'], axis=1)\n",
- "\n",
- " #Get filepaths\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.split(\",\") #split by ,\n",
- " out2 = out2.explode(\"CODING LIST\") #one row per file\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.strip()\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.replace(\"https://git.soton.ac.uk/meld/meldb-external/phenotype/-/tree/main/\", \"\")\n",
- " out2[\"CODING LIST\"] = out2[\"CODING LIST\"].str.replace(\"%20\", \" \")\n",
- "\n",
- " out2 = out2.rename(columns={\"CONCEPT NAME \":\"excel_concept\", \"CODING LIST\":\"filepath\"})\n",
- " return out2\n",
- "out2 = get_excel_files(path_excel)\n",
- "\n",
- "#Get all Files in /codes\n",
- "def get_code_files(path_codes):\n",
- " all_files = []\n",
- " for root, dirs, files in os.walk(path_codes, topdown=False):\n",
- " for name in files:\n",
- " if \".ipynb_checkpoint\" not in root: #exclude notebook checkpoints\n",
- " if name.endswith(\".csv\") or name.endswith(\".xlsx\") or name.endswith(\".dta\"): #exclude non-data files\n",
- " all_files.append(os.path.join(root, name)) \n",
- " all_files = pd.DataFrame(all_files)\n",
- " all_files = all_files.rename(columns={0:\"filepath\"})\n",
- " all_files[\"filepath\"] = all_files[\"filepath\"].astype(str)\n",
- " return all_files\n",
- "all_files = get_code_files(path_codes)\n",
- "\n",
- "\n",
- "print(\"ALL FILES\", len(all_files), len(all_files[\"filepath\"].unique()))\n",
- "print(\"JSON CONCEPTS\", len(out), len(out[\"filepath\"].unique()))\n",
- "print(\"EXCEL CONCEPTS\", len(out2), len(out2[\"filepath\"].unique()))\n",
- "\n",
- "outs = pd.merge(all_files, out, how=\"outer\", on=\"filepath\")\n",
- "outs = pd.merge(outs, out2, how=\"outer\", on=\"filepath\")\n",
- "print(len(outs), len(outs[\"filepath\"].unique()))\n",
- "outs.to_csv(\"output/MELD_file_to_concept.csv\", index=False)\n",
- "\n",
- "# display(outs[ outs[\"concept\"].isna()])\n",
- "\n",
- "# display(out ) "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f8e70c33-c869-46f8-953e-f6b52992cfbb",
- "metadata": {},
- "outputs": [],
- "source": [
- "display(\"JSON MISSING\", outs[outs[\"json_concept\"].isna() & outs[\"excel_concept\"].notna()])\n",
- "display(\"EXCEL MISSING\", outs[outs[\"json_concept\"].notna() & outs[\"excel_concept\"].isna()])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "9d84465f-f064-4df2-b0e4-2dfb217aea21",
- "metadata": {},
- "outputs": [],
- "source": [
- "f = open('concepts-output/MELD-report.md', 'a') as f:\n",
- " f.write(\n",
- " \"\"\"\n",
- "# Report\n",
- "- One thing\n",
- "- Two thing\n",
- "- Three thing\n",
- " \"\"\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7f7fc771-e406-42c7-8a09-16a20b5298f5",
- "metadata": {},
- "outputs": [],
- "source": [
- "total_length = 0\n",
- "for file in all_files[\"filepath\"]:\n",
- " if file.endswith(\".csv\"):\n",
- " df_file = pd.read_csv(file)\n",
- " total_length += len(df_file)\n",
- " elif file.endswith(\".xlsx\"):\n",
- " df_file = pd.read_excel(file)\n",
- " total_length += len(df_file)\n",
- " elif file.endswith(\".dta\"):\n",
- " df_file = pd.read_stata(file)\n",
- " total_length += len(df_file)\n",
- "total_length\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08a9c565-28d6-46ee-9fa8-6fa0ee28a4d5",
- "metadata": {},
- "outputs": [],
- "source": [
- "#turn filepaths into gitlab links\n",
- "outs2 = outs.copy()\n",
- "outs2[\"filepath\"] = \"https://git.soton.ac.uk/meld/meldb-external/phenotype/-/tree/main/\"+outs2[\"filepath\"].str.replace(\" \", \"%20\")\n",
- "\n",
- "#Groupby concepts and concat filepaths\n",
- "outs2 = outs2.groupby(\"concept\")[\"filepath\"].apply(', '.join).reset_index()\n",
- "outs2 = outs2.sort_values(by=[\"concept\"])\n",
- "outs2\n",
- "outs2.to_csv(\"output/MELD_GitLab_link_to_concept.csv\", index=False)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "357bb84c-90c2-4b5f-95c0-443191783a7f",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Analyse Output Files"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7d3f9cb7-be86-4f1f-92f6-991094eb7bb7",
- "metadata": {},
- "outputs": [],
- "source": [
- "version = \"V2_2_2\"\n",
- "output_files = [f\"output/{version}_MELD_concepts_readv2.csv\",\n",
- " f\"output/{version}_MELD_snomed_no_translate.csv\",\n",
- " f\"output/{version}_MELD_icd10_no_translate.csv\",\n",
- " # f\"output/{version}_MELD_med_no_translate.csv\",\n",
- " f\"output/{version}_MELD_atc_no_translate.csv\"\n",
- " ]\n",
- "error_file = f\"output/{version}_MELD_errors.csv\"\n",
- "\n",
- "for output_file in output_files:\n",
- " print(\"---\"*3,output_file,\"---\"*3,)\n",
- " df = pd.read_csv(output_file)\n",
- " # df[\"MELDB_concept\"].loc[df[\"CONCEPT TYPE\"].isna()]\n",
- " print(\"MELDB missing concepts \", len(df[df[\"CONCEPT TYPE\"].isna()]))\n",
- " if df[\"code\"].dtype == \"object\":\n",
- " print(\"Chars present:\", np.sort(df[\"code\"].apply(lambda x : set(x)).explode().unique()))\n",
- " \n",
- "# len(df[\"MELDB_concept\"].unique())\n",
- "\n",
- "print(\"---\"*3,error_file,\"---\"*3,)\n",
- "df = pd.read_csv(error_file)\n",
- "df = df.drop_duplicates()\n",
- "df[\"CODE_TYPE\"].value_counts()\n",
- "# for i, row in df.drop_duplicates().iterrows():\n",
- "# print(row[\"CODE\"], row[\"CODE_TYPE\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08e0ecc1-9271-48c3-9c5b-094800072906",
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_output_files(version):\n",
- " output_files = [f\"output/{version}_MELD_concepts_readv2.csv\",\n",
- " f\"output/{version}_MELD_snomed_no_translate.csv\",\n",
- " f\"output/{version}_MELD_icd10_no_translate.csv\",\n",
- " # f\"output/{version}_MELD_med_no_translate.csv\",\n",
- " f\"output/{version}_MELD_atc_no_translate.csv\"\n",
- " ]\n",
- " error_file = f\"output/{version}_MELD_errors.csv\"\n",
- " return output_files, error_file\n",
- "\n",
- "# version_1 = \"V1_0_0\"\n",
- "version_1 = \"V2_1_4\"\n",
- "version_2 = \"V2_2_3\"\n",
- "output1, err1 = get_output_files(version_1)\n",
- "output2, err2 = get_output_files(version_2)\n",
- "\n",
- "print(\"## Compare Concepts\", version_1, \"to\", version_2)\n",
- "\n",
- "for out1, out2 in zip(output1, output2):\n",
- " print(out1, out2 )\n",
- " df1 = pd.read_csv(out1)\n",
- " df1 = df1[[\"code\",\"MELDB_concept\"]].groupby(\"MELDB_concept\").count()\n",
- " df2 = pd.read_csv(out2)\n",
- " df2 = df2[[\"code\",\"MELDB_concept\"]].groupby(\"MELDB_concept\").count()\n",
- " \n",
- " #Added/Removed Concepts\n",
- " print(\"- Removed Concepts\", list(set(df1.index) - set(df2.index)))\n",
- " print(\"- Added Concepts\", list(set(df2.index) - set(df1.index)))\n",
- " \n",
- " #Changed Concepts\n",
- " diff = df2 - df1 #diff in counts \n",
- " diff = diff[(~(diff[\"code\"] == 0.0)) & diff[\"code\"].notna()] #get non-zero counts\n",
- " s = \"\\n\"\n",
- " for concept, row in diff.iterrows():\n",
- " s += \"\\t - {} {}\\n\".format(concept, row[\"code\"])\n",
- " print(\"- Changed Concepts\", s)\n",
- "\n",
- "\n",
- "# for output_file in output_files:\n",
- "# print(\"---\"*3,output_file,\"---\"*3,)\n",
- "# df = pd.read_csv(output_file)\n",
- "# # df[\"MELDB_concept\"].loc[df[\"CONCEPT TYPE\"].isna()]\n",
- "# print(\"MELDB missing concepts \", len(df[df[\"CONCEPT TYPE\"].isna()]))\n",
- "# if df[\"code\"].dtype == \"object\":\n",
- "# print(\"Chars present:\", np.sort(df[\"code\"].apply(lambda x : set(x)).explode().unique()))\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "cc60c137-5a85-4155-af6b-6796f8c05980",
- "metadata": {},
- "outputs": [],
- "source": [
- "import glob\n",
- "import os\n",
- "import pandas as pd\n",
- "\n",
- "df = pd.read_csv(\"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/PHEN_summary_working.csv\")\n",
- "df = df.set_index(\"#\")\n",
- "\n",
- "for vocab in [\"atc\", \"icd10\", \"readv2\", \"snomed\"]:\n",
- " df[vocab.upper()] = \"\"\n",
- "\n",
- " for file in glob.glob(f\"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/{vocab}/*.csv\"):\n",
- " concept_set = os.path.basename(file)[:-4]\n",
- " row_index = df[df[\"CONCEPT NAME \"] == concept_set].index[0]\n",
- "\n",
- " df.loc[row_index, vocab.upper()] = \"YES\"\n",
- "\n",
- "df = df.drop(columns=[\"READv2_CODE\", \"ICD10_CODE\"])\n",
- "df.to_csv(\"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/PHEN_summary_working_labelled.csv\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e5c4291f-847b-4c82-976e-bd5b3a7b6bcc",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### Mappings"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08e34750-413c-469e-bcb8-e71bb188ff42",
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "#NHS Read Browser\n",
- "import simpledbf\n",
- "import pandas as pd\n",
- "\n",
- "\n",
- "#r2 only\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/ANCESTOR.DBF').to_dataframe()\n",
- "df = pd.concat([df['READCODE'], df['DESCENDANT']])\n",
- "df = pd.DataFrame(df.drop_duplicates())\n",
- "df = df.rename(columns={0:\"read2_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> atc\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/ATC.DBF').to_dataframe()\n",
- "df = df[[\"READCODE\", \"ATC\"]]\n",
- "df = df.rename(columns={\"READCODE\":\"read2_code\", \"ATC\":\"atc_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code_to_atc_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> icd10\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/ICD10.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read2_code\", \"TARG_CODE\":\"icd10_code\"})\n",
- "df = df[~df[\"icd10_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read2_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read2_code_to_icd10_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> opcs4\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V2/OPCS4V3.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read2_code\", \"TARG_CODE\":\"opcs4_code\"})\n",
- "df = df[~df[\"opcs4_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read2_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read2_code_to_opcs4_code.parquet\", index=False)\n",
- "\n",
- "#r3 only\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/ANCESTOR.DBF').to_dataframe()\n",
- "df = pd.concat([df['READCODE'], df['DESCENDANT']])\n",
- "df = pd.DataFrame(df.drop_duplicates())\n",
- "df = df.rename(columns={0:\"read3_code\"})\n",
- "df.to_parquet(\"maps/processed/read3_code.parquet\", index=False)\n",
- "\n",
- "#r3 -> icd10\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/ICD10.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read3_code\", \"TARG_CODE\":\"icd10_code\"})\n",
- "df = df[~df[\"icd10_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read3_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_icd10_code.parquet\", index=False)\n",
- "\n",
- "#r3 -> icd9\n",
- "# dbf = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/ICD9V3.DBF')\n",
- "\n",
- "#r3 -> opcs4\n",
- "df = simpledbf.Dbf5('maps/nhs_readbrowser_25.0.0_20180401000001/Standard/V3/OPCS4V3.DBF').to_dataframe()\n",
- "df = df[[\"READ_CODE\", \"TARG_CODE\"]]\n",
- "df = df.rename(columns={\"READ_CODE\":\"read3_code\", \"TARG_CODE\":\"opcs4_code\"})\n",
- "df = df[~df[\"opcs4_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df = df[~df[\"read3_code\"].str.match(\"^.*-.*$\")] #remove codes with '-'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_opcs4_code.parquet\", index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5fe95638-1f25-45f3-803c-2fff74a2a4fd",
- "metadata": {},
- "outputs": [],
- "source": [
- "#NHS Data Migrations\n",
- "\n",
- "#r2 only\n",
- "# df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/rctcremap_uk_20200401000001.txt', sep='\\t')\n",
- "\n",
- "#r3 only\n",
- "# df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/ctv3cremap_uk_20200401000001.txt', sep='\\t')\n",
- "\n",
- "#snomed only\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/sctcremap_uk_20200401000001.txt', sep='\\t')\n",
- "df = df[[\"SCT_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"SCT_CONCEPTID\":\"snomed_code\"})\n",
- "df = df.drop_duplicates()\n",
- "df = df.astype(str)\n",
- "df.to_parquet(\"maps/processed/snomed_code.parquet\", index=False)\n",
- "\n",
- "#r2 -> r3\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/rctctv3map_uk_20200401000001.txt', sep='\\t')\n",
- "df = df[[\"V2_CONCEPTID\", \"CTV3_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"V2_CONCEPTID\":\"read2_code\",\n",
- " \"CTV3_CONCEPTID\":\"read3_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code_to_read3_code.parquet\", index=False)\n",
- "\n",
- "#r3->r2\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/ctv3rctmap_uk_20200401000002.txt', sep='\\t')\n",
- "df = df[[\"CTV3_CONCEPTID\", \"V2_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"CTV3_CONCEPTID\":\"read3_code\", \n",
- " \"V2_CONCEPTID\":\"read2_code\"})\n",
- "df = df.drop_duplicates()\n",
- "df = df[~df[\"read2_code\"].str.match(\"^.*_.*$\")] #remove r2 codes with '_'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_read2_code.parquet\", index=False)\n",
- "\n",
- "\n",
- "#r2 -> snomed\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/rcsctmap2_uk_20200401000001.txt', sep='\\t', dtype=str)\n",
- "df = df[[\"ReadCode\", \"ConceptId\"]]\n",
- "df = df.rename(columns={\"ReadCode\":\"read2_code\",\n",
- " \"ConceptId\":\"snomed_code\"})\n",
- "df.to_parquet(\"maps/processed/read2_code_to_snomed_code.parquet\", index=False)\n",
- "\n",
- "\n",
- "#r3->snomed\n",
- "df = pd.read_csv('maps/nhs_datamigration_29.0.0_20200401000001/Mapping Tables/Updated/Clinically Assured/ctv3sctmap2_uk_20200401000001.txt', sep='\\t')\n",
- "df = df[[\"CTV3_TERMID\", \"SCT_CONCEPTID\"]]\n",
- "df = df.rename(columns={\"CTV3_TERMID\":\"read3_code\",\n",
- " \"SCT_CONCEPTID\":\"snomed_code\"})\n",
- "df[\"snomed_code\"] = df[\"snomed_code\"].astype(str)\n",
- "df = df[~df[\"snomed_code\"].str.match(\"^.*_.*$\")] #remove snomed codes with '_'\n",
- "df.to_parquet(\"maps/processed/read3_code_to_snomed_code.parquet\", index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "267fa1cc-5159-48c4-9eee-19af5039d627",
- "metadata": {},
- "outputs": [],
- "source": [
- "#OPCS410 Data Files\n",
- "df = pd.read_csv(\"maps/OPCS410 Data files txt/OPCS410 CodesAndTitles Nov 2022 V1.0.txt\", sep='\\t', dtype=str, header=None)\n",
- "df = df.rename(columns={0:\"opcs4_code\", 1:\"description\"})\n",
- "df.to_parquet(\"maps/processed/opcs4_code.parquet\", index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "01d046fd-69af-44f3-acad-5d0edef3f745",
- "metadata": {},
- "outputs": [],
- "source": [
- "#ICD10_edition5\n",
- "df = pd.read_xml(\"maps/ICD10_Edition5_XML_20160401/Content/ICD10_Edition5_CodesAndTitlesAndMetadata_GB_20160401.xml\",)\n",
- "df = df[[\"CODE\", \"ALT_CODE\", \"DESCRIPTION\"]]\n",
- "df = df.rename(columns={\"CODE\":\"icd10_code\",\n",
- " \"ALT_CODE\":\"icd10_alt_code\",\n",
- " \"DESCRIPTION\":\"description\"\n",
- " })\n",
- "df.to_parquet(\"maps/processed/icd10_code.parquet\", index=False)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "36630e24-f56c-48e1-8ecf-4ccd2b41eaea",
- "metadata": {},
- "outputs": [],
- "source": [
- "code1=\"read2_code\"\n",
- "code2=\"icd10_code\"\n",
- "df_map = pd.read_parquet(f\"maps/processed/{code1}_to_{code2}.parquet\")\n",
- "\n",
- "codes=df_map[\"read2_code\"].iloc[:5]\n",
- "\n",
- "pd.merge(codes, df_map, how='left')[code2]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "9787adeb-8507-488b-9a91-b8df3fbbe21e",
- "metadata": {},
- "outputs": [],
- "source": [
- "#CPRD Code Browser\n",
- "df = pd.read_csv('maps/CPRD_CodeBrowser_202211_Aurum/CPRDAurumMedical.txt', sep='\\t')\n",
- "df = df[[\"MedCodeId\", \"CleansedReadCode\", \"SnomedCTConceptId\"]]\n",
- "df = df.rename(columns={\"MedCodeId\":\"med_code\",\n",
- " \"CleansedReadCode\":\"read2_code\",\n",
- " \"SnomedCTConceptId\":\"snomed_code\"})\n",
- "\n",
- "# df = pd.read_csv('maps/CPRD_CodeBrowser_202211_Aurum/CPRDAurumProduct.txt', sep='\\t', dtype=str)\n",
- "\n",
- "# df = pd.read_csv('maps/CPRD_CodeBrowser_202211_GOLD/medical.txt', sep='\\t')\n",
- "# df = df.reset_index().iloc[:,[1,6]]\n",
- "# df = df.rename(columns={\"level_1\":\"read2_code\", \"20220523\":\"description\"})\n",
- "# df = pd.read_csv('maps/CPRD_CodeBrowser_202211_GOLD/product.txt', sep='\\t', dtype=str) #CANNOT OPEN\n",
- "\n",
- "df"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a968ffb1-4337-456b-8d20-419888b4044f",
- "metadata": {},
- "outputs": [],
- "source": [
- "#BNF\n",
- "\n",
- "df = pd.read_excel(\"maps/BNF Snomed Mapping data 20231215.xlsx\")\n",
- "df = df.astype(str)\n",
- "df = df.rename(columns={\"BNF Code\":\"bnf_code\",\n",
- " \"SNOMED Code\":\"snomed_code\"})\n",
- "df[[\"bnf_code\", \"snomed_code\"]].to_parquet(\"maps/processed/bnf_code_to_snomed_code.parquet\", index=False)\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c70b1ce2-0f41-4d02-ad17-6fc44bc3c6bf",
- "metadata": {},
- "outputs": [],
- "source": [
- "#BNF to Readv2 Merge\n",
- "df1 = pd.read_parquet(\"maps/processed/bnf_code_to_snomed_code.parquet\").astype(str)\n",
- "df2 = pd.read_parquet(\"maps/processed/read2_code_to_snomed_code.parquet\").astype(str)\n",
- "# df1.merge(df2, how=\"inner\", on=\"snomed_code\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d5d34237-02d4-4dea-8c20-5adaf337f6b5",
- "metadata": {},
- "outputs": [],
- "source": [
- "df1.merge(df2, how='inner', on='snomed_code')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b3166cf0-e4a5-43e0-aeac-78827427422e",
- "metadata": {},
- "outputs": [],
- "source": [
- ".astype(str).dtypes"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c0a766f9-7959-4a10-b58f-cd946a878b60",
- "metadata": {},
- "outputs": [],
- "source": [
- "df = pd.read_csv(\"../concepts/PHEN_summary_working.csv\")\n",
- "cols = list(df.columns)\n",
- "cols.remove('CONCEPT NAME ')\n",
- "cols.remove('AGREED')\n",
- "df = df.applymap(lambda x: str(x) if isinstance(x, (int, float)) else x) #change to int\n",
- "\n",
- "df_copy = df.rename(columns={\n",
- " \"CONCEPT NAME \":\"concept_set_name\",\n",
- " \"AGREED\":\"concept_set_status\"\n",
- "})\n",
- "df_copy[\"concept_set_status\"] = df_copy[\"concept_set_status\"].replace(\"USE\", \"AGREED\")\n",
- "df_copy = df_copy[[\"concept_set_name\", \"concept_set_status\"]]\n",
- "outs = df_copy.to_dict(orient='records')\n",
- "\n",
- "for i, out in enumerate(outs):\n",
- " out[\"metadata\"] = dict(df[cols].iloc[i])\n",
- " \n",
- "json.dumps(outs)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8a204a95-dc4c-4183-9ea7-f5c5e95e9087",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5ce1ab58-50b4-4c22-b72b-c698de6830f7",
- "metadata": {},
- "outputs": [],
- "source": [
- "import json"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f1ea81c6-d1db-408f-9d3a-b96f44efe21f",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "markdown",
- "id": "5eb544a3-9dd1-41e8-88c2-a808646c6112",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### OMOP Database"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c9e58e62-9e44-4d0c-9d8d-35c175c07e6c",
- "metadata": {},
- "outputs": [],
- "source": [
- "import sqlite3\n",
- "import csv\n",
- "import pandas as pd\n",
- "import os"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4f67c9a1-373f-4799-8a85-72767662d912",
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d0ecdf69-ee90-42c1-ad25-d8357b603d1b",
- "metadata": {},
- "outputs": [],
- "source": [
- "#IMPORT OMOP VOCABS\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\") # change to 'sqlite:///your_filename.db'\n",
- "folder_path = \"codes/vocabulary_download_v5_{9424944c-2b76-4127-8f05-f535e0f15e2a}_1731661390540\"\n",
- "\n",
- "# Check if the folder exists\n",
- "if not os.path.isdir(folder_path):\n",
- " raise Exception(f\"Error: The folder '{folder_path}' does not exist.\") \n",
- "\n",
- "# Iterate through files in the folder\n",
- "for filename in os.listdir(folder_path):\n",
- " if filename.endswith(\".csv\"): # Check if the file is a CSV\n",
- " file_path = os.path.join(folder_path, filename)\n",
- " try:\n",
- " print(f\"Reading file: {file_path}\")\n",
- " # Read the CSV file with the specified delimiter\n",
- " df = pd.read_csv(file_path, delimiter=\"\\t\", low_memory=False)\n",
- " table_name = os.path.splitext(os.path.basename(file_path))[0] #Get name of file\n",
- " \n",
- " #Export Table to sqlite db\n",
- " df.to_sql(table_name, conn, if_exists='replace', index=False)\n",
- " \n",
- " except Exception as e:\n",
- " raise Exception(f\"Error reading file {file_path}: {e}\")\n",
- "\n",
- "conn.commit()\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b9cafd0c-a3bd-408b-bca8-b0de2acde1cd",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Create a SQL connection to our SQLite database\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\")\n",
- "cur = conn.cursor()\n",
- "\n",
- "#Print ALL Columns in Table\n",
- "# table=\"CONCEPT_SET\"\n",
- "# cur.execute(f\"PRAGMA table_info({table});\")\n",
- "# print(pd.DataFrame(cur.fetchall()))\n",
- "\n",
- "#Print ALL TABLE NAMES\n",
- "# cur.execute(\"SELECT name FROM sqlite_master WHERE type='table' AND name=? ;\", (\"VOCABULARY\",))\n",
- "# print(cur.fetchone())\n",
- " \n",
- "cur.execute(\"SELECT vocabulary_id FROM VOCABULARY WHERE vocabulary_id=? ;\", (\"MELDB\",))\n",
- "print(cur.fetchone())\n",
- "\n",
- " \n",
- " \n",
- "#Print WHOLE TABLE\n",
- "# cur.execute('SELECT * FROM CONCEPT;')\n",
- "# cur.execute('SELECT * FROM CONCEPT WHERE standard_concept = \"C\";')\n",
- "# cur.execute('SELECT * FROM CONCEPT WHERE concept_code = \"119768002\" LIMIT 1;')\n",
- "# cur.execute('SELECT * FROM CONCEPT WHERE concept_code IN (\"119768002\", \"5905001\");')\n",
- "# cur.execute('SELECT DISTINCT VOCABULARY_ID FROM CONCEPT;')\n",
- "# df = pd.DataFrame(cur.fetchall())\n",
- "# print(list(df[0]))\n",
- "# display(df)\n",
- "# for row in :\n",
- " # print(row)\n",
- "\n",
- "\n",
- "\n",
- "#Get Header of Table\n",
- "# table=\"CONCEPT_CLASS\"\n",
- "# cur.execute(f\"SELECT * FROM {table} LIMIT 3;\")\n",
- "# print(cur.fetchall())\n",
- "\n",
- "#create meldb VOCABULARY\n",
- "# meldb_version='v3.2.10'\n",
- "# meldb_description = 'Multidisciplinary Ecosystem to study Lifecourse Determinants and Prevention of Early-onset Burdensome Multimorbidity'\n",
- "# meldb_reference = 'https://www.it-innovation.soton.ac.uk/projects/meldb'\n",
- "# df_test = pd.DataFrame([{\n",
- "# \"vocabulary_id\": 'MELDB',\n",
- "# \"vocabulary_name\": meldb_description,\n",
- "# \"vocabulary_reference\": meldb_reference,\n",
- "# \"vocabulary_version\": meldb_version,\n",
- "# # \"vocabulary_concept_id\": 0,\n",
- "# }])\n",
- "# df_test.to_sql(\"VOCABULARY\", conn, if_exists='append', index=False)\n",
- "\n",
- "\n",
- "# cur.execute(\"\"\"\n",
- "# CREATE TABLE CONCEPT_SET (\n",
- "# concept_set_id INTEGER PRIMARY KEY AUTOINCREMENT, -- Unique identifier for each concept set\n",
- "# atlas_id INTEGER, -- Unique identifier generated by ATLAS\n",
- "# concept_set_name TEXT, -- Optional name for the concept set\n",
- "# concept_set_description TEXT, -- Optional description for the concept set\n",
- "# vocabulary_id TEXT NOT NULL, -- Foreign key to VOCABULARY table\n",
- "# FOREIGN KEY (vocabulary_id) REFERENCES VOCABULARY(vocabulary_id)\n",
- "# );\"\"\")\n",
- "# cur.execute(\"DROP TABLE CONCEPT_SET;\")\n",
- "\n",
- "# cur.execute(\"\"\"\n",
- "# CREATE TABLE CONCEPT_SET_ITEM (\n",
- "# concept_set_item_id INTEGER PRIMARY KEY AUTOINCREMENT, -- Unique identifier for each mapping\n",
- "# concept_set_id INTEGER NOT NULL, -- Foreign key to CONCEPT_SET table\n",
- "# concept_id INTEGER NOT NULL, -- Foreign key to CONCEPT table\n",
- "# FOREIGN KEY (concept_set_id) REFERENCES CONCEPT_SET(concept_set_id),\n",
- "# FOREIGN KEY (concept_id) REFERENCES CONCEPT(concept_id)\n",
- "# );\"\"\")\n",
- "# cur.execute(\"DROP TABLE CONCEPT_SET_ITEM;\")\n",
- "\n",
- "# Be sure to close the connection\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d03b75f3-902f-42d7-b52f-dac7e79ecb11",
- "metadata": {},
- "outputs": [],
- "source": [
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\") # change to 'sqlite:///your_filename.db'\n",
- "cur = conn.cursor()\n",
- "\n",
- "file_path = \"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/snomed/HEART_VALVE_DISORDERS.csv\"\n",
- "df = pd.read_csv(file_path, low_memory=False)\n",
- "df = df.set_index(\"code\")\n",
- "\n",
- "df.to_sql(name='test', con=conn, if_exists='replace')\n",
- "\n",
- "conn.commit()\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d96c3511-3831-400e-ba40-0a36abcc60d3",
- "metadata": {},
- "outputs": [],
- "source": [
- "#DISPLAY SQL TABLE\n",
- "table=\"CONCEPT_SET_ITEM\"\n",
- "\n",
- "# Create a SQL connection to our SQLite database\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\")\n",
- "cur = conn.cursor()\n",
- "\n",
- "#Print ALL Columns in Table\n",
- "cur.execute(f\"PRAGMA table_info({table});\")\n",
- "df_cols = pd.DataFrame(cur.fetchall())\n",
- "print(df_cols)\n",
- "df_cols = df_cols[1]\n",
- "\n",
- "#Print TABLE\n",
- "cur.execute(f\"SELECT * FROM {table};\")\n",
- "df = pd.DataFrame(cur.fetchall())\n",
- "df = df.rename(columns={i:s for i, s in enumerate(df_cols)})\n",
- "display(df)\n",
- "\n",
- "conn.close()\n",
- "\n",
- "\n",
- "# a+s = 13364 \n",
- "# a+s+i = 13591\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "42d49a00-9646-4ba4-afb6-12297289b7a7",
- "metadata": {},
- "outputs": [],
- "source": [
- "def sql_row_exist(conn, table, column, value):\n",
- "\t# Execute and check if a result exists\n",
- "\tcur = conn.cursor()\n",
- "\tquery = f\"SELECT 1 FROM {table} WHERE {column} = ? LIMIT 1;\"\n",
- "\tcur.execute(query, (value,))\n",
- "\texists = cur.fetchone() is not None\n",
- "\t\n",
- "\treturn exists"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f7b51bcd-6ee1-4023-8d36-7f419ce4120d",
- "metadata": {},
- "outputs": [],
- "source": [
- "#EXPORT MELDB CSV OUTPUT\n",
- "\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\") # change to 'sqlite:///your_filename.db'\n",
- "cur = conn.cursor()\n",
- "\n",
- "vocab_output = \"MELDB\"\n",
- "vocab_type = \"SNOMED\"\n",
- "file_path = \"/home/jjd1c23/ssd/meldb/jjd1c23/phenotype/output/V3_2_10_MELD_snomed_no_translate.csv\"\n",
- "# file_path = \"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/snomed/HEART_VALVE_DISORDERS.csv\"\n",
- "\n",
- "# Read the CSV file with the specified delimiter\n",
- "out = pd.read_csv(file_path, low_memory=False)\n",
- "print(df.columns)\n",
- "\n",
- "for concept_set_name, grp in out.groupby(\"MELDB_concept\"):\n",
- " # display(concept_set_name, grp[[\"code\", \"MELDB_concept\"]])\n",
- " \n",
- " #Create Concept_Set\n",
- " if not sql_row_exist(conn, \"CONCEPT_SET\", \"concept_set_name\", concept_set_name):\n",
- " cur.execute(f\"INSERT INTO CONCEPT_SET (concept_set_name, vocabulary_id) VALUES ('{concept_set_name}', 'MELDB');\")\n",
- " else:\n",
- " print(\"concept_set\", concept_set_name, \"already exists\")\n",
- " #TODO: ask to remove old concept_set?\n",
- " \n",
- " #Get Concept_set_Id\n",
- " query = \"SELECT concept_set_id FROM CONCEPT_SET WHERE concept_set_name = ? AND vocabulary_id = ?;\"\n",
- " cur.execute(query, (concept_set_name, vocab_output, )) \n",
- " concept_set_id = cur.fetchone()[0]\n",
- " \n",
- " #Get corresponing Concept_id (OMOP) for each Concept_code (e.g. SNOMED)\n",
- " concept_codes = \"'\"+\"', '\".join(list(grp[\"code\"].astype(str)))+\"'\"\n",
- " query = f\"SELECT concept_id FROM CONCEPT WHERE vocabulary_id = ? AND concept_code IN ({concept_codes});\"\n",
- " print(query)\n",
- " cur.execute(query, (vocab_type, ))\n",
- " df_out = pd.DataFrame(cur.fetchall(), columns=[\"concept_id\"])\n",
- " \n",
- " if not len(grp) == len(df_out):\n",
- " print(\"ERROR: Some\", vocab_type, \"Codes do not exist in OMOP Database\")\n",
- " \n",
- " #Create Concept_set_item\n",
- " df_out[\"concept_set_id\"] = concept_set_id\n",
- " df_out.to_sql(\"CONCEPT_SET_ITEM\", conn, if_exists='append', index=False)\n",
- " \n",
- " display(df_out)\n",
- " \n",
- " \n",
- " \n",
- " # break\n",
- " \n",
- " \n",
- "\n",
- "# #Create New CONCEPT_SET\n",
- "# table_name = os.path.splitext(os.path.basename(file_path))[0] #Get name of file\n",
- "# cur.execute(f\"INSERT INTO CONCEPT_SET (concept_class_name) VALUES ('{table_name}');\")\n",
- " \n",
- " \n",
- " \n",
- "\n",
- " \n",
- " \n",
- "\n",
- "\n",
- "conn.commit()\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "85007741-e34c-4112-a63c-9fb302b76958",
- "metadata": {},
- "outputs": [],
- "source": [
- "\"'\"+\"', '\".join(list(grp[\"code\"].astype(str)))+\"'\""
- ]
- },
- {
- "cell_type": "markdown",
- "id": "423e7c21-f3bd-439d-9dcb-c17cc2cc6854",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true,
- "tags": []
- },
- "source": [
- "### ATLAS"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c6b45e4d-c7d2-42e7-9b4a-0e9c1c86d34b",
- "metadata": {},
- "outputs": [],
- "source": [
- "#Create ATLAS Concept Set\n",
- "\n",
- "def atlas_create_concept(name, description=\"\", items=[]):\n",
- " data={\n",
- " \"id\": 0,\n",
- " \"name\": name,\n",
- " \"description\": description,\n",
- " \"expression\": {\n",
- " \"items\":items \n",
- " }\n",
- " }\n",
- "\n",
- " try:\n",
- " # Sending the POST request\n",
- " response = requests.post(url, json=data, headers=headers)\n",
- "\n",
- " # Check the response status\n",
- " if response.status_code == 200 or response.status_code == 201:\n",
- " print(\"POST request successful:\")\n",
- " print(response.json()) # Assuming the response is JSON\n",
- " return response[\"id\"]\n",
- " else:\n",
- " print(f\"POST request failed. HTTP Status Code: {response.status_code}\")\n",
- " print(\"Response content:\")\n",
- " print(response.text)\n",
- " return None\n",
- "\n",
- " except requests.exceptions.RequestException as e:\n",
- " print(f\"An error occurred: {e}\")\n",
- "\n",
- "# Heart Test 1 - 1885487\n",
- "# Heart Test 2 - 1885488\n",
- "# Heart Valve Disorders - 1885449\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "45497623-1da0-4f74-b21e-da8811c89b04",
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_omop_concepts(cur, codes, vocab_id): \n",
- " #Create List for SQL\n",
- " mask = \"\"\n",
- " for c in codes:\n",
- " mask+=f'\"{c}\", '\n",
- " mask = mask[:-2] #remove last comma\n",
- " \n",
- " #Execute SQL\n",
- " cur.execute(f'SELECT * FROM CONCEPT WHERE concept_code IN ({mask}) AND VOCABULARY_ID = \"{vocab_id}\";')\n",
- " df = pd.DataFrame(cur.fetchall()) #convert to pandas df\n",
- " \n",
- " print(\"Identified\", len(df[0]) ,\"OMOP Concepts:\", list(df[0]))\n",
- " \n",
- " return df\n",
- " \n",
- "def omop_concepts_to_atlas_json(df):\n",
- " json = []\n",
- " for i, row in df.iterrows():\n",
- " #template for atlas api\n",
- " out = { \n",
- " \"concept\": {\n",
- " 'CONCEPT_ID': row[0],\n",
- " 'CONCEPT_NAME': row[1],\n",
- " 'STANDARD_CONCEPT': 'S',\n",
- " 'STANDARD_CONCEPT_CAPTION': 'Standard',\n",
- " 'INVALID_REASON': 'V',\n",
- " 'INVALID_REASON_CAPTION': 'Valid',\n",
- " 'CONCEPT_CODE': row[6],\n",
- " 'DOMAIN_ID': row[2],\n",
- " 'VOCABULARY_ID': row[3],\n",
- " 'CONCEPT_CLASS_ID': row[4],\n",
- " 'VALID_START_DATE': int(row[7]),\n",
- " 'VALID_END_DATE': int(row[8])\n",
- " },\n",
- " 'isExcluded': False,\n",
- " 'includeDescendants': False,\n",
- " 'includeMapped': False\n",
- " }\n",
- " json.append(out)\n",
- " return json \n",
- "\n",
- "conn = sqlite3.connect(\"codes/omop_54.sqlite\")\n",
- "cur = conn.cursor()\n",
- "\n",
- "vocab_id=\"SNOMED\" #SNOMED, ATC, ICD10CM, ICD9CM, Read\n",
- "csv_output = \"/home/jjd1c23/ssd/meldb/jjd1c23/concepts/snomed/ANGER.csv\"\n",
- "\n",
- "#Load CSV Output File\n",
- "df_in = pd.read_csv(csv_output)\n",
- "print(len(df_in))\n",
- "\n",
- "# df = get_omop_concepts(cur, [\"119768002\", \"5905001\"], \"SNOMED\")\n",
- "df = get_omop_concepts(cur, list(df_in[\"code\"]), vocab_id)\n",
- "json = omop_concepts_to_atlas_json(df)\n",
- "# display(json)\n",
- "\n",
- "conn.close()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ea759907-c085-472a-82e2-07b6b19e2c8f",
- "metadata": {},
- "outputs": [],
- "source": [
- "#ATLAS GET CONCEPT SET\n",
- "import requests\n",
- "\n",
- "def request_get(url):\n",
- " try:\n",
- " # Sending the GET request\n",
- " response = requests.get(url)\n",
- "\n",
- " # Check if the response status code is 200 (OK)\n",
- " if response.status_code == 200:\n",
- " print(\"Response data:\")\n",
- " # print(response.json()) # Assuming the response is in JSON format\n",
- " return response.json()\n",
- " else:\n",
- " print(f\"Failed to fetch data. HTTP Status Code: {response.status_code}\")\n",
- " print(\"Response content:\")\n",
- " print(response.text)\n",
- " return None\n",
- "\n",
- " except requests.exceptions.RequestException as e:\n",
- " print(f\"An error occurred: {e}\")\n",
- "\n",
- "\n",
- "#GET SET INFO\n",
- "set_id = \"1885449\"\n",
- "url = f\"https://atlas-demo.ohdsi.org/WebAPI/conceptset/{set_id}\"\n",
- "request_get(url)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5a70e636-6051-4930-bf1b-30d093fd0552",
- "metadata": {},
- "outputs": [],
- "source": [
- "#GET SET ITEMS (Concepts)\n",
- "set_id = \"1885449\"\n",
- "url = f\"https://atlas-demo.ohdsi.org/WebAPI/conceptset/{set_id}/expression/ATLASPROD\"\n",
- "response = request_get(url)\n",
- "display(response)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "96bfcd9c-27e8-4be4-a680-7553d908790e",
- "metadata": {},
- "outputs": [],
- "source": [
- "#ATLAS CREATE CONCEPT SET\n"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.9.7"
- },
- "toc-showtags": false
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/run.sh b/run.sh
deleted file mode 100644
index b64772d21a5e2476e544c0afd6826de0502df4aa..0000000000000000000000000000000000000000
--- a/run.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#! /usr/bin/bash
-
-version="V3_2_10"
-previous="V3_2_9"
-
-python main.py -r2 PHEN_assign_v3.json CONC_summary_working.xlsx
-mv output/MELD_concepts_read.csv output/${version}_MELD_concepts_readv2.csv
-
-python main.py -i PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-mv output/MELD_concepts_read.csv output/${version}_MELD_icd10_no_translate.csv
-
-python main.py -s PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-mv output/MELD_concepts_read.csv output/${version}_MELD_snomed_no_translate.csv
-
-# python main.py -o PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-# mv output/MELD_concepts_read.csv output/${version}_MELD_opcs4_no_translate.csv
-
-python main.py -a PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-mv output/MELD_concepts_read.csv output/${version}_MELD_atc_no_translate.csv
-
-# python main.py -m PHEN_assign_v3.json CONC_summary_working.xlsx --no-translate
-# mv output/MELD_concepts_read.csv output/${version}_MELD_med_no_translate.csv
-
-mv output/MELD_errors.csv output/${version}_MELD_errors.csv
-
-
-#Generate Report
-rm concepts-output/MELD-report.md
-python report.py PHEN_assign_v3.json CONC_summary_working.xlsx codes/ concepts-output/MELD-report.md ${version} ${previous}
-
-#Divide Concepts to Output Repo
-rm -rf concepts-output/readv2/*
-rm -rf concepts-output/icd10/*
-rm -rf concepts-output/snomed/*
-rm -rf concepts-output/atc/*
-python publish.py output/${version}_MELD_concepts_readv2.csv concepts-output/readv2/
-python publish.py output/${version}_MELD_icd10_no_translate.csv concepts-output/icd10/
-python publish.py output/${version}_MELD_snomed_no_translate.csv concepts-output/snomed/
-python publish.py output/${version}_MELD_atc_no_translate.csv concepts-output/atc/
-cp output/${version}_MELD_errors.csv concepts-output/${version}_MELD_errors.csv
-
-# Show Changes in Output repo (should be same as report)
-cd concepts-output
-git diff --stat
\ No newline at end of file