init public release
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
from biocypher import BioCypher
|
||||
import networkx as nx
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import uuid
|
||||
import gc
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
from graphCreation import create_graph
|
||||
|
||||
from graphCreation.process_references import process_references
|
||||
|
||||
from graphCreation.property_convolution import property_convolution
|
||||
|
||||
from schema_config_generation import write_automated_schema
|
||||
|
||||
from fhirImport import get_patient_everything, get_bundle
|
||||
|
||||
is_test = os.getenv('TEST_MODE')
|
||||
test_depth = os.getenv('TEST_DEPTH')
|
||||
|
||||
|
||||
|
||||
def load_multiple_fhir_patients(n):
|
||||
init_load = True
|
||||
ids = []
|
||||
#get n ids
|
||||
init_ids = True
|
||||
while len(ids) < n and init_ids:
|
||||
if init_load:
|
||||
is_complex = os.getenv('COMPLEX_PATIENTS')
|
||||
selected_patients_file = os.getenv('SELECTED_PATIENTS_FILE')
|
||||
#print("> /dev/null")
|
||||
if selected_patients_file and os.path.isfile(selected_patients_file):
|
||||
print(f"-- Loading patient IDs from file: {selected_patients_file} --")
|
||||
with open(selected_patients_file, 'r') as f:
|
||||
for line in f:
|
||||
patient_id = line.strip()
|
||||
if patient_id:
|
||||
ids.append(patient_id)
|
||||
print(f"Loaded {len(ids)} patient IDs from file.")
|
||||
n = len(ids)
|
||||
break # Exit the while loop since we got IDs from the file
|
||||
|
||||
elif is_complex and is_complex.upper() != 'TRUE':
|
||||
bundle = get_bundle(None, '/Patient?_count=' + str(n))
|
||||
else:
|
||||
#print("-- Looking for complex patients --")
|
||||
print("-- Looking for gout patients --")
|
||||
#bundle = get_bundle(None, '/Patient?_has:Observation:subject:status=final&_count=' + str(n))
|
||||
bundle = get_bundle(None, '/Patient?_has:Condition:patient:code=M10.00,M10.01,M10.02,M10.03,M10.04,M10.05,M10.06,M10.07,M10.08,M10.09,M10.10,M10.11,M10.12,M10.13,M10.14,M10.15,M10.16,M10.17,M10.18,M10.19,M10.20,M10.21,M10.22,M10.23,M10.24,M10.25,M10.26,M10.27,M10.28,M10.29,M10.30,M10.31,M10.32,M10.33,M10.34,M10.35,M10.36,M10.37,M10.38,M10.39,M10.40,M10.41,M10.42,M10.43,M10.44,M10.45,M10.46,M10.47,M10.48,M10.49,M10.90,M10.91,M10.92,M10.93,M10.94,M10.95,M10.96,M10.97,M10.98,M10.99&_count=' + str(n))
|
||||
print("Got patient ids")
|
||||
init_load = False
|
||||
else:
|
||||
print("NEXT LINK: ", next_link, flush=True)
|
||||
bundle = get_bundle(next_link, None)
|
||||
|
||||
#print(bundle.json())
|
||||
if 'entry' not in bundle.json():
|
||||
print("ERROR -- No data found in the fhir bundle. Check the request and if the server is up and responding")
|
||||
sys.exit(1)
|
||||
|
||||
for entry in bundle.json()['entry']:
|
||||
ids.append(entry['resource']['id'])
|
||||
|
||||
init_ids = False
|
||||
for l in bundle.json()['link']:
|
||||
if l['relation'] == "next":
|
||||
next_link = l['url']
|
||||
init_ids = True
|
||||
|
||||
if len(ids) < n:
|
||||
n = len(ids)
|
||||
|
||||
print("####GRABBED ", n , " IDs from the DB####", flush=True)
|
||||
bundles_to_graph(ids, n)
|
||||
|
||||
|
||||
def bundles_to_graph(ids, n):
|
||||
|
||||
init = True
|
||||
batch_size = int(os.getenv('BATCH_SIZE'))
|
||||
c = 0
|
||||
|
||||
print(len(ids))
|
||||
|
||||
#get bundle for each ID
|
||||
for id in ids:
|
||||
|
||||
c += 1
|
||||
bundle = get_patient_everything(id).json()
|
||||
bundle = replace_single_quotes(bundle) ### maybe not needed for german data
|
||||
if init:
|
||||
graph = nx.DiGraph()
|
||||
init = False
|
||||
|
||||
create_graph.add_json_to_networkx(bundle, id + '_bundle', graph)
|
||||
|
||||
if c % 50 == 0:
|
||||
print("---------- ", c, " patients loaded ----------", flush=True)
|
||||
|
||||
if c % batch_size == 0 or c == n:
|
||||
print(c, " patients imported, reducing graph", flush = True)
|
||||
process_references(graph)
|
||||
property_convolution(graph)
|
||||
|
||||
run_biocypher(graph)
|
||||
init = True
|
||||
#print("Full batch:", c)
|
||||
del graph
|
||||
|
||||
def replace_single_quotes(obj):
|
||||
if isinstance(obj, str): # If it's a string, replace single quotes
|
||||
return obj.replace("'", "''")
|
||||
elif isinstance(obj, dict): # If it's a dictionary, process each key-value pair
|
||||
return {key: replace_single_quotes(value) for key, value in obj.items()}
|
||||
elif isinstance(obj, list): # If it's a list, process each item
|
||||
return [replace_single_quotes(item) for item in obj]
|
||||
else:
|
||||
return obj # Leave other data types unchanged
|
||||
|
||||
def main():
|
||||
## create networkX and run improvement scripts
|
||||
print("Creating the graph...", flush=True)
|
||||
|
||||
n_patients = int(os.getenv('NUMBER_OF_PATIENTS'))
|
||||
load_multiple_fhir_patients(n_patients)
|
||||
|
||||
#write the import script -- we are creating our own script since BC would only consider the last batch as an input
|
||||
print("CREATING THE SCRIPT")
|
||||
generate_neo4j_import_script()
|
||||
with open('/neo4j_import/shell-scipt-complete', 'w') as f:
|
||||
f.write('Import completed successfully')
|
||||
|
||||
print("FHIR import completed successfully")
|
||||
|
||||
|
||||
def run_biocypher(nx_graph):
|
||||
|
||||
#get lists of node and edge types
|
||||
print("Generate auto schema...", flush=True)
|
||||
write_automated_schema(nx_graph, 'config/automated_schema.yaml', 'config/manual_schema_config.yaml')
|
||||
|
||||
|
||||
# create Biocypher driver
|
||||
bc = BioCypher(
|
||||
biocypher_config_path="config/biocypher_config.yaml",
|
||||
)
|
||||
|
||||
#bc.show_ontology_structure() #very extensive
|
||||
#BioCypher preperation
|
||||
|
||||
bc.write_nodes(node_generator(nx_graph))
|
||||
bc.write_edges(edge_generator(nx_graph))
|
||||
|
||||
|
||||
def node_generator(nx_graph):
|
||||
for node in nx_graph.nodes():
|
||||
|
||||
label = nx_graph.nodes[node].get('label')
|
||||
|
||||
if(label == 'dummy' or label == 'Dummy'):
|
||||
print("Skipped dummy: ", nx_graph.nodes[node])
|
||||
continue
|
||||
|
||||
if label == "resource":
|
||||
label = nx_graph.nodes[node].get('resourceType')
|
||||
|
||||
label = label.capitalize()
|
||||
nx_graph.nodes[node]['label'] = label
|
||||
|
||||
if(nx_graph.nodes[node].get('label') in ['search', 'meta', 'link', 'Search', 'Meta', 'Link']):
|
||||
continue
|
||||
|
||||
|
||||
yield(
|
||||
nx_graph.nodes[node].get('unique_id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
|
||||
label,
|
||||
nx_graph.nodes[node] # get properties
|
||||
)
|
||||
|
||||
def edge_generator(nx_graph):
|
||||
for edge in nx_graph.edges(data = True):
|
||||
source, target, attributes = edge
|
||||
|
||||
|
||||
s_label = nx_graph.nodes[source].get('label')
|
||||
if s_label == 'resource':
|
||||
s_label = nx_graph.nodes[source].get('resourceType')
|
||||
elif s_label == 'dummy' or s_label == 'Dummy':
|
||||
s_label = nx_graph.nodes[source].get('edge_label')
|
||||
|
||||
t_label = nx_graph.nodes[target].get('label')
|
||||
if t_label == 'resource':
|
||||
t_label = nx_graph.nodes[target].get('resourceType')
|
||||
elif t_label == 'dummy' or t_label == 'Dummy':
|
||||
t_label = nx_graph.nodes[target].get('edge_label')
|
||||
|
||||
label = s_label.capitalize() + '_to_' + t_label.capitalize()
|
||||
|
||||
|
||||
yield(
|
||||
attributes.get('id', str(uuid.uuid4())), # Edge ID (if exists, otherwise use nx internal id)
|
||||
nx_graph.nodes[source].get('unique_id', source),
|
||||
nx_graph.nodes[target].get('unique_id', target),
|
||||
label,
|
||||
attributes # All edge attributes
|
||||
)
|
||||
|
||||
|
||||
def generate_neo4j_import_script(directory_path="/neo4j_import/", output_file="neo4j-admin-import-call.sh"):
|
||||
"""
|
||||
Reads files in a directory and generates a Neo4j import shell script.
|
||||
|
||||
Args:
|
||||
directory_path (str): Path to the directory containing CSV files
|
||||
output_file (str): Name of the output shell script file
|
||||
|
||||
Returns:
|
||||
str: Path to the generated shell script
|
||||
"""
|
||||
# Get all files in the directory
|
||||
all_files = os.listdir(directory_path)
|
||||
|
||||
# Dictionary to store entity types (nodes and relationships)
|
||||
entity_types = {}
|
||||
|
||||
|
||||
# Find all header files and use them to identify entity types
|
||||
for filename in all_files:
|
||||
if '-header.csv' in filename:
|
||||
entity_name = filename.split('-header.csv')[0]
|
||||
|
||||
# Check if it's a relationship (contains "To" and "Association")
|
||||
is_relationship = ("To" in entity_name and "Association" in entity_name) or "_has_" in entity_name or "Has_" in entity_name
|
||||
|
||||
# Store in entity_types dictionary
|
||||
if is_relationship:
|
||||
entity_type = "relationships"
|
||||
else:
|
||||
entity_type = "nodes"
|
||||
|
||||
# Initialize the entity if not already present
|
||||
if entity_name not in entity_types:
|
||||
entity_types[entity_name] = {
|
||||
"type": entity_type,
|
||||
"header": f"/neo4j_import/{filename}",
|
||||
"has_parts": False
|
||||
}
|
||||
|
||||
# Check for part files for each entity
|
||||
for entity_name in entity_types:
|
||||
# Create pattern to match part files for this entity
|
||||
part_pattern = f"{entity_name}-part"
|
||||
|
||||
# Check if any file matches the pattern
|
||||
for filename in all_files:
|
||||
if part_pattern in filename:
|
||||
entity_types[entity_name]["has_parts"] = True
|
||||
break
|
||||
|
||||
# Generate the import commands
|
||||
nodes_command = ""
|
||||
relationships_command = ""
|
||||
|
||||
for entity_name, info in entity_types.items():
|
||||
if info["has_parts"]:
|
||||
# Create the command string with wildcard for part files
|
||||
command = f" --{info['type']}=\"{info['header']},/neo4j_import/{entity_name}-part.*\""
|
||||
|
||||
# Add to appropriate command string
|
||||
if info['type'] == "nodes":
|
||||
nodes_command += command
|
||||
else: # relationships
|
||||
relationships_command += command
|
||||
|
||||
# Create the shell script content
|
||||
script_content = """#!/bin/bash
|
||||
version=$(bin/neo4j-admin --version | cut -d '.' -f 1)
|
||||
if [[ $version -ge 5 ]]; then
|
||||
\tbin/neo4j-admin database import full neo4j --delimiter="\\t" --array-delimiter="|" --quote="'" --overwrite-destination=true --skip-bad-relationships=true --skip-duplicate-nodes=true{nodes}{relationships}
|
||||
else
|
||||
\tbin/neo4j-admin import --database=neo4j --delimiter="\\t" --array-delimiter="|" --quote="'" --force=true --skip-bad-relationships=true --skip-duplicate-nodes=true{nodes}{relationships}
|
||||
fi
|
||||
""".format(nodes=nodes_command, relationships=relationships_command)
|
||||
|
||||
# Write the script to file
|
||||
script_path = os.path.join(directory_path, output_file)
|
||||
with open(script_path, 'w') as f:
|
||||
f.write(script_content)
|
||||
|
||||
# Make the script executable
|
||||
os.chmod(script_path, 0o755)
|
||||
|
||||
print("Shell import script created", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user