304 lines
11 KiB
Python
304 lines
11 KiB
Python
from biocypher import BioCypher
|
|
import networkx as nx
|
|
import json
|
|
import os
|
|
import sys
|
|
import re
|
|
import uuid
|
|
import gc
|
|
from dotenv import load_dotenv
|
|
|
|
|
|
from graphCreation import create_graph
|
|
|
|
from graphCreation.process_references import process_references
|
|
|
|
from graphCreation.property_convolution import property_convolution
|
|
|
|
from schema_config_generation import write_automated_schema
|
|
|
|
from fhirImport import get_patient_everything, get_bundle
|
|
|
|
is_test = os.getenv('TEST_MODE')
|
|
test_depth = os.getenv('TEST_DEPTH')
|
|
|
|
|
|
|
|
def load_multiple_fhir_patients(n):
|
|
init_load = True
|
|
ids = []
|
|
#get n ids
|
|
init_ids = True
|
|
while len(ids) < n and init_ids:
|
|
if init_load:
|
|
is_complex = os.getenv('COMPLEX_PATIENTS')
|
|
selected_patients_file = os.getenv('SELECTED_PATIENTS_FILE')
|
|
#print("> /dev/null")
|
|
if selected_patients_file and os.path.isfile(selected_patients_file):
|
|
print(f"-- Loading patient IDs from file: {selected_patients_file} --")
|
|
with open(selected_patients_file, 'r') as f:
|
|
for line in f:
|
|
patient_id = line.strip()
|
|
if patient_id:
|
|
ids.append(patient_id)
|
|
print(f"Loaded {len(ids)} patient IDs from file.")
|
|
n = len(ids)
|
|
break # Exit the while loop since we got IDs from the file
|
|
|
|
elif is_complex and is_complex.upper() != 'TRUE':
|
|
bundle = get_bundle(None, '/Patient?_count=' + str(n))
|
|
else:
|
|
#print("-- Looking for complex patients --")
|
|
print("-- Looking for gout patients --")
|
|
#bundle = get_bundle(None, '/Patient?_has:Observation:subject:status=final&_count=' + str(n))
|
|
bundle = get_bundle(None, '/Patient?_has:Condition:patient:code=M10.00,M10.01,M10.02,M10.03,M10.04,M10.05,M10.06,M10.07,M10.08,M10.09,M10.10,M10.11,M10.12,M10.13,M10.14,M10.15,M10.16,M10.17,M10.18,M10.19,M10.20,M10.21,M10.22,M10.23,M10.24,M10.25,M10.26,M10.27,M10.28,M10.29,M10.30,M10.31,M10.32,M10.33,M10.34,M10.35,M10.36,M10.37,M10.38,M10.39,M10.40,M10.41,M10.42,M10.43,M10.44,M10.45,M10.46,M10.47,M10.48,M10.49,M10.90,M10.91,M10.92,M10.93,M10.94,M10.95,M10.96,M10.97,M10.98,M10.99&_count=' + str(n))
|
|
print("Got patient ids")
|
|
init_load = False
|
|
else:
|
|
print("NEXT LINK: ", next_link, flush=True)
|
|
bundle = get_bundle(next_link, None)
|
|
|
|
#print(bundle.json())
|
|
if 'entry' not in bundle.json():
|
|
print("ERROR -- No data found in the fhir bundle. Check the request and if the server is up and responding")
|
|
sys.exit(1)
|
|
|
|
for entry in bundle.json()['entry']:
|
|
ids.append(entry['resource']['id'])
|
|
|
|
init_ids = False
|
|
for l in bundle.json()['link']:
|
|
if l['relation'] == "next":
|
|
next_link = l['url']
|
|
init_ids = True
|
|
|
|
if len(ids) < n:
|
|
n = len(ids)
|
|
|
|
print("####GRABBED ", n , " IDs from the DB####", flush=True)
|
|
bundles_to_graph(ids, n)
|
|
|
|
|
|
def bundles_to_graph(ids, n):
|
|
|
|
init = True
|
|
batch_size = int(os.getenv('BATCH_SIZE'))
|
|
c = 0
|
|
|
|
print(len(ids))
|
|
|
|
#get bundle for each ID
|
|
for id in ids:
|
|
|
|
c += 1
|
|
bundle = get_patient_everything(id).json()
|
|
bundle = replace_single_quotes(bundle) ### maybe not needed for german data
|
|
if init:
|
|
graph = nx.DiGraph()
|
|
init = False
|
|
|
|
create_graph.add_json_to_networkx(bundle, id + '_bundle', graph)
|
|
|
|
if c % 50 == 0:
|
|
print("---------- ", c, " patients loaded ----------", flush=True)
|
|
|
|
if c % batch_size == 0 or c == n:
|
|
print(c, " patients imported, reducing graph", flush = True)
|
|
process_references(graph)
|
|
property_convolution(graph)
|
|
|
|
run_biocypher(graph)
|
|
init = True
|
|
#print("Full batch:", c)
|
|
del graph
|
|
|
|
def replace_single_quotes(obj):
|
|
if isinstance(obj, str): # If it's a string, replace single quotes
|
|
return obj.replace("'", "''")
|
|
elif isinstance(obj, dict): # If it's a dictionary, process each key-value pair
|
|
return {key: replace_single_quotes(value) for key, value in obj.items()}
|
|
elif isinstance(obj, list): # If it's a list, process each item
|
|
return [replace_single_quotes(item) for item in obj]
|
|
else:
|
|
return obj # Leave other data types unchanged
|
|
|
|
def main():
|
|
## create networkX and run improvement scripts
|
|
print("Creating the graph...", flush=True)
|
|
|
|
n_patients = int(os.getenv('NUMBER_OF_PATIENTS'))
|
|
load_multiple_fhir_patients(n_patients)
|
|
|
|
#write the import script -- we are creating our own script since BC would only consider the last batch as an input
|
|
print("CREATING THE SCRIPT")
|
|
generate_neo4j_import_script()
|
|
with open('/neo4j_import/shell-scipt-complete', 'w') as f:
|
|
f.write('Import completed successfully')
|
|
|
|
print("FHIR import completed successfully")
|
|
|
|
|
|
def run_biocypher(nx_graph):
|
|
|
|
#get lists of node and edge types
|
|
print("Generate auto schema...", flush=True)
|
|
write_automated_schema(nx_graph, 'config/automated_schema.yaml', 'config/manual_schema_config.yaml')
|
|
|
|
|
|
# create Biocypher driver
|
|
bc = BioCypher(
|
|
biocypher_config_path="config/biocypher_config.yaml",
|
|
)
|
|
|
|
#bc.show_ontology_structure() #very extensive
|
|
#BioCypher preperation
|
|
|
|
bc.write_nodes(node_generator(nx_graph))
|
|
bc.write_edges(edge_generator(nx_graph))
|
|
|
|
|
|
def node_generator(nx_graph):
|
|
for node in nx_graph.nodes():
|
|
|
|
label = nx_graph.nodes[node].get('label')
|
|
|
|
if(label == 'dummy' or label == 'Dummy'):
|
|
print("Skipped dummy: ", nx_graph.nodes[node])
|
|
continue
|
|
|
|
if label == "resource":
|
|
label = nx_graph.nodes[node].get('resourceType')
|
|
|
|
label = label.capitalize()
|
|
nx_graph.nodes[node]['label'] = label
|
|
|
|
if(nx_graph.nodes[node].get('label') in ['search', 'meta', 'link', 'Search', 'Meta', 'Link']):
|
|
continue
|
|
|
|
|
|
yield(
|
|
nx_graph.nodes[node].get('unique_id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
|
|
label,
|
|
nx_graph.nodes[node] # get properties
|
|
)
|
|
|
|
def edge_generator(nx_graph):
|
|
for edge in nx_graph.edges(data = True):
|
|
source, target, attributes = edge
|
|
|
|
|
|
s_label = nx_graph.nodes[source].get('label')
|
|
if s_label == 'resource':
|
|
s_label = nx_graph.nodes[source].get('resourceType')
|
|
elif s_label == 'dummy' or s_label == 'Dummy':
|
|
s_label = nx_graph.nodes[source].get('edge_label')
|
|
|
|
t_label = nx_graph.nodes[target].get('label')
|
|
if t_label == 'resource':
|
|
t_label = nx_graph.nodes[target].get('resourceType')
|
|
elif t_label == 'dummy' or t_label == 'Dummy':
|
|
t_label = nx_graph.nodes[target].get('edge_label')
|
|
|
|
label = s_label.capitalize() + '_to_' + t_label.capitalize()
|
|
|
|
|
|
yield(
|
|
attributes.get('id', str(uuid.uuid4())), # Edge ID (if exists, otherwise use nx internal id)
|
|
nx_graph.nodes[source].get('unique_id', source),
|
|
nx_graph.nodes[target].get('unique_id', target),
|
|
label,
|
|
attributes # All edge attributes
|
|
)
|
|
|
|
|
|
def generate_neo4j_import_script(directory_path="/neo4j_import/", output_file="neo4j-admin-import-call.sh"):
|
|
"""
|
|
Reads files in a directory and generates a Neo4j import shell script.
|
|
|
|
Args:
|
|
directory_path (str): Path to the directory containing CSV files
|
|
output_file (str): Name of the output shell script file
|
|
|
|
Returns:
|
|
str: Path to the generated shell script
|
|
"""
|
|
# Get all files in the directory
|
|
all_files = os.listdir(directory_path)
|
|
|
|
# Dictionary to store entity types (nodes and relationships)
|
|
entity_types = {}
|
|
|
|
|
|
# Find all header files and use them to identify entity types
|
|
for filename in all_files:
|
|
if '-header.csv' in filename:
|
|
entity_name = filename.split('-header.csv')[0]
|
|
|
|
# Check if it's a relationship (contains "To" and "Association")
|
|
is_relationship = ("To" in entity_name and "Association" in entity_name) or "_has_" in entity_name or "Has_" in entity_name
|
|
|
|
# Store in entity_types dictionary
|
|
if is_relationship:
|
|
entity_type = "relationships"
|
|
else:
|
|
entity_type = "nodes"
|
|
|
|
# Initialize the entity if not already present
|
|
if entity_name not in entity_types:
|
|
entity_types[entity_name] = {
|
|
"type": entity_type,
|
|
"header": f"/neo4j_import/{filename}",
|
|
"has_parts": False
|
|
}
|
|
|
|
# Check for part files for each entity
|
|
for entity_name in entity_types:
|
|
# Create pattern to match part files for this entity
|
|
part_pattern = f"{entity_name}-part"
|
|
|
|
# Check if any file matches the pattern
|
|
for filename in all_files:
|
|
if part_pattern in filename:
|
|
entity_types[entity_name]["has_parts"] = True
|
|
break
|
|
|
|
# Generate the import commands
|
|
nodes_command = ""
|
|
relationships_command = ""
|
|
|
|
for entity_name, info in entity_types.items():
|
|
if info["has_parts"]:
|
|
# Create the command string with wildcard for part files
|
|
command = f" --{info['type']}=\"{info['header']},/neo4j_import/{entity_name}-part.*\""
|
|
|
|
# Add to appropriate command string
|
|
if info['type'] == "nodes":
|
|
nodes_command += command
|
|
else: # relationships
|
|
relationships_command += command
|
|
|
|
# Create the shell script content
|
|
script_content = """#!/bin/bash
|
|
version=$(bin/neo4j-admin --version | cut -d '.' -f 1)
|
|
if [[ $version -ge 5 ]]; then
|
|
\tbin/neo4j-admin database import full neo4j --delimiter="\\t" --array-delimiter="|" --quote="'" --overwrite-destination=true --skip-bad-relationships=true --skip-duplicate-nodes=true{nodes}{relationships}
|
|
else
|
|
\tbin/neo4j-admin import --database=neo4j --delimiter="\\t" --array-delimiter="|" --quote="'" --force=true --skip-bad-relationships=true --skip-duplicate-nodes=true{nodes}{relationships}
|
|
fi
|
|
""".format(nodes=nodes_command, relationships=relationships_command)
|
|
|
|
# Write the script to file
|
|
script_path = os.path.join(directory_path, output_file)
|
|
with open(script_path, 'w') as f:
|
|
f.write(script_content)
|
|
|
|
# Make the script executable
|
|
os.chmod(script_path, 0o755)
|
|
|
|
print("Shell import script created", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|