from biocypher import BioCypher import networkx as nx import json import os import sys import re import uuid import gc from dotenv import load_dotenv from graphCreation import create_graph from graphCreation.process_references import process_references from graphCreation.property_convolution import property_convolution from schema_config_generation import write_automated_schema from fhirImport import get_patient_everything, get_bundle is_test = os.getenv('TEST_MODE') test_depth = os.getenv('TEST_DEPTH') def load_multiple_fhir_patients(n): init_load = True ids = [] #get n ids init_ids = True while len(ids) < n and init_ids: if init_load: is_complex = os.getenv('COMPLEX_PATIENTS') selected_patients_file = os.getenv('SELECTED_PATIENTS_FILE') #print("> /dev/null") if selected_patients_file and os.path.isfile(selected_patients_file): print(f"-- Loading patient IDs from file: {selected_patients_file} --") with open(selected_patients_file, 'r') as f: for line in f: patient_id = line.strip() if patient_id: ids.append(patient_id) print(f"Loaded {len(ids)} patient IDs from file.") n = len(ids) break # Exit the while loop since we got IDs from the file elif is_complex and is_complex.upper() != 'TRUE': bundle = get_bundle(None, '/Patient?_count=' + str(n)) else: #print("-- Looking for complex patients --") print("-- Looking for gout patients --") #bundle = get_bundle(None, '/Patient?_has:Observation:subject:status=final&_count=' + str(n)) bundle = get_bundle(None, '/Patient?_has:Condition:patient:code=M10.00,M10.01,M10.02,M10.03,M10.04,M10.05,M10.06,M10.07,M10.08,M10.09,M10.10,M10.11,M10.12,M10.13,M10.14,M10.15,M10.16,M10.17,M10.18,M10.19,M10.20,M10.21,M10.22,M10.23,M10.24,M10.25,M10.26,M10.27,M10.28,M10.29,M10.30,M10.31,M10.32,M10.33,M10.34,M10.35,M10.36,M10.37,M10.38,M10.39,M10.40,M10.41,M10.42,M10.43,M10.44,M10.45,M10.46,M10.47,M10.48,M10.49,M10.90,M10.91,M10.92,M10.93,M10.94,M10.95,M10.96,M10.97,M10.98,M10.99&_count=' + str(n)) print("Got patient ids") init_load = False else: print("NEXT LINK: ", next_link, flush=True) bundle = get_bundle(next_link, None) #print(bundle.json()) if 'entry' not in bundle.json(): print("ERROR -- No data found in the fhir bundle. Check the request and if the server is up and responding") sys.exit(1) for entry in bundle.json()['entry']: ids.append(entry['resource']['id']) init_ids = False for l in bundle.json()['link']: if l['relation'] == "next": next_link = l['url'] init_ids = True if len(ids) < n: n = len(ids) print("####GRABBED ", n , " IDs from the DB####", flush=True) bundles_to_graph(ids, n) def bundles_to_graph(ids, n): init = True batch_size = int(os.getenv('BATCH_SIZE')) c = 0 print(len(ids)) #get bundle for each ID for id in ids: c += 1 bundle = get_patient_everything(id).json() bundle = replace_single_quotes(bundle) ### maybe not needed for german data if init: graph = nx.DiGraph() init = False create_graph.add_json_to_networkx(bundle, id + '_bundle', graph) if c % 50 == 0: print("---------- ", c, " patients loaded ----------", flush=True) if c % batch_size == 0 or c == n: print(c, " patients imported, reducing graph", flush = True) process_references(graph) property_convolution(graph) run_biocypher(graph) init = True #print("Full batch:", c) del graph def replace_single_quotes(obj): if isinstance(obj, str): # If it's a string, replace single quotes return obj.replace("'", "''") elif isinstance(obj, dict): # If it's a dictionary, process each key-value pair return {key: replace_single_quotes(value) for key, value in obj.items()} elif isinstance(obj, list): # If it's a list, process each item return [replace_single_quotes(item) for item in obj] else: return obj # Leave other data types unchanged def main(): ## create networkX and run improvement scripts print("Creating the graph...", flush=True) n_patients = int(os.getenv('NUMBER_OF_PATIENTS')) load_multiple_fhir_patients(n_patients) #write the import script -- we are creating our own script since BC would only consider the last batch as an input print("CREATING THE SCRIPT") generate_neo4j_import_script() with open('/neo4j_import/shell-scipt-complete', 'w') as f: f.write('Import completed successfully') print("FHIR import completed successfully") def run_biocypher(nx_graph): #get lists of node and edge types print("Generate auto schema...", flush=True) write_automated_schema(nx_graph, 'config/automated_schema.yaml', 'config/manual_schema_config.yaml') # create Biocypher driver bc = BioCypher( biocypher_config_path="config/biocypher_config.yaml", ) #bc.show_ontology_structure() #very extensive #BioCypher preperation bc.write_nodes(node_generator(nx_graph)) bc.write_edges(edge_generator(nx_graph)) def node_generator(nx_graph): for node in nx_graph.nodes(): label = nx_graph.nodes[node].get('label') if(label == 'dummy' or label == 'Dummy'): print("Skipped dummy: ", nx_graph.nodes[node]) continue if label == "resource": label = nx_graph.nodes[node].get('resourceType') label = label.capitalize() nx_graph.nodes[node]['label'] = label if(nx_graph.nodes[node].get('label') in ['search', 'meta', 'link', 'Search', 'Meta', 'Link']): continue yield( nx_graph.nodes[node].get('unique_id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx label, nx_graph.nodes[node] # get properties ) def edge_generator(nx_graph): for edge in nx_graph.edges(data = True): source, target, attributes = edge s_label = nx_graph.nodes[source].get('label') if s_label == 'resource': s_label = nx_graph.nodes[source].get('resourceType') elif s_label == 'dummy' or s_label == 'Dummy': s_label = nx_graph.nodes[source].get('edge_label') t_label = nx_graph.nodes[target].get('label') if t_label == 'resource': t_label = nx_graph.nodes[target].get('resourceType') elif t_label == 'dummy' or t_label == 'Dummy': t_label = nx_graph.nodes[target].get('edge_label') label = s_label.capitalize() + '_to_' + t_label.capitalize() yield( attributes.get('id', str(uuid.uuid4())), # Edge ID (if exists, otherwise use nx internal id) nx_graph.nodes[source].get('unique_id', source), nx_graph.nodes[target].get('unique_id', target), label, attributes # All edge attributes ) def generate_neo4j_import_script(directory_path="/neo4j_import/", output_file="neo4j-admin-import-call.sh"): """ Reads files in a directory and generates a Neo4j import shell script. Args: directory_path (str): Path to the directory containing CSV files output_file (str): Name of the output shell script file Returns: str: Path to the generated shell script """ # Get all files in the directory all_files = os.listdir(directory_path) # Dictionary to store entity types (nodes and relationships) entity_types = {} # Find all header files and use them to identify entity types for filename in all_files: if '-header.csv' in filename: entity_name = filename.split('-header.csv')[0] # Check if it's a relationship (contains "To" and "Association") is_relationship = ("To" in entity_name and "Association" in entity_name) or "_has_" in entity_name or "Has_" in entity_name # Store in entity_types dictionary if is_relationship: entity_type = "relationships" else: entity_type = "nodes" # Initialize the entity if not already present if entity_name not in entity_types: entity_types[entity_name] = { "type": entity_type, "header": f"/neo4j_import/{filename}", "has_parts": False } # Check for part files for each entity for entity_name in entity_types: # Create pattern to match part files for this entity part_pattern = f"{entity_name}-part" # Check if any file matches the pattern for filename in all_files: if part_pattern in filename: entity_types[entity_name]["has_parts"] = True break # Generate the import commands nodes_command = "" relationships_command = "" for entity_name, info in entity_types.items(): if info["has_parts"]: # Create the command string with wildcard for part files command = f" --{info['type']}=\"{info['header']},/neo4j_import/{entity_name}-part.*\"" # Add to appropriate command string if info['type'] == "nodes": nodes_command += command else: # relationships relationships_command += command # Create the shell script content script_content = """#!/bin/bash version=$(bin/neo4j-admin --version | cut -d '.' -f 1) if [[ $version -ge 5 ]]; then \tbin/neo4j-admin database import full neo4j --delimiter="\\t" --array-delimiter="|" --quote="'" --overwrite-destination=true --skip-bad-relationships=true --skip-duplicate-nodes=true{nodes}{relationships} else \tbin/neo4j-admin import --database=neo4j --delimiter="\\t" --array-delimiter="|" --quote="'" --force=true --skip-bad-relationships=true --skip-duplicate-nodes=true{nodes}{relationships} fi """.format(nodes=nodes_command, relationships=relationships_command) # Write the script to file script_path = os.path.join(directory_path, output_file) with open(script_path, 'w') as f: f.write(script_content) # Make the script executable os.chmod(script_path, 0o755) print("Shell import script created", flush=True) if __name__ == "__main__": main()