153 lines
5.6 KiB
Python
153 lines
5.6 KiB
Python
from biocypher import BioCypher
|
|
import networkx as nx
|
|
import json
|
|
import os
|
|
import uuid
|
|
from graphCreation import create_graph
|
|
from graphCreation.process_references import process_references
|
|
from graphCreation.property_convolution import property_convolution
|
|
|
|
from schema_config_generation import write_automated_schema
|
|
from graphCreation.node_typing import set_resource_type
|
|
|
|
|
|
|
|
def load_multiple_fhir_bundles(directory_path):
|
|
graph = nx.DiGraph()
|
|
init = True
|
|
|
|
# Iterate over all files in the directory
|
|
for filename in os.listdir(directory_path):
|
|
if filename.endswith('.json'): # Assuming FHIR bundles are in JSON format
|
|
file_path = os.path.join(directory_path, filename)
|
|
with open(file_path, 'r') as f:
|
|
bundle_json = json.load(f)
|
|
|
|
#fix all strings to to enable ' in neo4j
|
|
fixed_quotes = replace_single_quotes(bundle_json)
|
|
if init:
|
|
#print(bundle_json, filename, graph)
|
|
create_graph.json_to_networkx(fixed_quotes, filename, graph)
|
|
init = False
|
|
else:
|
|
create_graph.add_json_to_networkx(fixed_quotes, filename, graph)
|
|
print("Imported: ", filename)
|
|
|
|
return graph
|
|
|
|
def replace_single_quotes(obj):
|
|
if isinstance(obj, str): # If it's a string, replace single quotes
|
|
return obj.replace("'", "''")
|
|
elif isinstance(obj, dict): # If it's a dictionary, process each key-value pair
|
|
return {key: replace_single_quotes(value) for key, value in obj.items()}
|
|
elif isinstance(obj, list): # If it's a list, process each item
|
|
return [replace_single_quotes(item) for item in obj]
|
|
else:
|
|
return obj # Leave other data types unchanged
|
|
|
|
def main():
|
|
#get a list of nodes that should be imported
|
|
## create networkX and run improvement scripts
|
|
print("Creating the graph...", flush=True)
|
|
nx_graph = load_multiple_fhir_bundles('./testData/') # 'mockData' for unit test data, 'testData' for Synthea files
|
|
print(nx_graph)
|
|
|
|
print("Reducing references...", flush=True)
|
|
process_references(nx_graph)
|
|
print(nx_graph)
|
|
|
|
print("Convolute references...", flush=True)
|
|
property_convolution(nx_graph)
|
|
print(nx_graph)
|
|
|
|
print("Generate auto schema...")
|
|
write_automated_schema(nx_graph, 'config/automated_schema.yaml', None)
|
|
|
|
|
|
# create Biocypher driver
|
|
bc = BioCypher(
|
|
biocypher_config_path="config/biocypher_config.yaml",
|
|
)
|
|
|
|
bc.show_ontology_structure()
|
|
|
|
#BioCypher preperation
|
|
## node generator: extract id, label and property dictionary
|
|
def node_generator():
|
|
for node in nx_graph.nodes():
|
|
|
|
""" #single qoutes break neo4j import, e.g. 'CHILDREN'S Hospital'
|
|
checkDisplay = nx_graph.nodes[node].get('display')
|
|
if checkDisplay:
|
|
checkDisplay = checkDisplay.replace("'", "''")
|
|
nx_graph.nodes[node]['display'] = checkDisplay
|
|
#print("------->", nx_graph.nodes[node].get('display'))
|
|
|
|
checkName = nx_graph.nodes[node].get('name')
|
|
if checkName:
|
|
checkName = checkName.replace("'", "''")
|
|
nx_graph.nodes[node]['name'] = checkName
|
|
#print("------->", nx_graph.nodes[node].get('name')) """
|
|
|
|
label = nx_graph.nodes[node].get('label')
|
|
|
|
if label == "resource":
|
|
label = nx_graph.nodes[node].get('resourceType')
|
|
'''
|
|
elif label == 'identifier':
|
|
label = nx_graph.nodes[node].get('system')
|
|
print('/' in label)
|
|
if '/' in label:
|
|
lastSlash = label.rfind('/') + 1
|
|
label = label[lastSlash:] + '-ID'
|
|
elif label == 'telecom':
|
|
label = nx_graph.nodes[node].get('system')
|
|
print('/' in label)
|
|
if '/' in label:
|
|
lastSlash = label.rfind('/') + 1
|
|
label = 'telecom-' + label[lastSlash:]
|
|
elif label == 'address':
|
|
extension = nx_graph.nodes[node].get('extension_url')
|
|
print("EX!: ", extension)
|
|
if extension:
|
|
lastSlash = extension.rfind('/') + 1
|
|
label = label + '-' + extension[lastSlash:]
|
|
'''
|
|
|
|
yield(
|
|
nx_graph.nodes[node].get('id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
|
|
label,
|
|
nx_graph.nodes[node] # get properties
|
|
)
|
|
|
|
def edge_generator():
|
|
for edge in nx_graph.edges(data = True):
|
|
source, target, attributes = edge
|
|
|
|
s_label = nx_graph.nodes[source].get('label')
|
|
if s_label == 'resource':
|
|
s_label = nx_graph.nodes[source].get('resourceType')
|
|
t_label = nx_graph.nodes[target].get('label')
|
|
if t_label == 'resource':
|
|
t_label = nx_graph.nodes[target].get('resourceType')
|
|
label = s_label + '_to_' + t_label
|
|
|
|
yield(
|
|
attributes.get('id', str(uuid.uuid4())), # Edge ID (if exists, otherwise use nx internal id)
|
|
nx_graph.nodes[source].get('id', source),
|
|
nx_graph.nodes[target].get('id', target),
|
|
label,
|
|
attributes # All edge attributes
|
|
)
|
|
|
|
bc.write_nodes(node_generator())
|
|
bc.write_edges(edge_generator())
|
|
|
|
#write the import script
|
|
bc.write_import_call()
|
|
|
|
if __name__ == "__main__":
|
|
#print("Called import script. Should run its main function now...")
|
|
main()
|
|
|