MILA_public/medax_pipeline
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Compare commits

base: MILA_public/medax_pipeline:v1.1.1
MILA_public/medax_pipeline:main
MILA_public/medax_pipeline:v1.1.1 MILA_public/medax_pipeline:v1.0.1 MILA_public/medax_pipeline:release
..
compare: MILA_public/medax_pipeline:v1.0.1
MILA_public/medax_pipeline:main
MILA_public/medax_pipeline:v1.1.1 MILA_public/medax_pipeline:v1.0.1 MILA_public/medax_pipeline:release
4 Commits
v1.1.1 .. v1.0.1
Author SHA1 Message Date
gebhardtt a9c54bfe11 missing file update for v1.0.1 2025-05-12 13:29:31 +02:00
gebhardtt d5c2d2ca64 link fix for large DB imports 2025-05-12 12:42:53 +02:00
gebhardtt faf1ab1c99 Readme update 2025-04-24 15:32:28 +02:00
gebhardtt a9db0be88a release commit 2025-04-16 22:12:19 +02:00
137 changed files with 19349 additions and 643535 deletions
Expand all files Collapse all files

No files matched your search

.bumpversion.cfg
+3 -1
View File
@@ -1,8 +1,10 @@
[bumpversion]
current_version = 1.0.1
current_version = 0.6.0
commit = True
tag = True
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
serialize = {major}.{minor}.{patch}
[bumpversion:file:pyproject.toml]
[bumpversion:file:biocypher/_metadata.py]
.dockerignore
-32
View File
@@ -1,32 +0,0 @@
# Never copy a host virtual environment into the image
.venv
venv
env
# Python caches
__pycache__/
*.py[cod]
*.egg-info/
.pytest_cache/
.mypy_cache/
.ruff_cache/
# VCS / editor
.git
.gitignore
.idea/
.vscode/
# Local env / secrets (manage these via compose env_file or secrets instead)
.env
.env.*
!.env.example
# Docker
Dockerfile
.dockerignore
# Bind-mounted at runtime (see docker-compose.yml) - no need to bake into the image
neo4j_import/
importData/
data/
.env.example
+5 -32
View File
@@ -1,40 +1,13 @@
MODE= #possible values: none or testserver
COMPLEX_PATIENTS= #TRUE if only complex patients should be used otherwise FALSE
COMPLEX_FHIR_SEARCH= #URL for the patient selection on the server, e.g. all gout patients:
IS_SYNTHEA= #TRUE if synthea data is used otherwise FALSE
FHIR_SERVER_URL= #URL to the FHIR server, e.g. http://hapi.fhir.org/baseR4; use http://blaze:8080/fhir when COMPOSE_PROFILES includes blaze
BOLT_ADVERTISED_ADDRESS=localhost:8081 #change in production
NEO4J_AUTH= # set if the server needs authentification: none otherwise true
ENABLE_BOLT_TLS= #true to require TLS on the bolt connector (needs real certs in ./neo4j-certs/bolt/, used in production); leave unset/false for local runs
DISABLE_IMPORT= #false (default): run the complete pipeline (clears ./neo4j_import first); load: only run the neo4j admin import on the existing tables in ./neo4j_import; true: only start neo4j with its existing database
### Settings for deployment - might be adapted depending on the network its deployed in and
MODE=testserver
COMPLEX_PATIENTS=TRUE
FHIR_SERVER_URL=http://hapi.fhir.org/baseR4
#FHIR_SERVER_USER=
#FHIR_SERVER_PW=
#HTTP_PROXY=
#HTTPS_PROXY=
#NO_PROXY=
### Settings for how to get the data
NUMBER_OF_PATIENTS=2 # depends on the goal, very high to get all patients, lower numbers for testing; depends on disk space, plan with 30GB per 60k patients
BATCH_SIZE=10 # depends on the computational power, plan with 12GB per 250 patients; higher batch sizes improve the needed time to run the pipeline
# MDM
MDM_MODE= #biocypher or mdm (beta)
#OAUTH2_COOKIE_SECURE=true # or: NEO4J_COOKIE_SECRET= #for local setup
NUMBER_OF_PATIENTS=100
BATCH_SIZE=35
#Keycloak settings if a local keycloak instance is used
#KEYCLOAK_ISSUER_URL=https://---your productive keycloak url---/realms/yourRealm #if using the setup with a local keycloak use: http://localhost:4040/realms/testPipeline
#NEO4J_OAUTH_CLIENT_ID=neo4j-client
#NEO4J_OAUTH_SECRET=secret-from-production-keycloak or secret from local keycloak setup
#NEO4J_PROXY_REDIRECT_URL=https://your-domain.com/oauth2/callback
COMPOSE_PROFILES= #comma-separated: blaze, keycloak, proxy, server - activates parts of docker compose depending on the setup (see README)
#Test vars
TEST_MODE=FALSE
TEST_DEPTH=4
.gitea/actions/test/action.yaml
+4 -4
View File
@@ -27,14 +27,14 @@ runs:
#----------------------------------------------
- name: Run Tests (Windows)
run: |
uv run pytest --version
uv run pytest --password=your_password_here
poetry run pytest --version
poetry run pytest --password=your_password_here
shell: bash
if: runner.os == 'Windows'
- name: Run tests (Linux and MacOS)
run: |
uv run pytest --version
uv run pytest --password=your_password_here
poetry run pytest --version
poetry run pytest --password=your_password_here
shell: bash
if: runner.os != 'Windows'
- name: Check code quality
.gitignore
-5
View File
@@ -31,8 +31,3 @@ dist/*
.idea/*
.cache
*.iml
# Local env files with real secrets - keep *.example templates tracked
.env
.env.*
!.env.example
.gitmodules
-3
View File
@@ -1,6 +1,3 @@
[submodule "networkx-based"]
path = networkx-based
url = git@git.uni-greifswald.de:MeDaX/networkx-based.git
[submodule "mdm2neo4j"]
path = mdm2neo4j
url = https://git.uni-greifswald.de/guetebierl/mdm2neo4j.git
Dockerfile
+27 -19
View File
@@ -1,28 +1,36 @@
FROM python:3.11-slim
FROM python:3.9-slim
WORKDIR /app
# Bring in the uv binary from Astral's official image (pinned-ish, reproducible)
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
# Copy requirements file if you have one
COPY requirements.txt .
# Compile bytecode for faster cold starts; install into a project-local
# venv at /app/.venv and put it on PATH so `python` resolves to it directly
# (this replaces `poetry config virtualenvs.create false`).
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
VIRTUAL_ENV=/app/.venv \
PATH="/app/.venv/bin:$PATH"
RUN pip install --upgrade packaging
RUN pip install -r requirements.txt
# Install poetry
RUN pip install --no-cache-dir "poetry<2.0.0"
# --- Dependency layer: cached unless these files change ---
COPY pyproject.toml uv.lock ./
# --no-dev installs only the main deps (equivalent to poetry --only main).
# --frozen fails the build instead of silently re-resolving if uv.lock is stale.
RUN uv sync --no-dev --frozen
# --- Application layer ---
# Copy .env file
COPY .env ./
# Copy only pyproject.toml and poetry.lock (if exists) first
COPY pyproject.toml ./
COPY poetry.lock* ./
# Configure poetry to not create a virtual environment inside the container
RUN poetry config virtualenvs.create false
# Install dependencies
RUN poetry install --no-dev --no-interaction --no-ansi
# Copy your project files
COPY . .
# Normalize line endings and make the entrypoint executable
RUN sed -i 's/\r$//' /app/entrypoint.sh \
&& chmod +x /app/entrypoint.sh
# Make the entrypoint script executable
COPY entrypoint.sh .
RUN chmod +x entrypoint.sh
RUN sed -i 's/\r$//' /app/entrypoint.sh
ENTRYPOINT ["/app/entrypoint.sh"]
README.md
+8 -236
View File
@@ -9,15 +9,11 @@ The MeDaX pipeline transforms healthcare data from FHIR databases into Neo4j gra
- Configurable batch processing for handling large datasets
- Docker-based deployment for easy setup and portability
- Compatible with public FHIR servers (e.g., HAPI FHIR) and private authenticated instances
- Optional post-processing queries for patient timelines and shared clinical concepts
## ⚙️ Prerequisites
- [Docker](https://docs.docker.com/engine/install/) with the [Docker Compose plugin](https://docs.docker.com/compose/install/linux/)
- A FHIR database with API access and the `$everything` operation enabled for retrieving patient data
- Alternatively: Use a public FHIR server such as [HAPI FHIR](https://hapi.fhir.org/) (default configuration)
- Optional: [Semspect](https://www.semspect.de/) license
- free scientific licenses can be requested
- helps with graph exploration
## 🚀 Installation
@@ -37,7 +33,6 @@ The MeDaX pipeline transforms healthcare data from FHIR databases into Neo4j gra
### Running the Pipeline
#### Option A: Connect to an existing FHIR server (e.g. HAPI)
**Start the containers:**
```bash
docker compose up --build
@@ -53,95 +48,17 @@ docker compose down --volumes
docker compose down --volumes --rmi all
```
#### Option B: Build a local FHIR server with POLAR data
This spins up a local [Blaze](https://github.com/samply/blaze) FHIR server pre-loaded with synthetic POLAR test bundles, using the `blaze` Compose profile.
**Setup:** in your `.env`, set:
```
COMPOSE_PROFILES=blaze
FHIR_SERVER_URL=http://blaze:8080/fhir
```
**Start the containers:**
```bash
docker compose up --build
```
**Stop and clean up (between runs):**
```bash
docker compose down --volumes
```
**Complete removal (containers and images):**
```bash
docker compose down --volumes --rmi all
```
#### Option C: Build a local FHIR server with POLAR data and a local keycloak server
This builds on Option B and adds a local Keycloak instance plus an OAuth2 proxy in front of Neo4j, using the `blaze`, `keycloak`, and `proxy` Compose profiles.
**Setup:** in your `.env`, set:
```
COMPOSE_PROFILES=blaze,keycloak,proxy
FHIR_SERVER_URL=http://blaze:8080/fhir
```
**Start the containers:**
##### Init containers
```bash
docker compose up --build
```
##### Create realm, client and user
1. Open keycloak in a browser: localhost:4040 and login with user: admin and password: admin
2. Create the realm "testPipeline"
3. Create a client:
* Client ID: neo4j-client
* Always display in UI "on"
* Hit the "Next" button
* Client authentification: on
* Hit the "Next" button
* Root URL: http://localhost:8082
* Home URL: http://localhost:8082
* Valid redirects URIs: http://localhost:8082/oauth2/callback
* Valid post logout redirect URIs: http://localhost:8082/*
* Web origin: http://localhost:8082
* Hit the "Save" button
* Go to the "Credentials" tab
* Copy the "Client Secret" to "NEO4J_OAUTH_SECRET" in your .env
4. Create a user:
* "Email verified": on
* Choose a user name
* Use random mail, e.g.: test@example.com
* Hit the "Create button"
* Click on the new user, go to the "Credentials" tab
* Set a password and disable "Temporary"
##### Restart containers without destroying the volumes
```bash
docker compose down
docker compose up
```
**Stop**
```bash
docker compose down
```
> **Note:** When adding --volumes to down, the keycloak setup will be lost
**Complete removal (containers and images):**
```bash
docker compose down --volumes --rmi all
```
> **Note:** Depending on your Docker installation, you might need to use `docker-compose` instead of `docker compose`.
> **Note:** With `COMPOSE_PROFILES` set in your `.env`, every `docker compose` command automatically includes the right services - no `-f` files needed.
## 🔍 Accessing the Neo4j Database
Once the pipeline has completed processing, you can access the Neo4j database:
1. Open your browser and navigate to `http://localhost:8080/` or `http://localhost:8082/` if you used keycloak (Option C, served via the `proxy` profile's `neo4j-proxy` service)
2. Connect by clicking the button, user name and password are not needed (disabled by config)
1. Open your browser and navigate to `http://localhost:8080/`
2. Connect using the following credentials:
- Username: neo4j
- Password: neo4j
3. Set the new password and save it to a secure password manager
## 📊 Example Queries
@@ -160,147 +77,6 @@ MATCH (m:Medication)-[r]-(p:Patient)
RETURN m, r, p;
```
## 🛠️ Post-processing Options
After the import has finished, you can optionally enrich the graph with additional structure. The queries below are run manually against the Neo4j database, e.g. by pasting them into the Neo4j Browser (see [Accessing the Neo4j Database](#-accessing-the-neo4j-database)).
| Step | Adds | Purpose |
|------|------|---------|
| 1. Patient timeline | `SAME_TIME`, `NEXT_EVENT` relationships | Orders each patient's clinical events chronologically |
| 2. Concept nodes | `:Concept` nodes, `MAPS_TO` relationships | Merges identical codings into shared concept nodes |
| 3. Patient–concept links | `HAS_CONCEPT` relationships | Connects patients directly to the concepts in their record |
**Notes:**
- Step 2 requires the [APOC](https://neo4j.com/docs/apoc/current/) plugin (`apoc.text.join`, `apoc.coll.sort`).
- Step 3 depends on Step 2 and must be run afterwards. Step 1 is independent.
- All queries use `MERGE`, so re-running them does not create duplicate nodes or relationships.
- On large databases these queries can take a while; consider running them on a subset of patients first.
### 1. Patient Timeline Edges
Builds a per-patient timeline from `Condition`, `Observation` and `Procedure` events, using their respective timestamps (`recordedDate`, `effectiveDateTime`, `performedDateTime`). Events without a timestamp are skipped.
- Events sharing the same timestamp are grouped; the first event of each group is linked to the others via `SAME_TIME`.
- The first events of consecutive groups are linked via `NEXT_EVENT`, forming a chronological chain per patient.
```cypher
// CREATE PATIENT TIMELINE EDGES
// Iterate over all patients
MATCH (p:Patient)
// Get events for each patient
OPTIONAL MATCH (p)<--(e)
WHERE e:Condition OR e:Observation OR e:Procedure
// Extract the relevant datetime property for each event
WITH p, e,
CASE
WHEN e:Condition THEN e.recordedDate
WHEN e:Observation THEN e.effectiveDateTime
WHEN e:Procedure THEN e.performedDateTime
END AS eventTime
WHERE eventTime IS NOT NULL
// Normalize time format
WITH p, e, datetime(eventTime) AS dt
ORDER BY p, dt ASC
// Group events per patient and per timestamp
WITH p, dt, collect(e) AS events
ORDER BY p, dt ASC
WITH p, collect({time: dt, events: events}) AS timeline
// Create SAME_TIME edges
CALL {
WITH timeline
UNWIND timeline AS group
WITH group.events AS events
WITH events[0] AS firstNode, events[1..] AS others
UNWIND others AS o
MERGE (firstNode)-[:SAME_TIME]->(o)
}
// Create NEXT_EVENT
CALL {
WITH timeline
UNWIND range(0, size(timeline)-2) AS i
WITH timeline[i].events[0] AS firstCurr,
timeline[i+1].events[0] AS firstNext
MERGE (firstCurr)-[:NEXT_EVENT]->(firstNext)
}
```
**Example:** walk a patient's timeline in order:
```cypher
MATCH path = (start)-[:NEXT_EVENT*]->(end)
WHERE (start)-->(:Patient {id: 'patient-id'})
AND NOT ()-[:NEXT_EVENT]->(start)
AND NOT (end)-[:NEXT_EVENT]->()
RETURN path;
```
### 2. Concept Nodes
Collects all codings (`*_code`, `*_system`, `*_display` property triples) of each `Code` node and creates a shared `Concept` node for every distinct combination of `system::code` pairs. Resources coded identically — across patients — then point to the same concept.
- `Concept.id`: sorted `system::code` pairs joined with `||`
- `Concept.displays`: the human-readable display texts
- `Concept.type`: the label of the resource the code belongs to (e.g. `Condition`)
```cypher
// CREATE CONCEPT NODES
MATCH (parent)-[]->(code:Code)
UNWIND keys(code) AS k
WITH parent, code, k
WHERE k ENDS WITH '_code'
// Extract values for coding prefix
WITH parent, code,
code[k] AS codeValue,
code[replace(k, '_code', '_system')] AS systemValue,
code[replace(k, '_code', '_display')] AS displayValue
WHERE codeValue IS NOT NULL AND systemValue IS NOT NULL
// Collect all coding pairs + displays
WITH parent, code,
collect(systemValue + '::' + codeValue) AS pairIds,
[d IN collect(displayValue) WHERE d IS NOT NULL] AS displays
// Create Concept IDs
WITH parent, code, displays,
apoc.text.join(apoc.coll.sort(pairIds), '||') AS conceptId,
labels(parent)[-1] AS conceptType
// Create shared Concept node
MERGE (c:Concept {id: conceptId})
SET c.displays = displays,
c.type = conceptType
// Connect Code to Concept
MERGE (code)-[:MAPS_TO]->(c);
```
### 3. Connect Patients to Concepts
Adds a direct `HAS_CONCEPT` shortcut from each patient to every concept found in their record, which makes cohort-style queries much simpler.
```cypher
// CONNECT PATIENT NODES TO CONCEPTS
MATCH (p:Patient)<-[]-(parent)-[]->(code:Code)-[:MAPS_TO]->(c:Concept)
MERGE (p)-[:HAS_CONCEPT]->(c);
```
**Example:** find the most frequent concepts across all patients:
```cypher
MATCH (p:Patient)-[:HAS_CONCEPT]->(c:Concept)
RETURN c.type AS type, c.displays AS displays, count(DISTINCT p) AS patients
ORDER BY patients DESC
LIMIT 25;
```
## ❓ Troubleshooting
**Common Issues:**
@@ -308,8 +84,7 @@ LIMIT 25;
- **Connection refused to FHIR server**: Check your network settings and ensure the FHIR server is accessible from within the Docker container.
- **Authentication failures**: Verify your credentials in the `.env` file.
- **Container startup failures**: Ensure all required Docker ports are available and not used by other applications.
- **No data found in fhir bundle**: Ensure that the FHIR server is up and responding patient data. Try set the COMPLEX_PATIENTS variable to FALSE in your .env file. Some FHIR servers might not support the FHIR search logic.
- **`Unknown function 'apoc.text.join'` during post-processing**: The APOC plugin is not installed or enabled in your Neo4j instance.
- **No data found in fhir bundle**: Ensure that the FHIR server is up and responding patient data. Try sett the COMPLEX_PATIENTS variable to FALSE in your .env file. Some FHIR servers might not support the FHIR search logic.
## 📚 Architecture
@@ -318,9 +93,8 @@ The MeDaX pipeline consists of the following components:
1. **FHIR Client**: Connects to the FHIR server and retrieves patient data
2. **Data Transformer**: Converts FHIR resources into graph entities and relationships
3. **Reference Processor**: Converts references to relationships
4. **BioCypher Adapter**: Prepares the transformed data for Neo4j admin import
5. **Neo4j Database**: Stores and serves the graph representation of the health data
6. **Post-processing (optional)**: Cypher queries that add timelines and shared concept nodes
3. **BioCypher Adapter**: Prepares the transformed data for Neo4j admin import
4. **Neo4j Database**: Stores and serves the graph representation of the health data
## ✍️ Citation
@@ -333,5 +107,3 @@ If you use the MeDaX pipeline in your research, please cite: 10.5281/zenodo.1522
- We used BioCypher's git template as a starting point for our development:
- Lobentanzer, S., BioCypher Consortium, & Saez-Rodriguez, J. Democratizing knowledge representation with BioCypher [Computer software]. https://github.com/biocypher/biocypher
- We used synthetic data generated with [Synthea](https://doi.org/10.1093/jamia/ocx079) during the development process. This data is provided in the testData folder.
- We are using the [HAPI](https://hapifhir.io/) R4 server while developing and showcasing the capabilities of our tool.
- This project has been funded by the BMBF, FKZ: 01ZZ2019.
biocypher/__init__.py
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher: a unifying framework for biomedical knowledge graphs.
"""
__all__ = [
"__version__",
"__author__",
"module_data",
"config",
"logfile",
"log",
"Driver",
"BioCypher",
"Resource",
]
from ._get import Resource
from ._core import BioCypher
from ._config import config, module_data
from ._logger import log, logger, logfile
from ._metadata import __author__, __version__
class Driver(BioCypher):
# initialise parent class but log a warning
def __init__(self, *args, **kwargs):
logger.warning(
"The class `Driver` is deprecated and will be removed in a future "
"release. Please use `BioCypher` instead."
)
super().__init__(*args, **kwargs)
biocypher/_config/__init__.py
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
Module data directory, including:
* The BioLink database schema
* The default config files
"""
from typing import Any, Optional
import os
import warnings
import yaml
import appdirs
__all__ = ["module_data", "module_data_path", "read_config", "config", "reset"]
_USER_CONFIG_DIR = appdirs.user_config_dir("biocypher", "saezlab")
_USER_CONFIG_FILE = os.path.join(_USER_CONFIG_DIR, "conf.yaml")
class MyLoader(yaml.SafeLoader):
def construct_scalar(self, node):
# Check if the scalar contains double quotes and an escape sequence
value = super().construct_scalar(node)
q = bool(node.style == '"')
b = bool("\\" in value.encode("unicode_escape").decode("utf-8"))
if q and b:
warnings.warn(
(
"Double quotes detected in YAML configuration scalar: "
f"{value.encode('unicode_escape')}. "
"These allow escape sequences and may cause problems, for "
"instance with the Neo4j admin import files (e.g. '\\t'). "
"Make sure you wanted to do this, and use single quotes "
"whenever possible."
),
category=UserWarning,
)
return value
def module_data_path(name: str) -> str:
"""
Absolute path to a YAML file shipped with the module.
"""
here = os.path.dirname(os.path.abspath(__file__))
return os.path.join(here, f"{name}.yaml")
def module_data(name: str) -> Any:
"""
Retrieve the contents of a YAML file shipped with this module.
"""
path = module_data_path(name)
return _read_yaml(path)
def _read_yaml(path: str) -> Optional[dict]:
if os.path.exists(path):
with open(path, "r") as fp:
return yaml.load(fp.read(), Loader=MyLoader)
def read_config() -> dict:
"""
Read the module config.
Read and merge the built-in default, the user level and directory level
configuration, with the later taking precendence over the former.
TODO explain path configuration
"""
defaults = module_data("biocypher_config")
user = _read_yaml(_USER_CONFIG_FILE) or {}
# TODO account for .yml?
local = (
_read_yaml("biocypher_config.yaml")
or _read_yaml("config/biocypher_config.yaml")
or {}
)
for key in defaults:
value = (
local[key] if key in local else user[key] if key in user else None
)
if value is not None:
if isinstance(
defaults[key], str
): # first level config (like title)
defaults[key] = value
else:
defaults[key].update(value)
return defaults
def config(*args, **kwargs) -> Optional[Any]:
"""
Set or get module config parameters.
"""
if args and kwargs:
raise ValueError(
"Setting and getting values in the same call is not allowed.",
)
if args:
result = tuple(globals()["_config"].get(key, None) for key in args)
return result[0] if len(result) == 1 else result
for key, value in kwargs.items():
globals()["_config"][key].update(value)
def reset():
"""
Reload configuration from the config files.
"""
globals()["_config"] = read_config()
reset()
def update_from_file(path: str):
"""
Update the module configuration from a YAML file.
"""
config(**_read_yaml(path))
biocypher/_config/biocypher_config.yaml
+141
View File
@@ -0,0 +1,141 @@
Title: BioCypher python module configuration file
## Some options are not used by default. Uncomment them to use them.
biocypher:
### Required parameters ###
## DBMS type
dbms: neo4j
## Schema configuration
# schema_config_path: config/schema_config.yaml
## Offline mode: do not connect to a running DBMS instance
## Can be used e.g. for writing batch import files
offline: true
## Strict mode: do not allow to create new nodes or relationships without
## specifying source, version, and license parameters
strict_mode: false
## Ontology configuration
head_ontology:
url: https://github.com/biolink/biolink-model/raw/v3.2.1/biolink-model.owl.ttl
root_node: entity
# switch_label_and_id: true
### Optional parameters ###
## Logging
# Write log to disk
log_to_disk: true
# Activate more granular logging
debug: true
# Change the log directory
# log_directory: biocypher-log
## Data output directory
# output_directory: biocypher-out
## Resource cache directory
# cache_directory: .cache
## Optional tail ontologies
# tail_ontologies:
# so:
# url: test/ontologies/so.owl
# head_join_node: sequence variant
# tail_join_node: sequence_variant
# switch_label_and_id: true
# mondo:
# url: test/ontologies/mondo.owl
# head_join_node: disease
# tail_join_node: disease
# switch_label_and_id: true
### DBMS configuration ###
neo4j:
### Neo4j configuration ###
## Database name
database_name: neo4j
## Wipe DB before import (offline mode: --force)
wipe: true
## Neo4j authentication
uri: neo4j://localhost:7687
user: neo4j
password: neo4j
## Neo4j admin import batch writer settings
delimiter: ";"
array_delimiter: "|"
quote_character: "'"
## MultiDB functionality
## Set to false for using community edition or older versions of Neo4j
multi_db: true
## Import options
skip_duplicate_nodes: false
skip_bad_relationships: false
## Import call prefixes
# import_call_bin_prefix: bin/
# import_call_file_prefix: path/to/files/
postgresql:
### PostgreSQL configuration ###
# PostgreSQL connection credentials
database_name: postgres # DB name
user: postgres # user name
password: postgres # password
host: localhost # host
port: 5432 # port
# PostgreSQL import batch writer settings
quote_character: '"'
delimiter: '\t'
# import_call_bin_prefix: '' # path to "psql"
# import_call_file_prefix: '/path/to/files'
rdf:
### RDF configuration ###
rdf_format: turtle
sqlite:
### SQLite configuration ###
# SQLite connection credentials
database_name: sqlite.db # DB name
# SQLite import batch writer settings
quote_character: '"'
delimiter: '\t'
# import_call_bin_prefix: '' # path to "sqlite3"
# import_call_file_prefix: '/path/to/files'
csv:
### CSV/Pandas configuration ###
delimiter: ","
networkx:
### NetworkX configuration ###
some_config: some_value # placeholder for technical reasons TODO
biocypher/_config/test_config.yaml
+5
View File
@@ -0,0 +1,5 @@
# We test the quote detection
valid: 'This is a valid string'
also_valid: "This is also a valid string"
invalid: "\t"
biocypher/_config/test_schema_config.yaml
+140
View File
@@ -0,0 +1,140 @@
Title: BioCypher graph schema configuration file
# ---
# "Named Things"
# ---
protein:
represented_as: node
preferred_id: uniprot
input_label: protein
db_collection_name: proteins
properties:
name: str
score: float
taxon: int
genes: str[]
microRNA:
represented_as: node
preferred_id: mirbase.mature
input_label: mirna
complex:
synonym_for: macromolecular complex
represented_as: node
preferred_id: complexportal
input_label: complex
pathway:
represented_as: node
preferred_id: [reactome, wikipathways]
input_label: [reactome, wikipathways]
gene:
represented_as: node
preferred_id: hgnc
input_label: [hgnc, ensg]
exclude_properties: accession
disease:
represented_as: node
preferred_id: doid
input_label: Disease
side effect:
is_a: phenotypic feature
represented_as: node
preferred_id: sider.effect
input_label: sider
sequence variant:
represented_as: node
preferred_id: [clinically relevant, known, somatic]
input_label: [Clinically_relevant_variant, Known_variant, Somatic_mutation]
properties:
source: str
original_source: str
effect: str
biotype: str
snRNA sequence:
is_a: nucleic acid entity
represented_as: node
preferred_id: [intact, rnacentral]
input_label: [intact_snrna, rnacentral_snrna]
properties:
ac: str
fullName: str
shortName: str
preferredName: str
exclude_properties: sequence
DNA sequence:
is_a: nucleic acid entity
represented_as: node
preferred_id: ensembl
input_label: dna
properties:
ac: str
fullName: str
shortName: str
preferredName: str
sequence: str
dsDNA sequence:
is_a: [DNA sequence, nucleic acid entity]
inherit_properties: True
represented_as: node
preferred_id: [intact, uniparc]
input_label: [intact_dsdna, uniprot_archive_dsdna]
# ---
# Associations
# ---
post translational interaction:
is_a: pairwise molecular interaction
represented_as: node
label_as_edge: INTERACTS_POST_TRANSLATIONAL
input_label: post_translational
phosphorylation:
is_a: post translational interaction
represented_as: edge
input_label: phosphorylation
gene to disease association:
represented_as: edge
label_as_edge: PERTURBED_IN_DISEASE
input_label: [protein_disease, gene_disease]
exclude_properties: accession
mutation to tissue association:
is_a: [genotype to tissue association, entity to tissue association, association]
represented_as: edge
label_as_edge: Is_Mutated_In
input_label: Gene_Is_Mutated_In_Cell_Tissue
variant to gene association: # -> Known.... and Somatic....
represented_as: edge
source: [known.sequence variant, somatic.sequence variant]
target: gene
input_label: [
VARIANT_FOUND_IN_GENE_Known_variant_Gene,
VARIANT_FOUND_IN_GENE_Somatic_mutation_Gene
]
gene to gene association:
represented_as: edge
input_label: gene_gene
properties:
directional: bool
curated: bool
score: float
id: str # should be removed
gene to variant association: # should be removed
is_a: gene to variant association
represented_as: edge
input_label: gene_variant
biocypher/_config/test_schema_config_disconnected.yaml
+3
View File
@@ -0,0 +1,3 @@
disconnected:
represented_as: node
label_in_input: disconnected
biocypher/_config/test_schema_config_extended.yaml
+152
View File
@@ -0,0 +1,152 @@
Title: BioCypher graph schema configuration file
# ---
# "Named Things"
# ---
protein:
represented_as: node
preferred_id: uniprot
input_label: protein
db_collection_name: proteins
properties:
name: str
score: float
taxon: int
genes: str[]
microRNA:
represented_as: node
preferred_id: mirbase.mature
input_label: mirna
complex:
synonym_for: macromolecular complex
represented_as: node
preferred_id: complexportal
input_label: complex
pathway:
represented_as: node
preferred_id: [reactome, wikipathways]
input_label: [reactome, wikipathways]
gene:
represented_as: node
preferred_id: hgnc
input_label: [hgnc, ensg]
exclude_properties: accession
disease:
represented_as: node
preferred_id: doid
input_label: Disease
side effect:
is_a: phenotypic feature
represented_as: node
preferred_id: sider.effect
input_label: sider
sequence variant:
represented_as: node
preferred_id: [clinically relevant, known, somatic]
input_label: [Clinically_relevant_variant, Known_variant, Somatic_mutation]
properties:
source: str
original_source: str
effect: str
biotype: str
altered gene product level:
represented_as: node
input_label: agpl
decreased gene product level:
represented_as: node
input_label: agpl_decreased
lethal variant:
represented_as: node
input_label: lethal
snRNA sequence:
is_a: nucleic acid entity
represented_as: node
preferred_id: [intact, rnacentral]
input_label: [intact_snrna, rnacentral_snrna]
properties:
ac: str
fullName: str
shortName: str
preferredName: str
exclude_properties: sequence
DNA sequence:
is_a: nucleic acid entity
represented_as: node
preferred_id: ensembl
input_label: dna
properties:
ac: str
fullName: str
shortName: str
preferredName: str
sequence: str
dsDNA sequence:
is_a: [DNA sequence, nucleic acid entity]
inherit_properties: True
represented_as: node
preferred_id: [intact, uniparc]
input_label: [intact_dsdna, uniprot_archive_dsdna]
# ---
# Associations
# ---
post translational interaction:
is_a: pairwise molecular interaction
represented_as: node
label_as_edge: INTERACTS_POST_TRANSLATIONAL
input_label: post_translational
phosphorylation:
is_a: post translational interaction
represented_as: edge
use_id: false
input_label: phosphorylation
gene to disease association:
represented_as: edge
label_as_edge: PERTURBED_IN_DISEASE
input_label: [protein_disease, gene_disease]
exclude_properties: accession
mutation to tissue association:
is_a: [genotype to tissue association, entity to tissue association, association]
represented_as: edge
label_as_edge: Is_Mutated_In
input_label: Gene_Is_Mutated_In_Cell_Tissue
variant to gene association: # -> Known.... and Somatic....
represented_as: edge
source: [known.sequence variant, somatic.sequence variant]
target: gene
input_label: [
VARIANT_FOUND_IN_GENE_Known_variant_Gene,
VARIANT_FOUND_IN_GENE_Somatic_mutation_Gene
]
gene to gene association:
represented_as: edge
input_label: gene_gene
properties:
directional: bool
curated: bool
score: float
gene to variant association:
is_a: gene to variant association
represented_as: edge
input_label: gene_variant
biocypher/_core.py
+734
View File
@@ -0,0 +1,734 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher core module. Interfaces with the user and distributes tasks to
submodules.
"""
from typing import Optional
from datetime import datetime
import os
import json
from more_itertools import peekable
import yaml
import pandas as pd
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from ._get import Downloader
from ._config import config as _config
from ._config import update_from_file as _file_update
from ._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
from ._mapping import OntologyMapping
from ._ontology import Ontology
from ._translate import Translator
from ._deduplicate import Deduplicator
from .output.in_memory._pandas import Pandas
from .output.write._get_writer import DBMS_TO_CLASS, get_writer
from .output.connect._neo4j_driver import get_driver
__all__ = ["BioCypher"]
SUPPORTED_DBMS = DBMS_TO_CLASS.keys()
REQUIRED_CONFIG = [
"dbms",
"offline",
"strict_mode",
"head_ontology",
]
class BioCypher:
"""
Orchestration of BioCypher operations. Instantiate this class to interact
with BioCypher.
Args:
dbms (str): The database management system to use. For supported
systems see SUPPORTED_DBMS.
offline (bool): Whether to run in offline mode. If True, no
connection to the database will be made.
strict_mode (bool): Whether to run in strict mode. If True, the
translator will raise an error if a node or edge does not
provide source, version, and licence information.
biocypher_config_path (str): Path to the BioCypher config file.
schema_config_path (str): Path to the user schema config
file.
head_ontology (dict): The head ontology defined by URL ('url') and root
node ('root_node').
tail_ontologies (dict): The tail ontologies defined by URL and
join nodes for both head and tail ontology.
output_directory (str): Path to the output directory. If not
provided, the default value 'biocypher-out' will be used.
"""
def __init__(
self,
dbms: str = None,
offline: bool = None,
strict_mode: bool = None,
biocypher_config_path: str = None,
schema_config_path: str = None,
head_ontology: dict = None,
tail_ontologies: dict = None,
output_directory: str = None,
cache_directory: str = None,
# legacy params
db_name: str = None,
):
# Update configuration if custom path is provided
if biocypher_config_path:
_file_update(biocypher_config_path)
if db_name:
logger.warning(
"The parameter `db_name` is deprecated. Please set the "
"`database_name` setting in the `biocypher_config.yaml` file "
"instead."
)
_config(**{db_name: {"database_name": db_name}})
# Load configuration
self.base_config = _config("biocypher")
# Check for required configuration
for key in REQUIRED_CONFIG:
if key not in self.base_config:
raise ValueError(f"Configuration key {key} is required.")
# Set configuration - mandatory
self._dbms = dbms or self.base_config["dbms"]
if offline is None:
self._offline = self.base_config["offline"]
else:
self._offline = offline
if strict_mode is None:
self._strict_mode = self.base_config["strict_mode"]
else:
self._strict_mode = strict_mode
self._schema_config_path = schema_config_path or self.base_config.get(
"schema_config_path"
)
if not self._schema_config_path:
logger.warning("Running BioCypher without schema configuration.")
else:
logger.info(
f"Running BioCypher with schema configuration from {self._schema_config_path}."
)
self._head_ontology = head_ontology or self.base_config["head_ontology"]
# Set configuration - optional
self._output_directory = output_directory or self.base_config.get(
"output_directory"
)
self._cache_directory = cache_directory or self.base_config.get(
"cache_directory"
)
self._tail_ontologies = tail_ontologies or self.base_config.get(
"tail_ontologies"
)
if self._dbms not in SUPPORTED_DBMS:
raise ValueError(
f"DBMS {self._dbms} not supported. "
f"Please select from {SUPPORTED_DBMS}."
)
# Initialize
self._ontology_mapping = None
self._deduplicator = None
self._translator = None
self._downloader = None
self._ontology = None
self._writer = None
self._pd = None
def _get_deduplicator(self) -> Deduplicator:
"""
Create deduplicator if not exists and return.
"""
if not self._deduplicator:
self._deduplicator = Deduplicator()
return self._deduplicator
def _get_ontology_mapping(self) -> OntologyMapping:
"""
Create ontology mapping if not exists and return.
"""
if not self._schema_config_path:
self._ontology_mapping = OntologyMapping()
if not self._ontology_mapping:
self._ontology_mapping = OntologyMapping(
config_file=self._schema_config_path,
)
return self._ontology_mapping
def _get_ontology(self) -> Ontology:
"""
Create ontology if not exists and return.
"""
if not self._ontology:
self._ontology = Ontology(
ontology_mapping=self._get_ontology_mapping(),
head_ontology=self._head_ontology,
tail_ontologies=self._tail_ontologies,
)
return self._ontology
def _get_translator(self) -> Translator:
"""
Create translator if not exists and return.
"""
if not self._translator:
self._translator = Translator(
ontology=self._get_ontology(),
strict_mode=self._strict_mode,
)
return self._translator
def _get_writer(self):
"""
Create writer if not online. Set as instance variable `self._writer`.
"""
if self._offline:
timestamp = lambda: datetime.now().strftime("%Y%m%d%H%M%S")
outdir = self._output_directory or os.path.join(
"biocypher-out", timestamp()
)
self._output_directory = os.path.abspath(outdir)
self._writer = get_writer(
dbms=self._dbms,
translator=self._get_translator(),
deduplicator=self._get_deduplicator(),
output_directory=self._output_directory,
strict_mode=self._strict_mode,
)
else:
raise NotImplementedError("Cannot get writer in online mode.")
def _get_driver(self):
"""
Create driver if not exists. Set as instance variable `self._driver`.
"""
if not self._offline:
self._driver = get_driver(
dbms=self._dbms,
translator=self._get_translator(),
deduplicator=self._get_deduplicator(),
)
else:
raise NotImplementedError("Cannot get driver in offline mode.")
def write_nodes(
self, nodes, batch_size: int = int(1e6), force: bool = False
) -> bool:
"""
Write nodes to database. Either takes an iterable of tuples (if given,
translates to ``BioCypherNode`` objects) or an iterable of
``BioCypherNode`` objects.
Args:
nodes (iterable): An iterable of nodes to write to the database.
batch_size (int): The batch size to use when writing to disk.
force (bool): Whether to force writing to the output directory even
if the node type is not present in the schema config file.
Returns:
bool: True if successful.
"""
if not self._writer:
self._get_writer()
nodes = peekable(nodes)
if not isinstance(nodes.peek(), BioCypherNode):
tnodes = self._translator.translate_nodes(nodes)
else:
tnodes = nodes
# write node files
return self._writer.write_nodes(
tnodes, batch_size=batch_size, force=force
)
def write_edges(self, edges, batch_size: int = int(1e6)) -> bool:
"""
Write edges to database. Either takes an iterable of tuples (if given,
translates to ``BioCypherEdge`` objects) or an iterable of
``BioCypherEdge`` objects.
Args:
edges (iterable): An iterable of edges to write to the database.
Returns:
bool: True if successful.
"""
if not self._writer:
self._get_writer()
edges = peekable(edges)
if not isinstance(edges.peek(), BioCypherEdge):
tedges = self._translator.translate_edges(edges)
else:
tedges = edges
# write edge files
return self._writer.write_edges(tedges, batch_size=batch_size)
def to_df(self) -> list[pd.DataFrame]:
"""
Convert entities to a pandas DataFrame for each entity type and return
a list.
Args:
entities (iterable): An iterable of entities to convert to a
DataFrame.
Returns:
pd.DataFrame: A pandas DataFrame.
"""
if not self._pd:
raise ValueError(
"No pandas instance found. Please call `add()` first."
)
return self._pd.dfs
def add(self, entities) -> None:
"""
Function to add entities to the in-memory database. Accepts an iterable
of tuples (if given, translates to ``BioCypherNode`` or
``BioCypherEdge`` objects) or an iterable of ``BioCypherNode`` or
``BioCypherEdge`` objects.
Args:
entities (iterable): An iterable of entities to add to the database.
Can be 3-tuples (nodes) or 5-tuples (edges); also accepts
4-tuples for edges (deprecated).
Returns:
None
"""
if not self._pd:
self._pd = Pandas(
translator=self._get_translator(),
deduplicator=self._get_deduplicator(),
)
entities = peekable(entities)
if (
isinstance(entities.peek(), BioCypherNode)
or isinstance(entities.peek(), BioCypherEdge)
or isinstance(entities.peek(), BioCypherRelAsNode)
):
tentities = entities
elif len(entities.peek()) < 4:
tentities = self._translator.translate_nodes(entities)
else:
tentities = self._translator.translate_edges(entities)
self._pd.add_tables(tentities)
def add_nodes(self, nodes) -> None:
"""
Wrapper for ``add()`` to add nodes to the in-memory database.
Args:
nodes (iterable): An iterable of node tuples to add to the database.
Returns:
None
"""
self.add(nodes)
def add_edges(self, edges) -> None:
"""
Wrapper for ``add()`` to add edges to the in-memory database.
Args:
edges (iterable): An iterable of edge tuples to add to the database.
Returns:
None
"""
self.add(edges)
def merge_nodes(self, nodes) -> bool:
"""
Merge nodes into database. Either takes an iterable of tuples (if given,
translates to ``BioCypherNode`` objects) or an iterable of
``BioCypherNode`` objects.
Args:
nodes (iterable): An iterable of nodes to merge into the database.
Returns:
bool: True if successful.
"""
if not self._driver:
self._get_driver()
nodes = peekable(nodes)
if not isinstance(nodes.peek(), BioCypherNode):
tnodes = self._translator.translate_nodes(nodes)
else:
tnodes = nodes
# write node files
return self._driver.add_biocypher_nodes(tnodes)
def merge_edges(self, edges) -> bool:
"""
Merge edges into database. Either takes an iterable of tuples (if given,
translates to ``BioCypherEdge`` objects) or an iterable of
``BioCypherEdge`` objects.
Args:
edges (iterable): An iterable of edges to merge into the database.
Returns:
bool: True if successful.
"""
if not self._driver:
self._get_driver()
edges = peekable(edges)
if not isinstance(edges.peek(), BioCypherEdge):
tedges = self._translator.translate_edges(edges)
else:
tedges = edges
# write edge files
return self._driver.add_biocypher_edges(tedges)
# DOWNLOAD AND CACHE MANAGEMENT METHODS ###
def _get_downloader(self, cache_dir: Optional[str] = None):
"""
Create downloader if not exists.
"""
if not self._downloader:
self._downloader = Downloader(self._cache_directory)
def download(self, *resources) -> None:
"""
Use the :class:`Downloader` class to download or load from cache the
resources given by the adapter.
"""
self._get_downloader()
return self._downloader.download(*resources)
# OVERVIEW AND CONVENIENCE METHODS ###
def log_missing_input_labels(self) -> Optional[dict[str, list[str]]]:
"""
Get the set of input labels encountered without an entry in the
`schema_config.yaml` and print them to the logger.
Returns:
Optional[Dict[str, List[str]]]: A dictionary of Biolink types
encountered without an entry in the `schema_config.yaml` file.
"""
mt = self._translator.get_missing_biolink_types()
if mt:
msg = (
"Input entities not accounted for due to them not being "
f"present in the schema configuration file {self._schema_config_path} "
"(this is not necessarily a problem, if you did not intend "
"to include them in the database; see the log for details): \n"
)
for k, v in mt.items():
msg += f" {k}: {v} \n"
logger.info(msg)
return mt
else:
logger.info("No missing labels in input.")
return None
def log_duplicates(self) -> None:
"""
Get the set of duplicate nodes and edges encountered and print them to
the logger.
"""
dn = self._deduplicator.get_duplicate_nodes()
if dn:
ntypes = dn[0]
nids = dn[1]
msg = "Duplicate node types encountered (IDs in log): \n"
for typ in ntypes:
msg += f" {typ}\n"
logger.info(msg)
idmsg = "Duplicate node IDs encountered: \n"
for _id in nids:
idmsg += f" {_id}\n"
logger.debug(idmsg)
else:
logger.info("No duplicate nodes in input.")
de = self._deduplicator.get_duplicate_edges()
if de:
etypes = de[0]
eids = de[1]
msg = "Duplicate edge types encountered (IDs in log): \n"
for typ in etypes:
msg += f" {typ}\n"
logger.info(msg)
idmsg = "Duplicate edge IDs encountered: \n"
for _id in eids:
idmsg += f" {_id}\n"
logger.debug(idmsg)
else:
logger.info("No duplicate edges in input.")
def show_ontology_structure(self, **kwargs) -> None:
"""
Show the ontology structure using treelib or write to GRAPHML file.
Args:
to_disk (str): If specified, the ontology structure will be saved
to disk as a GRAPHML file, to be opened in your favourite
graph visualisation tool.
full (bool): If True, the full ontology structure will be shown,
including all nodes and edges. If False, only the nodes and
edges that are relevant to the extended schema will be shown.
"""
if not self._ontology:
self._get_ontology()
return self._ontology.show_ontology_structure(**kwargs)
def write_import_call(self) -> str:
"""
Write a shell script to import the database depending on the chosen
DBMS.
Returns:
str: path toward the file holding the import call.
"""
if not self._offline:
raise NotImplementedError(
"Cannot write import call in online mode."
)
return self._writer.write_import_call()
def write_schema_info(self, as_node: bool = False) -> None:
"""
Write an extended schema info YAML file that extends the
`schema_config.yaml` with run-time information of the built KG. For
instance, include information on whether something present in the actual
knowledge graph, whether it is a relationship (which is important in the
case of representing relationships as nodes) and the actual sources and
targets of edges. Since this file can be used in place of the original
`schema_config.yaml` file, it indicates that it is the extended schema
by setting `is_schema_info` to `true`.
We start by using the `extended_schema` dictionary from the ontology
class instance, which contains all expanded entities and relationships.
The information of whether something is a relationship can be gathered
from the deduplicator instance, which keeps track of all entities that
have been seen.
"""
if not self._offline:
raise NotImplementedError(
"Cannot write schema info in online mode."
)
ontology = self._get_ontology()
schema = ontology.mapping.extended_schema.copy()
schema["is_schema_info"] = True
deduplicator = self._get_deduplicator()
for node in deduplicator.entity_types:
if node in schema.keys():
schema[node]["present_in_knowledge_graph"] = True
schema[node]["is_relationship"] = False
else:
logger.info(
f"Node {node} not present in extended schema. "
"Skipping schema info."
)
# find 'label_as_edge' cases in schema entries
changed_labels = {}
for k, v in schema.items():
if not isinstance(v, dict):
continue
if "label_as_edge" in v.keys():
if v["label_as_edge"] in deduplicator.seen_relationships.keys():
changed_labels[v["label_as_edge"]] = k
for edge in deduplicator.seen_relationships.keys():
if edge in changed_labels.keys():
edge = changed_labels[edge]
if edge in schema.keys():
schema[edge]["present_in_knowledge_graph"] = True
schema[edge]["is_relationship"] = True
# TODO information about source and target nodes
else:
logger.info(
f"Edge {edge} not present in extended schema. "
"Skipping schema info."
)
# write to output directory as YAML file
path = os.path.join(self._output_directory, "schema_info.yaml")
with open(path, "w") as f:
f.write(yaml.dump(schema))
if as_node:
# write as node
node = BioCypherNode(
node_id="schema_info",
node_label="schema_info",
properties={"schema_info": json.dumps(schema)},
)
self.write_nodes([node], force=True)
# override import call with added schema info node
self.write_import_call()
return schema
# TRANSLATION METHODS ###
def translate_term(self, term: str) -> str:
"""
Translate a term to its BioCypher equivalent.
Args:
term (str): The term to translate.
Returns:
str: The BioCypher equivalent of the term.
"""
# instantiate adapter if not exists
self.start_ontology()
return self._translator.translate_term(term)
def summary(self) -> None:
"""
Wrapper for showing ontology structure and logging duplicates and
missing input types.
"""
self.show_ontology_structure()
self.log_duplicates()
self.log_missing_input_labels()
def reverse_translate_term(self, term: str) -> str:
"""
Reverse translate a term from its BioCypher equivalent.
Args:
term (str): The BioCypher term to reverse translate.
Returns:
str: The original term.
"""
# instantiate adapter if not exists
self.start_ontology()
return self._translator.reverse_translate_term(term)
def translate_query(self, query: str) -> str:
"""
Translate a query to its BioCypher equivalent.
Args:
query (str): The query to translate.
Returns:
str: The BioCypher equivalent of the query.
"""
# instantiate adapter if not exists
self.start_ontology()
return self._translator.translate(query)
def reverse_translate_query(self, query: str) -> str:
"""
Reverse translate a query from its BioCypher equivalent.
Args:
query (str): The BioCypher query to reverse translate.
Returns:
str: The original query.
"""
# instantiate adapter if not exists
self.start_ontology()
return self._translator.reverse_translate(query)
biocypher/_create.py
+356
View File
@@ -0,0 +1,356 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'create' module. Handles the creation of BioCypher node and edge
dataclasses.
"""
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from typing import Union
from dataclasses import field, dataclass
import os
__all__ = [
"BioCypherEdge",
"BioCypherNode",
"BioCypherRelAsNode",
]
@dataclass(frozen=True)
class BioCypherNode:
"""
Handoff class to represent biomedical entities as Neo4j nodes.
Has id, label, property dict; id and label (in the Neo4j sense of a
label, ie, the entity descriptor after the colon, such as
":Protein") are non-optional and called node_id and node_label to
avoid confusion with "label" properties. Node labels are written in
PascalCase and as nouns, as per Neo4j consensus.
Args:
node_id (string): consensus "best" id for biological entity
node_label (string): primary type of entity, capitalised
**properties (kwargs): collection of all other properties to be
passed to neo4j for the respective node (dict)
Todo:
- check and correct small inconsistencies such as capitalisation
of ID names ("uniprot" vs "UniProt")
- check for correct ID patterns (eg "ENSG" + string of numbers,
uniprot length)
- ID conversion using pypath translation facilities for now
"""
node_id: str
node_label: str
preferred_id: str = "id"
properties: dict = field(default_factory=dict)
def __post_init__(self):
"""
Add id field to properties.
Check for reserved keywords.
Replace unwanted characters in properties.
"""
self.properties["id"] = self.node_id
self.properties["preferred_id"] = self.preferred_id or None
# TODO actually make None possible here; as is, "id" is the default in
# the dataclass as well as in the configuration file
if ":TYPE" in self.properties.keys():
logger.warning(
"Keyword ':TYPE' is reserved for Neo4j. "
"Removing from properties.",
# "Renaming to 'type'."
)
# self.properties["type"] = self.properties[":TYPE"]
del self.properties[":TYPE"]
for k, v in self.properties.items():
if isinstance(v, str):
self.properties[k] = (
v.replace(
os.linesep,
" ",
)
.replace(
"\n",
" ",
)
.replace(
"\r",
" ",
)
)
elif isinstance(v, list):
#modified biocypher, because the data contained intgers in lists
self.properties[k] = [
(str(val) if isinstance(val, (int, float)) else val)
.replace(os.linesep, " ")
.replace("\n", " ")
.replace("\r", " ")
for val in v
]
def get_id(self) -> str:
"""
Returns primary node identifier.
Returns:
str: node_id
"""
return self.node_id
def get_label(self) -> str:
"""
Returns primary node label.
Returns:
str: node_label
"""
return self.node_label
def get_type(self) -> str:
"""
Returns primary node label.
Returns:
str: node_label
"""
return self.node_label
def get_preferred_id(self) -> str:
"""
Returns preferred id.
Returns:
str: preferred_id
"""
return self.preferred_id
def get_properties(self) -> dict:
"""
Returns all other node properties apart from primary id and
label as key-value pairs.
Returns:
dict: properties
"""
return self.properties
def get_dict(self) -> dict:
"""
Return dict of id, labels, and properties.
Returns:
dict: node_id and node_label as top-level key-value pairs,
properties as second-level dict.
"""
return {
"node_id": self.node_id,
"node_label": self.node_label,
"properties": self.properties,
}
@dataclass(frozen=True)
class BioCypherEdge:
"""
Handoff class to represent biomedical relationships in Neo4j.
Has source and target ids, label, property dict; ids and label (in
the Neo4j sense of a label, ie, the entity descriptor after the
colon, such as ":TARGETS") are non-optional and called source_id,
target_id, and relationship_label to avoid confusion with properties
called "label", which usually denotes the human-readable form.
Relationship labels are written in UPPERCASE and as verbs, as per
Neo4j consensus.
Args:
source_id (string): consensus "best" id for biological entity
target_id (string): consensus "best" id for biological entity
relationship_label (string): type of interaction, UPPERCASE
properties (dict): collection of all other properties of the
respective edge
"""
source_id: str
target_id: str
relationship_label: str
relationship_id: str = None
properties: dict = field(default_factory=dict)
def __post_init__(self):
"""
Check for reserved keywords.
"""
if ":TYPE" in self.properties.keys():
logger.debug(
"Keyword ':TYPE' is reserved for Neo4j. "
"Removing from properties.",
# "Renaming to 'type'."
)
# self.properties["type"] = self.properties[":TYPE"]
del self.properties[":TYPE"]
elif "id" in self.properties.keys():
logger.debug(
"Keyword 'id' is reserved for Neo4j. "
"Removing from properties.",
# "Renaming to 'type'."
)
# self.properties["type"] = self.properties[":TYPE"]
del self.properties["id"]
elif "_ID" in self.properties.keys():
logger.debug(
"Keyword '_ID' is reserved for Postgres. "
"Removing from properties.",
# "Renaming to 'type'."
)
# self.properties["type"] = self.properties[":TYPE"]
del self.properties["_ID"]
def get_id(self) -> Union[str, None]:
"""
Returns primary node identifier or None.
Returns:
str: node_id
"""
return self.relationship_id
def get_source_id(self) -> str:
"""
Returns primary node identifier of relationship source.
Returns:
str: source_id
"""
return self.source_id
def get_target_id(self) -> str:
"""
Returns primary node identifier of relationship target.
Returns:
str: target_id
"""
return self.target_id
def get_label(self) -> str:
"""
Returns relationship label.
Returns:
str: relationship_label
"""
return self.relationship_label
def get_type(self) -> str:
"""
Returns relationship label.
Returns:
str: relationship_label
"""
return self.relationship_label
def get_properties(self) -> dict:
"""
Returns all other relationship properties apart from primary ids
and label as key-value pairs.
Returns:
dict: properties
"""
return self.properties
def get_dict(self) -> dict:
"""
Return dict of ids, label, and properties.
Returns:
dict: source_id, target_id and relationship_label as
top-level key-value pairs, properties as second-level
dict.
"""
return {
"relationship_id": self.relationship_id or None,
"source_id": self.source_id,
"target_id": self.target_id,
"relationship_label": self.relationship_label,
"properties": self.properties,
}
@dataclass(frozen=True)
class BioCypherRelAsNode:
"""
Class to represent relationships as nodes (with in- and outgoing
edges) as a triplet of a BioCypherNode and two BioCypherEdges. Main
usage in type checking (instances where the receiving function needs
to check whether it receives a relationship as a single edge or as
a triplet).
Args:
node (BioCypherNode): node representing the relationship
source_edge (BioCypherEdge): edge representing the source of the
relationship
target_edge (BioCypherEdge): edge representing the target of the
relationship
"""
node: BioCypherNode
source_edge: BioCypherEdge
target_edge: BioCypherEdge
def __post_init__(self):
if not isinstance(self.node, BioCypherNode):
raise TypeError(
f"BioCypherRelAsNode.node must be a BioCypherNode, "
f"not {type(self.node)}.",
)
if not isinstance(self.source_edge, BioCypherEdge):
raise TypeError(
f"BioCypherRelAsNode.source_edge must be a BioCypherEdge, "
f"not {type(self.source_edge)}.",
)
if not isinstance(self.target_edge, BioCypherEdge):
raise TypeError(
f"BioCypherRelAsNode.target_edge must be a BioCypherEdge, "
f"not {type(self.target_edge)}.",
)
def get_node(self) -> BioCypherNode:
return self.node
def get_source_edge(self) -> BioCypherEdge:
return self.source_edge
def get_target_edge(self) -> BioCypherEdge:
return self.target_edge
biocypher/_deduplicate.py
+147
View File
@@ -0,0 +1,147 @@
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from ._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
class Deduplicator:
"""
Singleton class responsible of deduplicating BioCypher inputs. Maintains
sets/dictionaries of node and edge types and their unique identifiers.
Nodes identifiers should be globally unique (represented as a set), while
edge identifiers are only unique per edge type (represented as a dict of
sets, keyed by edge type).
Stores collection of duplicate node and edge identifiers and types for
troubleshooting and to avoid overloading the log.
"""
def __init__(self):
self.seen_entity_ids = set()
self.duplicate_entity_ids = set()
self.entity_types = set()
self.duplicate_entity_types = set()
self.seen_relationships = {}
self.duplicate_relationship_ids = set()
self.duplicate_relationship_types = set()
def node_seen(self, entity: BioCypherNode) -> bool:
"""
Adds a node to the instance and checks if it has been seen before.
Args:
node: BioCypherNode to be added.
Returns:
True if the node has been seen before, False otherwise.
"""
if entity.get_label() not in self.entity_types:
self.entity_types.add(entity.get_label())
if entity.get_id() in self.seen_entity_ids:
self.duplicate_entity_ids.add(entity.get_id())
if entity.get_label() not in self.duplicate_entity_types:
logger.warning(
f"Duplicate node type {entity.get_label()} found. "
)
self.duplicate_entity_types.add(entity.get_label())
return True
self.seen_entity_ids.add(entity.get_id())
return False
def edge_seen(self, relationship: BioCypherEdge) -> bool:
"""
Adds an edge to the instance and checks if it has been seen before.
Args:
edge: BioCypherEdge to be added.
Returns:
True if the edge has been seen before, False otherwise.
"""
if relationship.get_type() not in self.seen_relationships:
self.seen_relationships[relationship.get_type()] = set()
# concatenate source and target if no id is present
if not relationship.get_id():
_id = (
f"{relationship.get_source_id()}_{relationship.get_target_id()}"
)
else:
_id = relationship.get_id()
if _id in self.seen_relationships[relationship.get_type()]:
self.duplicate_relationship_ids.add(_id)
if relationship.get_type() not in self.duplicate_relationship_types:
logger.warning(
f"Duplicate edge type {relationship.get_type()} found. "
)
self.duplicate_relationship_types.add(relationship.get_type())
return True
self.seen_relationships[relationship.get_type()].add(_id)
return False
def rel_as_node_seen(self, rel_as_node: BioCypherRelAsNode) -> bool:
"""
Adds a rel_as_node to the instance (one entity and two relationships)
and checks if it has been seen before. Only the node is relevant for
identifying the rel_as_node as a duplicate.
Args:
rel_as_node: BioCypherRelAsNode to be added.
Returns:
True if the rel_as_node has been seen before, False otherwise.
"""
node = rel_as_node.get_node()
if node.get_label() not in self.seen_relationships:
self.seen_relationships[node.get_label()] = set()
# rel as node always has an id
_id = node.get_id()
if _id in self.seen_relationships[node.get_type()]:
self.duplicate_relationship_ids.add(_id)
if node.get_type() not in self.duplicate_relationship_types:
logger.warning(f"Duplicate edge type {node.get_type()} found. ")
self.duplicate_relationship_types.add(node.get_type())
return True
self.seen_relationships[node.get_type()].add(_id)
return False
def get_duplicate_nodes(self):
"""
Function to return a list of duplicate nodes.
Returns:
list: list of duplicate nodes
"""
if self.duplicate_entity_types:
return (self.duplicate_entity_types, self.duplicate_entity_ids)
else:
return None
def get_duplicate_edges(self):
"""
Function to return a list of duplicate edges.
Returns:
list: list of duplicate edges
"""
if self.duplicate_relationship_types:
return (
self.duplicate_relationship_types,
self.duplicate_relationship_ids,
)
else:
return None
biocypher/_get.py
+443
View File
@@ -0,0 +1,443 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher get module. Used to download and cache data from external sources.
"""
from __future__ import annotations
from typing import Optional
import shutil
import requests
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from abc import ABC
from datetime import datetime, timedelta
from tempfile import TemporaryDirectory
import os
import json
import ftplib
import pooch
from ._misc import to_list, is_nested
class Resource(ABC):
def __init__(
self,
name: str,
url_s: str | list[str],
lifetime: int = 0,
):
"""
A Resource is a file, a list of files, an API request, or a list of API
requests, any of which can be downloaded from the given URL(s) and
cached locally. This class implements checks of the minimum requirements
for a resource, to be implemented by a biocypher adapter.
Args:
name (str): The name of the resource.
url_s (str | list[str]): The URL or URLs of the resource.
lifetime (int): The lifetime of the resource in days. If 0, the
resource is considered to be permanent.
"""
self.name = name
self.url_s = url_s
self.lifetime = lifetime
class FileDownload(Resource):
def __init__(
self,
name: str,
url_s: str | list[str],
lifetime: int = 0,
is_dir: bool = False,
):
"""
Represents basic information for a File Download.
Args:
name(str): The name of the File Download.
url_s(str|list[str]): The URL(s) of the File Download.
lifetime(int): The lifetime of the File Download in days. If 0, the
File Download is cached indefinitely.
is_dir (bool): Whether the URL points to a directory or not.
"""
super().__init__(name, url_s, lifetime)
self.is_dir = is_dir
class APIRequest(Resource):
def __init__(self, name: str, url_s: str | list[str], lifetime: int = 0):
"""
Represents basic information for an API Request.
Args:
name(str): The name of the API Request.
url_s(str|list): The URL of the API endpoint.
lifetime(int): The lifetime of the API Request in days. If 0, the
API Request is cached indefinitely.
"""
super().__init__(name, url_s, lifetime)
class Downloader:
def __init__(self, cache_dir: Optional[str] = None) -> None:
"""
The Downloader is a class that manages resources that can be downloaded
and cached locally. It manages the lifetime of downloaded resources by
keeping a JSON record of the download date of each resource.
Args:
cache_dir (str): The directory where the resources are cached. If
not given, a temporary directory is created.
"""
self.cache_dir = cache_dir or TemporaryDirectory().name
self.cache_file = os.path.join(self.cache_dir, "cache.json")
self.cache_dict = self._load_cache_dict()
def download(self, *resources: Resource):
"""
Download one or multiple resources. Load from cache if the resource is
already downloaded and the cache is not expired.
Args:
resources (Resource): The resource(s) to download or load from
cache.
Returns:
list[str]: The path or paths to the resource(s) that were downloaded
or loaded from cache.
"""
paths = []
for resource in resources:
paths.append(self._download_or_cache(resource))
# flatten list if it is nested
if is_nested(paths):
paths = [path for sublist in paths for path in sublist]
return paths
def _download_or_cache(self, resource: Resource, cache: bool = True):
"""
Download a resource if it is not cached or exceeded its lifetime.
Args:
resource (Resource): The resource to download.
Returns:
list[str]: The path or paths to the downloaded resource(s).
"""
expired = self._is_cache_expired(resource)
if expired or not cache:
self._delete_expired_cache(resource)
if isinstance(resource, FileDownload):
logger.info(f"Asking for download of resource {resource.name}.")
paths = self._download_files(cache, resource)
elif isinstance(resource, APIRequest):
logger.info(
f"Asking for download of api request {resource.name}."
)
paths = self._download_api_request(resource)
else:
raise TypeError(f"Unknown resource type: {type(resource)}")
else:
paths = self.get_cached_version(resource)
self._update_cache_record(resource)
return paths
def _is_cache_expired(self, resource: Resource) -> bool:
"""
Check if resource or API request cache is expired.
Args:
resource (Resource): The resource or API request to download.
Returns:
bool: True if cache is expired, False if not.
"""
cache_record = self._get_cache_record(resource)
if cache_record:
download_time = datetime.strptime(
cache_record.get("date_downloaded"), "%Y-%m-%d %H:%M:%S.%f"
)
lifetime = timedelta(days=resource.lifetime)
expired = download_time + lifetime < datetime.now()
else:
expired = True
return expired
def _delete_expired_cache(self, resource: Resource):
cache_resource_path = self.cache_dir + "/" + resource.name
if os.path.exists(cache_resource_path) and os.path.isdir(
cache_resource_path
):
shutil.rmtree(cache_resource_path)
def _download_files(self, cache, file_download: FileDownload):
"""
Download a resource given it is a file or a directory and return the
path.
Args:
cache (bool): Whether to cache the resource or not.
file_download (FileDownload): The resource to download.
Returns:
list[str]: The path or paths to the downloaded resource(s).
"""
if file_download.is_dir:
files = self._get_files(file_download)
file_download.url_s = [
file_download.url_s + "/" + file for file in files
]
file_download.is_dir = False
paths = self._download_or_cache(file_download, cache)
elif isinstance(file_download.url_s, list):
paths = []
for url in file_download.url_s:
fname = url[url.rfind("/") + 1 :].split("?")[0]
paths.append(
self._retrieve(
url=url,
fname=fname,
path=os.path.join(self.cache_dir, file_download.name),
)
)
else:
paths = []
fname = file_download.url_s[
file_download.url_s.rfind("/") + 1 :
].split("?")[0]
results = self._retrieve(
url=file_download.url_s,
fname=fname,
path=os.path.join(self.cache_dir, file_download.name),
)
if isinstance(results, list):
paths.extend(results)
else:
paths.append(results)
# sometimes a compressed file contains multiple files
# TODO ask for a list of files in the archive to be used from the
# adapter
return paths
def _download_api_request(self, api_request: APIRequest):
"""
Download an API request and return the path.
Args:
api_request(APIRequest): The API request result that is being
cached.
Returns:
list[str]: The path to the cached API request.
"""
urls = (
api_request.url_s
if isinstance(api_request.url_s, list)
else [api_request.url_s]
)
paths = []
for url in urls:
fname = url[url.rfind("/") + 1 :].rsplit(".", 1)[0]
logger.info(
f"Asking for caching API of {api_request.name} {fname}."
)
response = requests.get(url=url)
if response.status_code != 200:
response.raise_for_status()
response_data = response.json()
api_path = os.path.join(
self.cache_dir, api_request.name, f"{fname}.json"
)
os.makedirs(os.path.dirname(api_path), exist_ok=True)
with open(api_path, "w") as f:
json.dump(response_data, f)
logger.info(f"Caching API request to {api_path}.")
paths.append(api_path)
return paths
def get_cached_version(self, resource: Resource) -> list[str]:
"""Get the cached version of a resource.
Args:
resource(Resource): The resource to get the cached version of.
Returns:
list[str]: The paths to the cached resource(s).
"""
cached_location = os.path.join(self.cache_dir, resource.name)
logger.info(f"Use cached version from {cached_location}.")
paths = []
for file in os.listdir(cached_location):
paths.append(os.path.join(cached_location, file))
return paths
def _retrieve(
self,
url: str,
fname: str,
path: str,
known_hash: str = None,
):
"""
Retrieve a file from a URL using Pooch. Infer type of file from
extension and use appropriate processor.
Args:
url (str): The URL to retrieve the file from.
fname (str): The name of the file.
path (str): The path to the file.
"""
if fname.endswith(".zip"):
return pooch.retrieve(
url=url,
known_hash=known_hash,
fname=fname,
path=path,
processor=pooch.Unzip(),
progressbar=True,
)
elif fname.endswith(".tar.gz"):
return pooch.retrieve(
url=url,
known_hash=known_hash,
fname=fname,
path=path,
processor=pooch.Untar(),
progressbar=True,
)
elif fname.endswith(".gz"):
return pooch.retrieve(
url=url,
known_hash=known_hash,
fname=fname,
path=path,
processor=pooch.Decompress(),
progressbar=True,
)
else:
return pooch.retrieve(
url=url,
known_hash=known_hash,
fname=fname,
path=path,
progressbar=True,
)
def _get_files(self, file_download: FileDownload):
"""
Get the files contained in a directory file.
Args:
file_download (FileDownload): The directory file.
Returns:
list: The files contained in the directory.
"""
if file_download.url_s.startswith("ftp://"):
# remove protocol
url = file_download.url_s[6:]
# get base url
url = url[: url.find("/")]
# get directory (remove initial slash as well)
dir = file_download.url_s[7 + len(url) :]
# get files
ftp = ftplib.FTP(url)
ftp.login()
ftp.cwd(dir)
files = ftp.nlst()
ftp.quit()
else:
raise NotImplementedError(
"Only FTP directories are supported at the moment."
)
return files
def _load_cache_dict(self):
"""
Load the cache dictionary from the cache file. Create an empty cache
file if it does not exist.
"""
if not os.path.exists(self.cache_dir):
logger.info(f"Creating cache directory {self.cache_dir}.")
os.makedirs(self.cache_dir)
if not os.path.exists(self.cache_file):
logger.info(f"Creating cache file {self.cache_file}.")
with open(self.cache_file, "w") as f:
json.dump({}, f)
with open(self.cache_file, "r") as f:
logger.info(f"Loading cache file {self.cache_file}.")
return json.load(f)
def _get_cache_record(self, resource: Resource):
"""
Get the cache record of a resource.
Args:
resource (Resource): The resource to get the cache record of.
Returns:
The cache record of the resource.
"""
return self.cache_dict.get(resource.name, {})
def _update_cache_record(self, resource: Resource):
"""
Update the cache record of a resource.
Args:
resource (Resource): The resource to update the cache record of.
"""
cache_record = {}
cache_record["url"] = to_list(resource.url_s)
cache_record["date_downloaded"] = str(datetime.now())
cache_record["lifetime"] = resource.lifetime
self.cache_dict[resource.name] = cache_record
with open(self.cache_file, "w") as f:
json.dump(self.cache_dict, f, default=str)
biocypher/_logger.py
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
Configuration of the module logger.
"""
__all__ = ["get_logger", "log", "logfile"]
from datetime import datetime
import os
import pydoc
import logging
from biocypher import _config
from biocypher._metadata import __version__
def get_logger(name: str = "biocypher") -> logging.Logger:
"""
Access the module logger, create a new one if does not exist yet.
Method providing central logger instance to main module. Is called
only from main submodule, :mod:`biocypher.driver`. In child modules,
the standard Python logging facility is called
(using ``logging.getLogger(__name__)``), automatically inheriting
the handlers from the central logger.
The file handler creates a log file named after the current date and
time. Levels to output to file and console can be set here.
Args:
name:
Name of the logger instance.
Returns:
An instance of the Python :py:mod:`logging.Logger`.
"""
if not logging.getLogger(name).hasHandlers():
# create logger
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
logger.propagate = True
# formatting
file_formatter = logging.Formatter(
"%(asctime)s\t%(levelname)s\tmodule:%(module)s\n%(message)s",
)
stdout_formatter = logging.Formatter("%(levelname)s -- %(message)s")
# file name and creation
now = datetime.now()
date_time = now.strftime("%Y%m%d-%H%M%S")
log_to_disk = _config.config("biocypher").get("log_to_disk")
if log_to_disk:
logdir = (
_config.config("biocypher").get("log_directory")
or "biocypher-log"
)
os.makedirs(logdir, exist_ok=True)
logfile = os.path.join(logdir, f"biocypher-{date_time}.log")
# file handler
file_handler = logging.FileHandler(logfile)
if _config.config("biocypher").get("debug"):
file_handler.setLevel(logging.DEBUG)
else:
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
# handlers
# stream handler
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(logging.INFO)
stdout_handler.setFormatter(stdout_formatter)
# add handlers
logger.addHandler(stdout_handler)
# startup message
logger.info(f"This is BioCypher v{__version__}.")
if log_to_disk:
logger.info(f"Logging into `{logfile}`.")
else:
logger.info("Logging into stdout.")
return logging.getLogger(name)
def logfile() -> str:
"""
Path to the log file.
"""
return get_logger().handlers[0].baseFilename
def log():
"""
Browse the log file.
"""
with open(logfile()) as fp:
pydoc.pager(fp.read())
logger = get_logger()
biocypher/_mapping.py
+307
View File
@@ -0,0 +1,307 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'mapping' module. Handles the mapping of user-defined schema to the
underlying ontology.
"""
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from typing import Optional
from urllib.request import urlopen
import yaml
from . import _misc
from ._config import config as _config
class OntologyMapping:
"""
Class to store the ontology mapping and extensions.
"""
def __init__(self, config_file: str = None):
self.schema = self._read_config(config_file)
self.extended_schema = self._extend_schema()
def _read_config(self, config_file: str = None):
"""
Read the configuration file and store the ontology mapping and extensions.
"""
if config_file is None:
schema_config = {}
# load yaml file from web
elif config_file.startswith("http"):
with urlopen(config_file) as f:
schema_config = yaml.safe_load(f)
# get graph state from config (assume file is local)
else:
with open(config_file, "r") as f:
schema_config = yaml.safe_load(f)
return schema_config
def _extend_schema(self, d: Optional[dict] = None) -> dict:
"""
Get leaves of the tree hierarchy from the data structure dict
contained in the `schema_config.yaml`. Creates virtual leaves
(as children) from entries that provide more than one preferred
id type (and corresponding inputs).
Args:
d:
Data structure dict from yaml file.
"""
d = d or self.schema
extended_schema = dict()
# first pass: get parent leaves with direct representation in ontology
for k, v in d.items():
# k is not an entity
if "represented_as" not in v:
continue
# preferred_id optional: if not provided, use `id`
if not v.get("preferred_id"):
v["preferred_id"] = "id"
# k is an entity that is present in the ontology
if "is_a" not in v:
extended_schema[k] = v
# second pass: "vertical" inheritance
d = self._vertical_property_inheritance(d)
for k, v in d.items():
if "is_a" in v:
# prevent loops
if k == v["is_a"]:
logger.warning(
f"Loop detected in ontology mapping: {k} -> {v}. "
"Removing item. Please fix the inheritance if you want "
"to use this item."
)
continue
extended_schema[k] = v
# "horizontal" inheritance: create siblings for multiple identifiers or
# sources -> virtual leaves or implicit children
mi_leaves = {}
ms_leaves = {}
for k, v in d.items():
# k is not an entity
if "represented_as" not in v:
continue
if isinstance(v.get("preferred_id"), list):
mi_leaves = self._horizontal_inheritance_pid(k, v)
extended_schema.update(mi_leaves)
elif isinstance(v.get("source"), list):
ms_leaves = self._horizontal_inheritance_source(k, v)
extended_schema.update(ms_leaves)
return extended_schema
def _vertical_property_inheritance(self, d):
"""
Inherit properties from parents to children and update `d` accordingly.
"""
for k, v in d.items():
# k is not an entity
if "represented_as" not in v:
continue
# k is an entity that is present in the ontology
if "is_a" not in v:
continue
# "vertical" inheritance: inherit properties from parent
if v.get("inherit_properties", False):
# get direct ancestor
if isinstance(v["is_a"], list):
parent = v["is_a"][0]
else:
parent = v["is_a"]
# ensure child has properties and exclude_properties
if "properties" not in v:
v["properties"] = {}
if "exclude_properties" not in v:
v["exclude_properties"] = {}
# update properties of child
parent_props = self.schema[parent].get("properties", {})
if parent_props:
v["properties"].update(parent_props)
parent_excl_props = self.schema[parent].get(
"exclude_properties", {}
)
if parent_excl_props:
v["exclude_properties"].update(parent_excl_props)
# update schema (d)
d[k] = v
return d
def _horizontal_inheritance_pid(self, key, value):
"""
Create virtual leaves for multiple preferred id types or sources.
If we create virtual leaves, input_label/label_in_input always has to be
a list.
"""
leaves = {}
preferred_id = value["preferred_id"]
input_label = value.get("input_label") or value["label_in_input"]
represented_as = value["represented_as"]
# adjust lengths
max_l = max(
[
len(_misc.to_list(preferred_id)),
len(_misc.to_list(input_label)),
len(_misc.to_list(represented_as)),
],
)
# adjust pid length if necessary
if isinstance(preferred_id, str):
pids = [preferred_id] * max_l
else:
pids = preferred_id
# adjust rep length if necessary
if isinstance(represented_as, str):
reps = [represented_as] * max_l
else:
reps = represented_as
for pid, lab, rep in zip(pids, input_label, reps):
skey = pid + "." + key
svalue = {
"preferred_id": pid,
"input_label": lab,
"represented_as": rep,
# mark as virtual
"virtual": True,
}
# inherit is_a if exists
if "is_a" in value.keys():
# treat as multiple inheritance
if isinstance(value["is_a"], list):
v = list(value["is_a"])
v.insert(0, key)
svalue["is_a"] = v
else:
svalue["is_a"] = [key, value["is_a"]]
else:
# set parent as is_a
svalue["is_a"] = key
# inherit everything except core attributes
for k, v in value.items():
if k not in [
"is_a",
"preferred_id",
"input_label",
"label_in_input",
"represented_as",
]:
svalue[k] = v
leaves[skey] = svalue
return leaves
def _horizontal_inheritance_source(self, key, value):
"""
Create virtual leaves for multiple sources.
If we create virtual leaves, input_label/label_in_input always has to be
a list.
"""
leaves = {}
source = value["source"]
input_label = value.get("input_label") or value["label_in_input"]
represented_as = value["represented_as"]
# adjust lengths
src_l = len(source)
# adjust label length if necessary
if isinstance(input_label, str):
labels = [input_label] * src_l
else:
labels = input_label
# adjust rep length if necessary
if isinstance(represented_as, str):
reps = [represented_as] * src_l
else:
reps = represented_as
for src, lab, rep in zip(source, labels, reps):
skey = src + "." + key
svalue = {
"source": src,
"input_label": lab,
"represented_as": rep,
# mark as virtual
"virtual": True,
}
# inherit is_a if exists
if "is_a" in value.keys():
# treat as multiple inheritance
if isinstance(value["is_a"], list):
v = list(value["is_a"])
v.insert(0, key)
svalue["is_a"] = v
else:
svalue["is_a"] = [key, value["is_a"]]
else:
# set parent as is_a
svalue["is_a"] = key
# inherit everything except core attributes
for k, v in value.items():
if k not in [
"is_a",
"source",
"input_label",
"label_in_input",
"represented_as",
]:
svalue[k] = v
leaves[skey] = svalue
return leaves
biocypher/_metadata.py
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
Package metadata (version, authors, etc).
"""
__all__ = ["get_metadata"]
import os
import pathlib
import importlib.metadata
import toml
_VERSION = "0.6.0"
def get_metadata():
"""
Basic package metadata.
Retrieves package metadata from the current project directory or from
the installed package.
"""
here = pathlib.Path(__file__).parent
pyproj_toml = "pyproject.toml"
meta = {}
for project_dir in (here, here.parent):
toml_path = str(project_dir.joinpath(pyproj_toml).absolute())
if os.path.exists(toml_path):
pyproject = toml.load(toml_path)
meta = {
"name": pyproject["tool"]["poetry"]["name"],
"version": pyproject["tool"]["poetry"]["version"],
"author": pyproject["tool"]["poetry"]["authors"],
"license": pyproject["tool"]["poetry"]["license"],
"full_metadata": pyproject,
}
break
if not meta:
try:
meta = {
k.lower(): v
for k, v in importlib.metadata.metadata(here.name).items()
}
except importlib.metadata.PackageNotFoundError:
pass
meta["version"] = meta.get("version", None) or _VERSION
return meta
metadata = get_metadata()
__version__ = metadata.get("version", None)
__author__ = metadata.get("author", None)
__license__ = metadata.get("license", None)
biocypher/_misc.py
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
Handy functions for use in various places.
"""
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from typing import (
Any,
Union,
Mapping,
KeysView,
Generator,
ItemsView,
ValuesView,
)
from collections.abc import Iterable
import re
from treelib import Tree
import networkx as nx
import stringcase
__all__ = ["LIST_LIKE", "SIMPLE_TYPES", "ensure_iterable", "to_list"]
SIMPLE_TYPES = (
bytes,
str,
int,
float,
bool,
type(None),
)
LIST_LIKE = (
list,
set,
tuple,
Generator,
ItemsView,
KeysView,
Mapping,
ValuesView,
)
def to_list(value: Any) -> list:
"""
Ensures that ``value`` is a list.
"""
if isinstance(value, LIST_LIKE):
value = list(value)
else:
value = [value]
return value
def ensure_iterable(value: Any) -> Iterable:
"""
Returns iterables, except strings, wraps simple types into tuple.
"""
return value if isinstance(value, LIST_LIKE) else (value,)
def create_tree_visualisation(inheritance_graph: Union[dict, nx.Graph]) -> Tree:
"""
Creates a visualisation of the inheritance tree using treelib.
"""
inheritance_tree = _get_inheritance_tree(inheritance_graph)
classes, root = _find_root_node(inheritance_tree)
tree = Tree()
tree.create_node(root, root)
while classes:
for child in classes:
parent = inheritance_tree[child]
if parent in tree.nodes.keys() or parent == root:
tree.create_node(child, child, parent=parent)
for node in tree.nodes.keys():
if node in classes:
classes.remove(node)
return tree
def _get_inheritance_tree(inheritance_graph: Union[dict, nx.Graph]) -> dict:
"""Transforms an inheritance_graph into an inheritance_tree.
Args:
inheritance_graph: A dict or nx.Graph representing the inheritance graph.
Returns:
A dict representing the inheritance tree.
"""
if isinstance(inheritance_graph, nx.Graph):
inheritance_tree = nx.to_dict_of_lists(inheritance_graph)
multiple_parents_present = _multiple_inheritance_present(
inheritance_tree
)
if multiple_parents_present:
logger.warning(
"The ontology contains multiple inheritance (one child node "
"has multiple parent nodes). This is not visualized in the "
"following hierarchy tree (the child node is only added once). "
"If you wish to browse all relationships of the parsed "
"ontologies, write a graphml file to disk using "
"`to_disk = <directory>` and view this file."
)
# unlist values
inheritance_tree = {k: v[0] for k, v in inheritance_tree.items() if v}
return inheritance_tree
elif not _multiple_inheritance_present(inheritance_graph):
return inheritance_graph
def _multiple_inheritance_present(inheritance_tree: dict) -> bool:
"""Checks if multiple inheritance is present in the inheritance_tree."""
return any(len(value) > 1 for value in inheritance_tree.values())
def _find_root_node(inheritance_tree: dict) -> tuple[set, str]:
classes = set(inheritance_tree.keys())
parents = set(inheritance_tree.values())
root = list(parents - classes)
if len(root) > 1:
if "entity" in root:
root = "entity" # TODO: default: good standard?
else:
raise ValueError(
"Inheritance tree cannot have more than one root node. "
f"Found {len(root)}: {root}."
)
else:
root = root[0]
if not root:
# find key whose value is None
root = list(inheritance_tree.keys())[
list(inheritance_tree.values()).index(None)
]
return classes, root
# string conversion, adapted from Biolink Model Toolkit
lowercase_pattern = re.compile(r"[a-zA-Z]*[a-z][a-zA-Z]*")
underscore_pattern = re.compile(r"(?<!^)(?=[A-Z][a-z])")
def from_pascal(s: str, sep: str = " ") -> str:
underscored = underscore_pattern.sub(sep, s)
lowercased = lowercase_pattern.sub(
lambda match: match.group(0).lower(),
underscored,
)
return lowercased
def pascalcase_to_sentencecase(s: str) -> str:
"""
Convert PascalCase to sentence case.
Args:
s: Input string in PascalCase
Returns:
string in sentence case form
"""
return from_pascal(s, sep=" ")
def snakecase_to_sentencecase(s: str) -> str:
"""
Convert snake_case to sentence case.
Args:
s: Input string in snake_case
Returns:
string in sentence case form
"""
return stringcase.sentencecase(s).lower()
def sentencecase_to_snakecase(s: str) -> str:
"""
Convert sentence case to snake_case.
Args:
s: Input string in sentence case
Returns:
string in snake_case form
"""
return stringcase.snakecase(s).lower()
def sentencecase_to_pascalcase(s: str, sep: str = r"\s") -> str:
"""
Convert sentence case to PascalCase.
Args:
s: Input string in sentence case
Returns:
string in PascalCase form
"""
return re.sub(
r"(?:^|[" + sep + "])([a-zA-Z])",
lambda match: match.group(1).upper(),
s,
)
def to_lower_sentence_case(s: str) -> str:
"""
Convert any string to lower sentence case. Works with snake_case,
PascalCase, and sentence case.
Args:
s: Input string
Returns:
string in lower sentence case form
"""
if "_" in s:
return snakecase_to_sentencecase(s)
elif " " in s:
return s.lower()
elif s[0].isupper():
return pascalcase_to_sentencecase(s)
else:
return s
def is_nested(lst) -> bool:
"""
Check if a list is nested.
Args:
lst (list): The list to check.
Returns:
bool: True if the list is nested, False otherwise.
"""
for item in lst:
if isinstance(item, list):
return True
return False
biocypher/_ontology.py
+886
View File
@@ -0,0 +1,886 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'ontology' module. Contains classes and functions to handle parsing
and representation of single ontologies as well as their hybridisation and
other advanced operations.
"""
import os
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from typing import Optional
from datetime import datetime
from rdflib import Graph
from rdflib.extras.external_graph_libs import rdflib_to_networkx_digraph
import rdflib
import networkx as nx
from ._misc import (
to_list,
to_lower_sentence_case,
create_tree_visualisation,
sentencecase_to_pascalcase,
)
from ._mapping import OntologyMapping
class OntologyAdapter:
"""
Class that represents an ontology to be used in the Biocypher framework. Can
read from a variety of formats, including OWL, OBO, and RDF/XML. The
ontology is represented by a networkx.DiGraph object; an RDFlib graph is
also kept. By default, the DiGraph reverses the label and identifier of the
nodes, such that the node name in the graph is the human-readable label. The
edges are oriented from child to parent.
Labels are formatted in lower sentence case and underscores are replaced by spaces.
Identifiers are taken as defined and the prefixes are removed by default.
"""
def __init__(
self,
ontology_file: str,
root_label: str,
ontology_file_format: Optional[str] = None,
head_join_node_label: Optional[str] = None,
merge_nodes: Optional[bool] = True,
switch_label_and_id: bool = True,
remove_prefixes: bool = True,
):
"""
Initialize the OntologyAdapter class.
Args:
ontology_file (str): Path to the ontology file. Can be local or
remote.
root_label (str): The label of the root node in the ontology. In
case of a tail ontology, this is the tail join node.
ontology_file_format (str): The format of the ontology file (e.g. "application/rdf+xml")
If format is not passed, it is determined automatically.
head_join_node_label (str): Optional variable to store the label of the
node in the head ontology that should be used to join to the
root node of the tail ontology. Defaults to None.
merge_nodes (bool): If True, head and tail join nodes will be
merged, using the label of the head join node. If False, the
tail join node will be attached as a child of the head join
node.
switch_label_and_id (bool): If True, the node names in the graph will be
the human-readable labels. If False, the node names will be the
identifiers. Defaults to True.
remove_prefixes (bool): If True, the prefixes of the identifiers will
be removed. Defaults to True.
"""
logger.info(f"Instantiating OntologyAdapter class for {ontology_file}.")
self._ontology_file = ontology_file
self._root_label = root_label
self._format = ontology_file_format
self._merge_nodes = merge_nodes
self._head_join_node = head_join_node_label
self._switch_label_and_id = switch_label_and_id
self._remove_prefixes = remove_prefixes
self._rdf_graph = self._load_rdf_graph(ontology_file)
self._nx_graph = self._rdf_to_nx(
self._rdf_graph, root_label, switch_label_and_id
)
def _rdf_to_nx(
self,
_rdf_graph: rdflib.Graph,
root_label: str,
switch_label_and_id: bool,
rename_nodes: bool = True,
) -> nx.DiGraph:
one_to_one_triples, one_to_many_dict = self._get_relevant_rdf_triples(
_rdf_graph
)
nx_graph = self._convert_to_nx(one_to_one_triples, one_to_many_dict)
nx_graph = self._add_labels_to_nodes(nx_graph, switch_label_and_id)
nx_graph = self._change_nodes_to_biocypher_format(
nx_graph, switch_label_and_id, rename_nodes
)
nx_graph = self._get_all_ancestors(
nx_graph, root_label, switch_label_and_id, rename_nodes
)
return nx.DiGraph(nx_graph)
def _get_relevant_rdf_triples(self, g: rdflib.Graph) -> tuple:
one_to_one_inheritance_graph = self._get_one_to_one_inheritance_triples(
g
)
intersection = self._get_multiple_inheritance_dict(g)
return one_to_one_inheritance_graph, intersection
def _get_one_to_one_inheritance_triples(
self, g: rdflib.Graph
) -> rdflib.Graph:
"""Get the one to one inheritance triples from the RDF graph.
Args:
g (rdflib.Graph): The RDF graph
Returns:
rdflib.Graph: The one to one inheritance graph
"""
one_to_one_inheritance_graph = Graph()
for s, p, o in g.triples((None, rdflib.RDFS.subClassOf, None)):
if self.has_label(s, g):
one_to_one_inheritance_graph.add((s, p, o))
return one_to_one_inheritance_graph
def _get_multiple_inheritance_dict(self, g: rdflib.Graph) -> dict:
"""Get the multiple inheritance dictionary from the RDF graph.
Args:
g (rdflib.Graph): The RDF graph
Returns:
dict: The multiple inheritance dictionary
"""
multiple_inheritance = g.triples(
(None, rdflib.OWL.intersectionOf, None)
)
intersection = {}
for (
node,
has_multiple_parents,
first_node_of_intersection_list,
) in multiple_inheritance:
parents = self._retrieve_rdf_linked_list(
first_node_of_intersection_list
)
child_name = None
for s_, _, _ in g.triples((None, rdflib.RDFS.subClassOf, node)):
child_name = s_
# Handle Snomed CT post coordinated expressions
if not child_name:
for s_, _, _ in g.triples(
(None, rdflib.OWL.equivalentClass, node)
):
child_name = s_
if child_name:
intersection[node] = {
"child_name": child_name,
"parent_node_names": parents,
}
return intersection
def has_label(self, node: rdflib.URIRef, g: rdflib.Graph) -> bool:
"""Does the node have a label in g?
Args:
node (rdflib.URIRef): The node to check
g (rdflib.Graph): The graph to check in
Returns:
bool: True if the node has a label, False otherwise
"""
return (node, rdflib.RDFS.label, None) in g
def _retrieve_rdf_linked_list(self, subject: rdflib.URIRef) -> list:
"""Recursively retrieves a linked list from RDF.
Example RDF list with the items [item1, item2]:
list_node - first -> item1
list_node - rest -> list_node2
list_node2 - first -> item2
list_node2 - rest -> nil
Args:
subject (rdflib.URIRef): One list_node of the RDF list
Returns:
list: The items of the RDF list
"""
g = self._rdf_graph
rdf_list = []
for s, p, o in g.triples((subject, rdflib.RDF.first, None)):
rdf_list.append(o)
for s, p, o in g.triples((subject, rdflib.RDF.rest, None)):
if o != rdflib.RDF.nil:
rdf_list.extend(self._retrieve_rdf_linked_list(o))
return rdf_list
def _convert_to_nx(
self, one_to_one: rdflib.Graph, one_to_many: dict
) -> nx.DiGraph:
"""Convert the one to one and one to many inheritance graphs to networkx.
Args:
one_to_one (rdflib.Graph): The one to one inheritance graph
one_to_many (dict): The one to many inheritance dictionary
Returns:
nx.DiGraph: The networkx graph
"""
nx_graph = rdflib_to_networkx_digraph(
one_to_one, edge_attrs=lambda s, p, o: {}, calc_weights=False
)
for key, value in one_to_many.items():
nx_graph.add_edges_from(
[
(value["child_name"], parent)
for parent in value["parent_node_names"]
]
)
if key in nx_graph.nodes:
nx_graph.remove_node(key)
return nx_graph
def _add_labels_to_nodes(
self, nx_graph: nx.DiGraph, switch_label_and_id: bool
) -> nx.DiGraph:
"""Add labels to the nodes in the networkx graph.
Args:
nx_graph (nx.DiGraph): The networkx graph
switch_label_and_id (bool): If True, id and label are switched
Returns:
nx.DiGraph: The networkx graph with labels
"""
for node in list(nx_graph.nodes):
nx_id, nx_label = self._get_nx_id_and_label(
node, switch_label_and_id
)
if nx_id == "none":
# remove node if it has no id
nx_graph.remove_node(node)
continue
nx_graph.nodes[node]["label"] = nx_label
return nx_graph
def _change_nodes_to_biocypher_format(
self,
nx_graph: nx.DiGraph,
switch_label_and_id: bool,
rename_nodes: bool = True,
) -> nx.DiGraph:
"""Change the nodes in the networkx graph to BioCypher format:
- remove the prefix of the identifier
- switch id and label
- adapt the labels (replace _ with space and convert to lower sentence case)
Args:
nx_graph (nx.DiGraph): The networkx graph
switch_label_and_id (bool): If True, id and label are switched
rename_nodes (bool): If True, the nodes are renamed
Returns:
nx.DiGraph: The networkx ontology graph in BioCypher format
"""
mapping = {
node: self._get_nx_id_and_label(
node, switch_label_and_id, rename_nodes
)[0]
for node in nx_graph.nodes
}
renamed = nx.relabel_nodes(nx_graph, mapping, copy=False)
return renamed
def _get_all_ancestors(
self,
renamed: nx.DiGraph,
root_label: str,
switch_label_and_id: bool,
rename_nodes: bool = True,
) -> nx.DiGraph:
"""Get all ancestors of the root node in the networkx graph.
Args:
renamed (nx.DiGraph): The renamed networkx graph
root_label (str): The label of the root node in the ontology
switch_label_and_id (bool): If True, id and label are switched
rename_nodes (bool): If True, the nodes are renamed
Returns:
nx.DiGraph: The filtered networkx graph
"""
root = self._get_nx_id_and_label(
self._find_root_label(self._rdf_graph, root_label),
switch_label_and_id,
rename_nodes,
)[0]
ancestors = nx.ancestors(renamed, root)
ancestors.add(root)
filtered_graph = renamed.subgraph(ancestors)
return filtered_graph
def _get_nx_id_and_label(
self, node, switch_id_and_label: bool, rename_nodes: bool = True
) -> tuple[str, str]:
"""Rename node id and label for nx graph.
Args:
node (str): The node to rename
switch_id_and_label (bool): If True, switch id and label
Returns:
tuple[str, str]: The renamed node id and label
"""
node_id_str = self._remove_prefix(str(node))
node_label_str = str(self._rdf_graph.value(node, rdflib.RDFS.label))
if rename_nodes:
node_label_str = node_label_str.replace("_", " ")
node_label_str = to_lower_sentence_case(node_label_str)
nx_id = node_label_str if switch_id_and_label else node_id_str
nx_label = node_id_str if switch_id_and_label else node_label_str
return nx_id, nx_label
def _find_root_label(self, g, root_label):
# Loop through all labels in the ontology
for label_subject, _, label_in_ontology in g.triples(
(None, rdflib.RDFS.label, None)
):
# If the label is the root label, set the root node to the label's subject
if str(label_in_ontology) == root_label:
root = label_subject
break
else:
labels_in_ontology = []
for label_subject, _, label_in_ontology in g.triples(
(None, rdflib.RDFS.label, None)
):
labels_in_ontology.append(str(label_in_ontology))
raise ValueError(
f"Could not find root node with label '{root_label}'. "
f"The ontology contains the following labels: {labels_in_ontology}"
)
return root
def _remove_prefix(self, uri: str) -> str:
"""
Remove the prefix of a URI. URIs can contain either "#" or "/" as a
separator between the prefix and the local name. The prefix is
everything before the last separator.
"""
if self._remove_prefixes:
return uri.rsplit("#", 1)[-1].rsplit("/", 1)[-1]
else:
return uri
def _load_rdf_graph(self, ontology_file):
"""
Load the ontology into an RDFlib graph. The ontology file can be in
OWL, OBO, or RDF/XML format.
"""
g = rdflib.Graph()
g.parse(ontology_file, format=self._get_format(ontology_file))
return g
def _get_format(self, ontology_file):
"""
Get the format of the ontology file.
"""
if self._format:
if self._format == "owl":
return "application/rdf+xml"
elif self._format == "obo":
raise NotImplementedError("OBO format not yet supported")
elif self._format == "rdf":
return "application/rdf+xml"
elif self._format == "ttl":
return self._format
else:
raise ValueError(
f"Could not determine format of ontology file {ontology_file}"
)
if ontology_file.endswith(".owl"):
return "application/rdf+xml"
elif ontology_file.endswith(".obo"):
raise NotImplementedError("OBO format not yet supported")
elif ontology_file.endswith(".rdf"):
return "application/rdf+xml"
elif ontology_file.endswith(".ttl"):
return "ttl"
else:
raise ValueError(
f"Could not determine format of ontology file {ontology_file}"
)
def get_nx_graph(self):
"""
Get the networkx graph representing the ontology.
"""
return self._nx_graph
def get_rdf_graph(self):
"""
Get the RDFlib graph representing the ontology.
"""
return self._rdf_graph
def get_root_node(self):
"""
Get root node in the ontology.
Returns:
root_node: If _switch_label_and_id is True, the root node label is returned,
otherwise the root node id is returned.
"""
root_node = None
root_label = self._root_label.replace("_", " ")
if self._switch_label_and_id:
root_node = to_lower_sentence_case(root_label)
elif not self._switch_label_and_id:
for node, data in self.get_nx_graph().nodes(data=True):
if "label" in data and data["label"] == to_lower_sentence_case(
root_label
):
root_node = node
break
return root_node
def get_ancestors(self, node_label):
"""
Get the ancestors of a node in the ontology.
"""
return nx.dfs_preorder_nodes(self._nx_graph, node_label)
def get_head_join_node(self):
"""
Get the head join node of the ontology.
"""
return self._head_join_node
class Ontology:
"""
A class that represents the ontological "backbone" of a BioCypher knowledge
graph. The ontology can be built from a single resource, or hybridised from
a combination of resources, with one resource being the "head" ontology,
while an arbitrary number of other resources can become "tail" ontologies at
arbitrary fusion points inside the "head" ontology.
"""
def __init__(
self,
head_ontology: dict,
ontology_mapping: Optional["OntologyMapping"] = None,
tail_ontologies: Optional[dict] = None,
):
"""
Initialize the Ontology class.
Args:
head_ontology (OntologyAdapter): The head ontology.
tail_ontologies (list): A list of OntologyAdapters that will be
added to the head ontology. Defaults to None.
"""
self._head_ontology_meta = head_ontology
self.mapping = ontology_mapping
self._tail_ontology_meta = tail_ontologies
self._tail_ontologies = None
self._nx_graph = None
# keep track of nodes that have been extended
self._extended_nodes = set()
self._main()
def _main(self) -> None:
"""
Main method to be run on instantiation. Loads the ontologies, joins
them, and returns the hybrid ontology. Loads only the head ontology
if nothing else is given. Adds user extensions and properties from
the mapping.
"""
self._load_ontologies()
if self._tail_ontologies:
for adapter in self._tail_ontologies.values():
head_join_node = self._get_head_join_node(adapter)
self._join_ontologies(adapter, head_join_node)
else:
self._nx_graph = self._head_ontology.get_nx_graph()
if self.mapping:
self._extend_ontology()
# experimental: add connections of disjoint classes to entity
# self._connect_biolink_classes()
self._add_properties()
def _load_ontologies(self) -> None:
"""
For each ontology, load the OntologyAdapter object and store it as an
instance variable (head) or a dictionary (tail).
"""
logger.info("Loading ontologies...")
self._head_ontology = OntologyAdapter(
ontology_file=self._head_ontology_meta["url"],
root_label=self._head_ontology_meta["root_node"],
ontology_file_format=self._head_ontology_meta.get("format", None),
switch_label_and_id=self._head_ontology_meta.get(
"switch_label_and_id", True
),
)
if self._tail_ontology_meta:
self._tail_ontologies = {}
for key, value in self._tail_ontology_meta.items():
self._tail_ontologies[key] = OntologyAdapter(
ontology_file=value["url"],
root_label=value["tail_join_node"],
head_join_node_label=value["head_join_node"],
ontology_file_format=value.get("format", None),
merge_nodes=value.get("merge_nodes", True),
switch_label_and_id=value.get("switch_label_and_id", True),
)
def _get_head_join_node(self, adapter: OntologyAdapter) -> str:
"""
Tries to find the head join node of the given ontology adapter in the
head ontology. If the join node is not found, the method will raise an
error.
Args:
adapter (OntologyAdapter): The ontology adapter of which to find the
join node in the head ontology.
"""
head_join_node = None
user_defined_head_join_node_label = adapter.get_head_join_node()
head_join_node_label_in_bc_format = to_lower_sentence_case(
user_defined_head_join_node_label.replace("_", " ")
)
if self._head_ontology._switch_label_and_id:
head_join_node = head_join_node_label_in_bc_format
elif not self._head_ontology._switch_label_and_id:
for node_id, data in self._head_ontology.get_nx_graph().nodes(
data=True
):
if (
"label" in data
and data["label"] == head_join_node_label_in_bc_format
):
head_join_node = node_id
break
if head_join_node not in self._head_ontology.get_nx_graph().nodes:
head_ontology = self._head_ontology._rdf_to_nx(
self._head_ontology.get_rdf_graph(),
self._head_ontology._root_label,
self._head_ontology._switch_label_and_id,
rename_nodes=False,
)
raise ValueError(
f"Head join node '{head_join_node}' not found in head ontology. "
f"The head ontology contains the following nodes: {head_ontology.nodes}."
)
return head_join_node
def _join_ontologies(
self, adapter: OntologyAdapter, head_join_node
) -> None:
"""
Joins the ontologies by adding the tail ontology as a subgraph to the
head ontology at the specified join nodes.
Args:
adapter (OntologyAdapter): The ontology adapter of the tail ontology
to be added to the head ontology.
"""
if not self._nx_graph:
self._nx_graph = self._head_ontology.get_nx_graph().copy()
tail_join_node = adapter.get_root_node()
tail_ontology = adapter.get_nx_graph()
# subtree of tail ontology at join node
tail_ontology_subtree = nx.dfs_tree(
tail_ontology.reverse(), tail_join_node
).reverse()
# transfer node attributes from tail ontology to subtree
for node in tail_ontology_subtree.nodes:
tail_ontology_subtree.nodes[node].update(tail_ontology.nodes[node])
# if merge_nodes is False, create parent of tail join node from head
# join node
if not adapter._merge_nodes:
# add head join node from head ontology to tail ontology subtree
# as parent of tail join node
tail_ontology_subtree.add_node(
head_join_node,
**self._head_ontology.get_nx_graph().nodes[head_join_node],
)
tail_ontology_subtree.add_edge(tail_join_node, head_join_node)
# else rename tail join node to match head join node if necessary
elif not tail_join_node == head_join_node:
tail_ontology_subtree = nx.relabel_nodes(
tail_ontology_subtree, {tail_join_node: head_join_node}
)
# combine head ontology and tail subtree
self._nx_graph = nx.compose(self._nx_graph, tail_ontology_subtree)
def _extend_ontology(self) -> None:
"""
Adds the user extensions to the ontology. Tries to find the parent in
the ontology, adds it if necessary, and adds the child and a directed
edge from child to parent. Can handle multiple parents.
"""
if not self._nx_graph:
self._nx_graph = self._head_ontology.get_nx_graph().copy()
for key, value in self.mapping.extended_schema.items():
if not value.get("is_a"):
if self._nx_graph.has_node(value.get("synonym_for")):
continue
if not self._nx_graph.has_node(key):
raise ValueError(
f"Node {key} not found in ontology, but also has no "
"inheritance definition. Please check your schema for "
"spelling errors, first letter not in lower case, use of underscores, a missing `is_a` definition (SubClassOf a root node), or missing labels in class or super-classes."
)
continue
parents = to_list(value.get("is_a"))
child = key
while parents:
parent = parents.pop(0)
if parent not in self._nx_graph.nodes:
self._nx_graph.add_node(parent)
self._nx_graph.nodes[parent][
"label"
] = sentencecase_to_pascalcase(parent)
# mark parent as user extension
self._nx_graph.nodes[parent]["user_extension"] = True
self._extended_nodes.add(parent)
if child not in self._nx_graph.nodes:
self._nx_graph.add_node(child)
self._nx_graph.nodes[child][
"label"
] = sentencecase_to_pascalcase(child)
# mark child as user extension
self._nx_graph.nodes[child]["user_extension"] = True
self._extended_nodes.add(child)
self._nx_graph.add_edge(child, parent)
child = parent
def _connect_biolink_classes(self) -> None:
"""
Experimental: Adds edges from disjoint classes to the entity node.
"""
if not self._nx_graph:
self._nx_graph = self._head_ontology.get_nx_graph().copy()
if "entity" not in self._nx_graph.nodes:
return
# biolink classes that are disjoint from entity
disjoint_classes = [
"frequency qualifier mixin",
"chemical entity to entity association mixin",
"ontology class",
"relationship quantifier",
"physical essence or occurrent",
"gene or gene product",
"subject of investigation",
]
for node in disjoint_classes:
if not self._nx_graph.nodes.get(node):
self._nx_graph.add_node(node)
self._nx_graph.nodes[node][
"label"
] = sentencecase_to_pascalcase(node)
self._nx_graph.add_edge(node, "entity")
def _add_properties(self) -> None:
"""
For each entity in the mapping, update the ontology with the properties
specified in the mapping. Updates synonym information in the graph,
setting the synonym as the primary node label.
"""
for key, value in self.mapping.extended_schema.items():
if key in self._nx_graph.nodes:
self._nx_graph.nodes[key].update(value)
if value.get("synonym_for"):
# change node label to synonym
if value["synonym_for"] not in self._nx_graph.nodes:
raise ValueError(
f'Node {value["synonym_for"]} not found in ontology.'
)
self._nx_graph = nx.relabel_nodes(
self._nx_graph, {value["synonym_for"]: key}
)
def get_ancestors(self, node_label: str) -> list:
"""
Get the ancestors of a node in the ontology.
Args:
node_label (str): The label of the node in the ontology.
Returns:
list: A list of the ancestors of the node.
"""
return nx.dfs_tree(self._nx_graph, node_label)
def show_ontology_structure(self, to_disk: str = None, full: bool = False):
"""
Show the ontology structure using treelib or write to GRAPHML file.
Args:
to_disk (str): If specified, the ontology structure will be saved
to disk as a GRAPHML file at the location (directory) specified
by the `to_disk` string, to be opened in your favourite graph
visualisation tool.
full (bool): If True, the full ontology structure will be shown,
including all nodes and edges. If False, only the nodes and
edges that are relevant to the extended schema will be shown.
"""
if not full and not self.mapping.extended_schema:
raise ValueError(
"You are attempting to visualise a subset of the loaded"
"ontology, but have not provided a schema configuration. "
"To display a partial ontology graph, please provide a schema "
"configuration file; to visualise the full graph, please use "
"the parameter `full = True`."
)
if not self._nx_graph:
raise ValueError("Ontology not loaded.")
if not self._tail_ontologies:
msg = f"Showing ontology structure based on {self._head_ontology._ontology_file}"
else:
msg = f"Showing ontology structure based on {len(self._tail_ontology_meta)+1} ontologies: "
logger.info(msg)
if not full:
# set of leaves and their intermediate parents up to the root
filter_nodes = set(self.mapping.extended_schema.keys())
for node in self.mapping.extended_schema.keys():
filter_nodes.update(self.get_ancestors(node).nodes)
# filter graph
G = self._nx_graph.subgraph(filter_nodes)
else:
G = self._nx_graph
if not to_disk:
# create tree
tree = create_tree_visualisation(G)
# add synonym information
for node in self.mapping.extended_schema:
if not isinstance(self.mapping.extended_schema[node], dict):
continue
if self.mapping.extended_schema[node].get("synonym_for"):
tree.nodes[node].tag = (
f"{node} = "
f"{self.mapping.extended_schema[node].get('synonym_for')}"
)
logger.info(f"\n{tree}")
return tree
else:
# convert lists/dicts to strings for vis only
for node in G.nodes:
# rename node and use former id as label
label = G.nodes[node].get("label")
if not label:
label = node
G = nx.relabel_nodes(G, {node: label})
G.nodes[label]["label"] = node
for attrib in G.nodes[label]:
if type(G.nodes[label][attrib]) in [list, dict]:
G.nodes[label][attrib] = str(G.nodes[label][attrib])
path = os.path.join(to_disk, "ontology_structure.graphml")
logger.info(f"Writing ontology structure to {path}.")
nx.write_graphml(G, path)
return True
def get_dict(self) -> dict:
"""
Returns a dictionary compatible with a BioCypher node for compatibility
with the Neo4j driver.
"""
d = {
"node_id": self._get_current_id(),
"node_label": "BioCypher",
"properties": {
"schema": "self.ontology_mapping.extended_schema",
},
}
return d
def _get_current_id(self):
"""
Instantiate a version ID for the current session. For now does simple
versioning using datetime.
Can later implement incremental versioning, versioning from
config file, or manual specification via argument.
"""
now = datetime.now()
return now.strftime("v%Y%m%d-%H%M%S")
biocypher/_translate.py
+480
View File
@@ -0,0 +1,480 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'translation' module. Responsible for translating between the raw
input data and the BioCypherNode and BioCypherEdge objects.
"""
from ._logger import logger
logger.debug(f"Loading module {__name__}.")
from typing import Any, Union, Optional
from collections.abc import Iterable, Generator
from more_itertools import peekable
from . import _misc
from ._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
from ._ontology import Ontology
__all__ = ["BiolinkAdapter", "Translator"]
class Translator:
"""
Class responsible for exacting the translation process that is configured in
the schema_config.yaml file. Creates a mapping dictionary from that file,
and, given nodes and edges, translates them into BioCypherNodes and
BioCypherEdges. During this process, can also filter the properties of the
entities if the schema_config.yaml file specifies a property whitelist or
blacklist.
Provides utility functions for translating between input and output labels
and cypher queries.
"""
def __init__(self, ontology: "Ontology", strict_mode: bool = False):
"""
Args:
leaves:
Dictionary detailing the leaves of the hierarchy
tree representing the structure of the graph; the leaves are
the entities that will be direct components of the graph,
while the intermediary nodes are additional labels for
filtering purposes.
strict_mode:
If True, the translator will raise an error if input data do not
carry source, licence, and version information.
"""
self.ontology = ontology
self.strict_mode = strict_mode
# record nodes without biolink type configured in schema_config.yaml
self.notype = {}
# mapping functionality for translating terms and queries
self.mappings = {}
self.reverse_mappings = {}
self._update_ontology_types()
def translate_nodes(
self,
node_tuples: Iterable,
) -> Generator[BioCypherNode, None, None]:
"""
Translates input node representation to a representation that
conforms to the schema of the given BioCypher graph. For now
requires explicit statement of node type on pass.
Args:
node_tuples (list of tuples): collection of tuples
representing individual nodes by their unique id and a type
that is translated from the original database notation to
the corresponding BioCypher notation.
"""
self._log_begin_translate(node_tuples, "nodes")
for _id, _type, _props in node_tuples:
# check for strict mode requirements
required_props = ["source", "licence", "version"]
if self.strict_mode:
# rename 'license' to 'licence' in _props
if _props.get("license"):
_props["licence"] = _props.pop("license")
for prop in required_props:
if prop not in _props:
raise ValueError(
f"Property `{prop}` missing from node {_id}. "
"Strict mode is enabled, so this is not allowed."
)
# find the node in leaves that represents ontology node type
_ontology_class = self._get_ontology_mapping(_type)
if _ontology_class:
# filter properties for those specified in schema_config if any
_filtered_props = self._filter_props(_ontology_class, _props)
# preferred id
_preferred_id = self._get_preferred_id(_ontology_class)
yield BioCypherNode(
node_id=_id,
node_label=_ontology_class,
preferred_id=_preferred_id,
properties=_filtered_props,
)
else:
self._record_no_type(_type, _id)
self._log_finish_translate("nodes")
def _get_preferred_id(self, _bl_type: str) -> str:
"""
Returns the preferred id for the given Biolink type.
"""
return (
self.ontology.mapping.extended_schema[_bl_type]["preferred_id"]
if "preferred_id"
in self.ontology.mapping.extended_schema.get(_bl_type, {})
else "id"
)
def _filter_props(self, bl_type: str, props: dict) -> dict:
"""
Filters properties for those specified in schema_config if any.
"""
filter_props = self.ontology.mapping.extended_schema[bl_type].get(
"properties", {}
)
# strict mode: add required properties (only if there is a whitelist)
if self.strict_mode and filter_props:
filter_props.update(
{"source": "str", "licence": "str", "version": "str"},
)
exclude_props = self.ontology.mapping.extended_schema[bl_type].get(
"exclude_properties", []
)
if isinstance(exclude_props, str):
exclude_props = [exclude_props]
if filter_props and exclude_props:
filtered_props = {
k: v
for k, v in props.items()
if (k in filter_props.keys() and k not in exclude_props)
}
elif filter_props:
filtered_props = {
k: v for k, v in props.items() if k in filter_props.keys()
}
elif exclude_props:
filtered_props = {
k: v for k, v in props.items() if k not in exclude_props
}
else:
return props
missing_props = [
k for k in filter_props.keys() if k not in filtered_props.keys()
]
# add missing properties with default values
for k in missing_props:
filtered_props[k] = None
return filtered_props
def translate_edges(
self,
edge_tuples: Iterable,
) -> Generator[Union[BioCypherEdge, BioCypherRelAsNode], None, None]:
"""
Translates input edge representation to a representation that
conforms to the schema of the given BioCypher graph. For now
requires explicit statement of edge type on pass.
Args:
edge_tuples (list of tuples):
collection of tuples representing source and target of
an interaction via their unique ids as well as the type
of interaction in the original database notation, which
is translated to BioCypher notation using the `leaves`.
Can optionally possess its own ID.
"""
self._log_begin_translate(edge_tuples, "edges")
# legacy: deal with 4-tuples (no edge id)
# TODO remove for performance reasons once safe
edge_tuples = peekable(edge_tuples)
if len(edge_tuples.peek()) == 4:
edge_tuples = [
(None, src, tar, typ, props)
for src, tar, typ, props in edge_tuples
]
for _id, _src, _tar, _type, _props in edge_tuples:
# check for strict mode requirements
if self.strict_mode:
if not "source" in _props:
raise ValueError(
f"Edge {_id if _id else (_src, _tar)} does not have a `source` property.",
" This is required in strict mode.",
)
if not "licence" in _props:
raise ValueError(
f"Edge {_id if _id else (_src, _tar)} does not have a `licence` property.",
" This is required in strict mode.",
)
# match the input label (_type) to
# a Biolink label from schema_config
bl_type = self._get_ontology_mapping(_type)
if bl_type:
# filter properties for those specified in schema_config if any
_filtered_props = self._filter_props(bl_type, _props)
rep = self.ontology.mapping.extended_schema[bl_type][
"represented_as"
]
if rep == "node":
if _id:
# if it brings its own ID, use it
node_id = _id
else:
# source target concat
node_id = (
str(_src)
+ "_"
+ str(_tar)
+ "_"
+ "_".join(str(v) for v in _filtered_props.values())
)
n = BioCypherNode(
node_id=node_id,
node_label=bl_type,
properties=_filtered_props,
)
# directionality check TODO generalise to account for
# different descriptions of directionality or find a
# more consistent solution for indicating directionality
if _filtered_props.get("directed") == True:
l1 = "IS_SOURCE_OF"
l2 = "IS_TARGET_OF"
elif _filtered_props.get(
"src_role",
) and _filtered_props.get("tar_role"):
l1 = _filtered_props.get("src_role")
l2 = _filtered_props.get("tar_role")
else:
l1 = l2 = "IS_PART_OF"
e_s = BioCypherEdge(
source_id=_src,
target_id=node_id,
relationship_label=l1,
# additional here
)
e_t = BioCypherEdge(
source_id=_tar,
target_id=node_id,
relationship_label=l2,
# additional here
)
yield BioCypherRelAsNode(n, e_s, e_t)
else:
edge_label = self.ontology.mapping.extended_schema[
bl_type
].get("label_as_edge")
if edge_label is None:
edge_label = bl_type
yield BioCypherEdge(
relationship_id=_id,
source_id=_src,
target_id=_tar,
relationship_label=edge_label,
properties=_filtered_props,
)
else:
self._record_no_type(_type, (_src, _tar))
self._log_finish_translate("edges")
def _record_no_type(self, _type: Any, what: Any) -> None:
"""
Records the type of a node or edge that is not represented in the
schema_config.
"""
logger.debug(f"No ontology type defined for `{_type}`: {what}")
if self.notype.get(_type, None):
self.notype[_type] += 1
else:
self.notype[_type] = 1
def get_missing_biolink_types(self) -> dict:
"""
Returns a dictionary of types that were not represented in the
schema_config.
"""
return self.notype
@staticmethod
def _log_begin_translate(_input: Iterable, what: str):
n = f"{len(_input)} " if hasattr(_input, "__len__") else ""
logger.debug(f"Translating {n}{what} to BioCypher")
@staticmethod
def _log_finish_translate(what: str):
logger.debug(f"Finished translating {what} to BioCypher.")
def _update_ontology_types(self):
"""
Creates a dictionary to translate from input labels to ontology labels.
If multiple input labels, creates mapping for each.
"""
self._ontology_mapping = {}
for key, value in self.ontology.mapping.extended_schema.items():
labels = value.get("input_label") or value.get("label_in_input")
if isinstance(labels, str):
self._ontology_mapping[labels] = key
elif isinstance(labels, list):
for label in labels:
self._ontology_mapping[label] = key
if value.get("label_as_edge"):
self._add_translation_mappings(labels, value["label_as_edge"])
else:
self._add_translation_mappings(labels, key)
def _get_ontology_mapping(self, label: str) -> Optional[str]:
"""
For each given input type ("input_label" or "label_in_input"), find the
corresponding ontology class in the leaves dictionary (from the
`schema_config.yam`).
Args:
label:
The input type to find (`input_label` or `label_in_input` in
`schema_config.yaml`).
"""
# commented out until behaviour of _update_bl_types is fixed
return self._ontology_mapping.get(label, None)
def translate_term(self, term):
"""
Translate a single term.
"""
return self.mappings.get(term, None)
def reverse_translate_term(self, term):
"""
Reverse translate a single term.
"""
return self.reverse_mappings.get(term, None)
def translate(self, query):
"""
Translate a cypher query. Only translates labels as of now.
"""
for key in self.mappings:
query = query.replace(":" + key, ":" + self.mappings[key])
return query
def reverse_translate(self, query):
"""
Reverse translate a cypher query. Only translates labels as of
now.
"""
for key in self.reverse_mappings:
a = ":" + key + ")"
b = ":" + key + "]"
# TODO this conditional probably does not cover all cases
if a in query or b in query:
if isinstance(self.reverse_mappings[key], list):
raise NotImplementedError(
"Reverse translation of multiple inputs not "
"implemented yet. Many-to-one mappings are "
"not reversible. "
f"({key} -> {self.reverse_mappings[key]})",
)
else:
query = query.replace(
a,
":" + self.reverse_mappings[key] + ")",
).replace(b, ":" + self.reverse_mappings[key] + "]")
return query
def _add_translation_mappings(self, original_name, biocypher_name):
"""
Add translation mappings for a label and name. We use here the
PascalCase version of the BioCypher name, since sentence case is
not useful for Cypher queries.
"""
if isinstance(original_name, list):
for on in original_name:
self.mappings[on] = self.name_sentence_to_pascal(
biocypher_name,
)
else:
self.mappings[original_name] = self.name_sentence_to_pascal(
biocypher_name,
)
if isinstance(biocypher_name, list):
for bn in biocypher_name:
self.reverse_mappings[
self.name_sentence_to_pascal(
bn,
)
] = original_name
else:
self.reverse_mappings[
self.name_sentence_to_pascal(
biocypher_name,
)
] = original_name
@staticmethod
def name_sentence_to_pascal(name: str) -> str:
"""
Converts a name in sentence case to pascal case.
"""
# split on dots if dot is present
if "." in name:
return ".".join(
[_misc.sentencecase_to_pascalcase(n) for n in name.split(".")],
)
else:
return _misc.sentencecase_to_pascalcase(name)
tests/simplePatient/import.json → biocypher/output/__init__.py
View File
File renamed without changes.
tests/simplePatient/improve.json → biocypher/output/connect/__init__.py
View File
File renamed without changes.
biocypher/output/connect/_neo4j_driver.py
+422
View File
@@ -0,0 +1,422 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# ...
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'online' mode. Handles connection and manipulation of a running DBMS.
"""
import subprocess
from biocypher._logger import logger
logger.debug(f"Loading module {__name__}.")
from collections.abc import Iterable
import itertools
import neo4j_utils
from biocypher import _misc
from biocypher._config import config as _config
from biocypher._create import BioCypherEdge, BioCypherNode
from biocypher._translate import Translator
__all__ = ["_Neo4jDriver"]
class _Neo4jDriver:
"""
Manages a BioCypher connection to a Neo4j database using the
``neo4j_utils.Driver`` class.
Args:
database_name (str): The name of the database to connect to.
wipe (bool): Whether to wipe the database before importing.
uri (str): The URI of the database.
user (str): The username to use for authentication.
password (str): The password to use for authentication.
multi_db (bool): Whether to use multi-database mode.
fetch_size (int): The number of records to fetch at a time.
increment_version (bool): Whether to increment the version number.
translator (Translator): The translator to use for mapping.
"""
def __init__(
self,
database_name: str,
uri: str,
user: str,
password: str,
multi_db: bool,
translator: Translator,
wipe: bool = False,
fetch_size: int = 1000,
increment_version: bool = True,
):
self.translator = translator
self._driver = neo4j_utils.Driver(
db_name=database_name,
db_uri=uri,
db_user=user,
db_passwd=password,
fetch_size=fetch_size,
wipe=wipe,
multi_db=multi_db,
raise_errors=True,
)
# check for biocypher config in connected graph
if wipe:
self.init_db()
if increment_version:
# set new current version node
self._update_meta_graph()
def _update_meta_graph(self):
logger.info("Updating Neo4j meta graph.")
# find current version node
db_version = self._driver.query(
"MATCH (v:BioCypher) " "WHERE NOT (v)-[:PRECEDES]->() " "RETURN v",
)
# add version node
self.add_biocypher_nodes(self.translator.ontology)
# connect version node to previous
if db_version[0]:
previous = db_version[0][0]
previous_id = previous["v"]["id"]
e_meta = BioCypherEdge(
previous_id,
self.translator.ontology.get_dict().get("node_id"),
"PRECEDES",
)
self.add_biocypher_edges(e_meta)
def init_db(self):
"""
Used to initialise a property graph database by setting up new
constraints. Wipe has been performed by the ``neo4j_utils.Driver``
class` already.
Todo:
- set up constraint creation interactively depending on the
need of the database
"""
logger.info("Initialising database.")
self._create_constraints()
def _create_constraints(self):
"""
Creates constraints on node types in the graph. Used for
initial setup.
Grabs leaves of the ``schema_config.yaml`` file and creates
constraints on the id of all entities represented as nodes.
"""
logger.info("Creating constraints for node types in config.")
major_neo4j_version = int(self._get_neo4j_version().split(".")[0])
# get structure
for leaf in self.translator.ontology.mapping.extended_schema.items():
label = _misc.sentencecase_to_pascalcase(leaf[0], sep=r"\s\.")
if leaf[1]["represented_as"] == "node":
if major_neo4j_version >= 5:
s = (
f"CREATE CONSTRAINT `{label}_id` "
f"IF NOT EXISTS FOR (n:`{label}`) "
"REQUIRE n.id IS UNIQUE"
)
self._driver.query(s)
else:
s = (
f"CREATE CONSTRAINT `{label}_id` "
f"IF NOT EXISTS ON (n:`{label}`) "
"ASSERT n.id IS UNIQUE"
)
self._driver.query(s)
def _get_neo4j_version(self):
"""Get neo4j version."""
try:
neo4j_version = self._driver.query(
"""
CALL dbms.components()
YIELD name, versions, edition
UNWIND versions AS version
RETURN version AS version
""",
)[0][0]["version"]
return neo4j_version
except Exception as e:
logger.warning(
f"Error detecting Neo4j version: {e} use default version 4.0.0."
)
return "4.0.0"
def add_nodes(self, id_type_tuples: Iterable[tuple]) -> tuple:
"""
Generic node adder method to add any kind of input to the graph via the
:class:`biocypher.create.BioCypherNode` class. Employs translation
functionality and calls the :meth:`add_biocypher_nodes()` method.
Args:
id_type_tuples (iterable of 3-tuple): for each node to add to
the biocypher graph, a 3-tuple with the following layout:
first, the (unique if constrained) ID of the node; second, the
type of the node, capitalised or PascalCase and in noun form
(Neo4j primary label, eg `:Protein`); and third, a dictionary
of arbitrary properties the node should possess (can be empty).
Returns:
2-tuple: the query result of :meth:`add_biocypher_nodes()`
- first entry: data
- second entry: Neo4j summary.
"""
bn = self.translator.translate_nodes(id_type_tuples)
return self.add_biocypher_nodes(bn)
def add_edges(self, id_src_tar_type_tuples: Iterable[tuple]) -> tuple:
"""
Generic edge adder method to add any kind of input to the graph
via the :class:`biocypher.create.BioCypherEdge` class. Employs
translation functionality and calls the
:meth:`add_biocypher_edges()` method.
Args:
id_src_tar_type_tuples (iterable of 5-tuple):
for each edge to add to the biocypher graph, a 5-tuple
with the following layout: first, the optional unique ID
of the interaction. This can be `None` if there is no
systematic identifier (which for many interactions is
the case). Second and third, the (unique if constrained)
IDs of the source and target nodes of the relationship;
fourth, the type of the relationship; and fifth, a
dictionary of arbitrary properties the edge should
possess (can be empty).
Returns:
2-tuple: the query result of :meth:`add_biocypher_edges()`
- first entry: data
- second entry: Neo4j summary.
"""
bn = self.translator.translate_edges(id_src_tar_type_tuples)
return self.add_biocypher_edges(bn)
def add_biocypher_nodes(
self,
nodes: Iterable[BioCypherNode],
explain: bool = False,
profile: bool = False,
) -> bool:
"""
Accepts a node type handoff class
(:class:`biocypher.create.BioCypherNode`) with id,
label, and a dict of properties (passing on the type of
property, ie, ``int``, ``str``, ...).
The dict retrieved by the
:meth:`biocypher.create.BioCypherNode.get_dict()` method is
passed into Neo4j as a map of maps, explicitly encoding node id
and label, and adding all other properties from the 'properties'
key of the dict. The merge is performed via APOC, matching only
on node id to prevent duplicates. The same properties are set on
match and on create, irrespective of the actual event.
Args:
nodes:
An iterable of :class:`biocypher.create.BioCypherNode` objects.
explain:
Call ``EXPLAIN`` on the CYPHER query.
profile:
Do profiling on the CYPHER query.
Returns:
True for success, False otherwise.
"""
try:
nodes = _misc.to_list(nodes)
entities = [node.get_dict() for node in nodes]
except AttributeError:
msg = "Nodes must have a `get_dict` method."
logger.error(msg)
raise ValueError(msg)
logger.info(f"Merging {len(entities)} nodes.")
entity_query = (
"UNWIND $entities AS ent "
"CALL apoc.merge.node([ent.node_label], "
"{id: ent.node_id}, ent.properties, ent.properties) "
"YIELD node "
"RETURN node"
)
method = "explain" if explain else "profile" if profile else "query"
result = getattr(self._driver, method)(
entity_query,
parameters={
"entities": entities,
},
)
logger.info("Finished merging nodes.")
return result
def add_biocypher_edges(
self,
edges: Iterable[BioCypherEdge],
explain: bool = False,
profile: bool = False,
) -> bool:
"""
Accepts an edge type handoff class
(:class:`biocypher.create.BioCypherEdge`) with source
and target ids, label, and a dict of properties (passing on the
type of property, ie, int, string ...).
The individual edge is either passed as a singleton, in the case
of representation as an edge in the graph, or as a 4-tuple, in
the case of representation as a node (with two edges connecting
to interaction partners).
The dict retrieved by the
:meth:`biocypher.create.BioCypherEdge.get_dict()` method is
passed into Neo4j as a map of maps, explicitly encoding source
and target ids and the relationship label, and adding all edge
properties from the 'properties' key of the dict. The merge is
performed via APOC, matching only on source and target id to
prevent duplicates. The same properties are set on match and on
create, irrespective of the actual event.
Args:
edges:
An iterable of :class:`biocypher.create.BioCypherEdge` objects.
explain:
Call ``EXPLAIN`` on the CYPHER query.
profile:
Do profiling on the CYPHER query.
Returns:
`True` for success, `False` otherwise.
"""
edges = _misc.ensure_iterable(edges)
edges = itertools.chain(*(_misc.ensure_iterable(i) for i in edges))
nodes = []
rels = []
try:
for e in edges:
if hasattr(e, "get_node"):
nodes.append(e.get_node())
rels.append(e.get_source_edge().get_dict())
rels.append(e.get_target_edge().get_dict())
else:
rels.append(e.get_dict())
except AttributeError:
msg = "Edges and nodes must have a `get_dict` method."
logger.error(msg)
raise ValueError(msg)
self.add_biocypher_nodes(nodes)
logger.info(f"Merging {len(rels)} edges.")
# cypher query
# merging only on the ids of the entities, passing the
# properties on match and on create;
# TODO add node labels?
node_query = (
"UNWIND $rels AS r "
"MERGE (src {id: r.source_id}) "
"MERGE (tar {id: r.target_id}) "
)
self._driver.query(node_query, parameters={"rels": rels})
edge_query = (
"UNWIND $rels AS r "
"MATCH (src {id: r.source_id}) "
"MATCH (tar {id: r.target_id}) "
"WITH src, tar, r "
"CALL apoc.merge.relationship"
"(src, r.relationship_label, NULL, "
"r.properties, tar, r.properties) "
"YIELD rel "
"RETURN rel"
)
method = "explain" if explain else "profile" if profile else "query"
result = getattr(self._driver, method)(
edge_query, parameters={"rels": rels}
)
logger.info("Finished merging edges.")
return result
def get_driver(
dbms: str,
translator: "Translator",
):
"""
Function to return the writer class.
Returns:
class: the writer class
"""
dbms_config = _config(dbms)
if dbms == "neo4j":
return _Neo4jDriver(
database_name=dbms_config["database_name"],
wipe=dbms_config["wipe"],
uri=dbms_config["uri"],
user=dbms_config["user"],
password=dbms_config["password"],
multi_db=dbms_config["multi_db"],
translator=translator,
)
return None
biocypher/output/in_memory/__init__.py
View File
Whitespace-only changes.
biocypher/output/in_memory/_pandas.py
+90
View File
@@ -0,0 +1,90 @@
import pandas as pd
from biocypher._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
class Pandas:
def __init__(self, translator, deduplicator):
self.translator = translator
self.deduplicator = deduplicator
self.dfs = {}
def _separate_entity_types(self, entities):
"""
Given mixed iterable of BioCypher objects, separate them into lists by
type. Also deduplicates using the `Deduplicator` instance.
"""
lists = {}
for entity in entities:
if (
not isinstance(entity, BioCypherNode)
and not isinstance(entity, BioCypherEdge)
and not isinstance(entity, BioCypherRelAsNode)
):
raise TypeError(
"Expected a BioCypherNode / BioCypherEdge / "
f"BioCypherRelAsNode, got {type(entity)}."
)
if isinstance(entity, BioCypherNode):
seen = self.deduplicator.node_seen(entity)
elif isinstance(entity, BioCypherEdge):
seen = self.deduplicator.edge_seen(entity)
elif isinstance(entity, BioCypherRelAsNode):
seen = self.deduplicator.rel_as_node_seen(entity)
if seen:
continue
if isinstance(entity, BioCypherRelAsNode):
node = entity.get_node()
source_edge = entity.get_source_edge()
target_edge = entity.get_target_edge()
_type = node.get_type()
if not _type in lists:
lists[_type] = []
lists[_type].append(node)
_source_type = source_edge.get_type()
if not _source_type in lists:
lists[_source_type] = []
lists[_source_type].append(source_edge)
_target_type = target_edge.get_type()
if not _target_type in lists:
lists[_target_type] = []
lists[_target_type].append(target_edge)
continue
_type = entity.get_type()
if not _type in lists:
lists[_type] = []
lists[_type].append(entity)
return lists
def add_tables(self, entities):
"""
Add Pandas dataframes for each node and edge type in the input.
"""
lists = self._separate_entity_types(entities)
for _type, _entities in lists.items():
self._add_entity_df(_type, _entities)
def _add_entity_df(self, _type, _entities):
df = pd.DataFrame(
pd.json_normalize([node.get_dict() for node in _entities])
)
# replace "properties." with "" in column names
df.columns = [col.replace("properties.", "") for col in df.columns]
if _type not in self.dfs:
self.dfs[_type] = df
else:
self.dfs[_type] = pd.concat(
[self.dfs[_type], df], ignore_index=True
)
return self.dfs[_type]
biocypher/output/write/__init__.py
View File
Whitespace-only changes.
biocypher/output/write/_batch_writer.py
+1046
View File
File diff suppressed because it is too large. Load diff
biocypher/output/write/_get_writer.py
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Sebastian Lobentanzer
# Michael Hartung
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'offline' module. Handles the writing of node and edge representations
suitable for import into a DBMS.
"""
from biocypher._logger import logger
from biocypher.output.write.graph._rdf import _RDFWriter
from biocypher.output.write.graph._neo4j import _Neo4jBatchWriter
from biocypher.output.write.graph._arangodb import _ArangoDBBatchWriter
from biocypher.output.write.graph._networkx import _NetworkXWriter
from biocypher.output.write.relational._csv import _PandasCSVWriter
from biocypher.output.write.relational._sqlite import _SQLiteBatchWriter
from biocypher.output.write.relational._postgresql import _PostgreSQLBatchWriter
logger.debug(f"Loading module {__name__}.")
from typing import TYPE_CHECKING
from biocypher._config import config as _config
__all__ = ["get_writer", "DBMS_TO_CLASS"]
if TYPE_CHECKING:
from biocypher._translate import Translator
from biocypher._deduplicate import Deduplicator
DBMS_TO_CLASS = {
"neo": _Neo4jBatchWriter,
"neo4j": _Neo4jBatchWriter,
"Neo4j": _Neo4jBatchWriter,
"postgres": _PostgreSQLBatchWriter,
"postgresql": _PostgreSQLBatchWriter,
"PostgreSQL": _PostgreSQLBatchWriter,
"arango": _ArangoDBBatchWriter,
"arangodb": _ArangoDBBatchWriter,
"ArangoDB": _ArangoDBBatchWriter,
"sqlite": _SQLiteBatchWriter,
"sqlite3": _SQLiteBatchWriter,
"rdf": _RDFWriter,
"RDF": _RDFWriter,
"csv": _PandasCSVWriter,
"CSV": _PandasCSVWriter,
"pandas": _PandasCSVWriter,
"Pandas": _PandasCSVWriter,
"networkx": _NetworkXWriter,
"NetworkX": _NetworkXWriter,
}
def get_writer(
dbms: str,
translator: "Translator",
deduplicator: "Deduplicator",
output_directory: str,
strict_mode: bool,
):
"""
Function to return the writer class based on the selection in the config
file.
Args:
dbms: the database management system; for options, see DBMS_TO_CLASS.
translator: the Translator object.
deduplicator: the Deduplicator object.
output_directory: the directory to output.write the output files to.
strict_mode: whether to use strict mode.
Returns:
instance: an instance of the selected writer class.
"""
dbms_config = _config(dbms)
writer = DBMS_TO_CLASS[dbms]
if not writer:
raise ValueError(f"Unknown dbms: {dbms}")
if writer is not None:
return writer(
translator=translator,
deduplicator=deduplicator,
delimiter=dbms_config.get("delimiter"),
array_delimiter=dbms_config.get("array_delimiter"),
quote=dbms_config.get("quote_character"),
output_directory=output_directory,
db_name=dbms_config.get("database_name"),
import_call_bin_prefix=dbms_config.get("import_call_bin_prefix"),
import_call_file_prefix=dbms_config.get("import_call_file_prefix"),
wipe=dbms_config.get("wipe"),
strict_mode=strict_mode,
skip_bad_relationships=dbms_config.get(
"skip_bad_relationships"
), # neo4j
skip_duplicate_nodes=dbms_config.get(
"skip_duplicate_nodes"
), # neo4j
db_user=dbms_config.get("user"), # psql
db_password=dbms_config.get("password"), # psql
db_port=dbms_config.get("port"), # psql
rdf_format=dbms_config.get("rdf_format"), # rdf
rdf_namespaces=dbms_config.get("rdf_namespaces"), # rdf
)
biocypher/output/write/_writer.py
+200
View File
@@ -0,0 +1,200 @@
from abc import ABC, abstractmethod
from typing import Union, Optional
from collections.abc import Iterable
import os
from biocypher._create import BioCypherEdge, BioCypherNode, BioCypherRelAsNode
from biocypher._logger import logger
from biocypher._translate import Translator
from biocypher._deduplicate import Deduplicator
__all__ = ["_Writer"]
class _Writer(ABC):
"""Abstract class for writing node and edge representations to disk.
Specifics of the different writers (e.g. neo4j, postgresql, csv, etc.)
are implemented in the child classes. Any concrete writer needs to
implement at least:
- _write_node_data
- _write_edge_data
- _construct_import_call
- _get_import_script_name
Args:
translator (Translator): Instance of :py:class:`Translator` to enable translation of
nodes and manipulation of properties.
deduplicator (Deduplicator): Instance of :py:class:`Deduplicator` to enable deduplication
of nodes and edges.
output_directory (str, optional): Path for exporting CSV files. Defaults to None.
strict_mode (bool, optional): Whether to enforce source, version, and license properties. Defaults to False.
strict_mode (bool, optional): Whether to enforce source, version, and license properties. Defaults to False.
Raises:
NotImplementedError: Writer implementation must override '_write_node_data'
NotImplementedError: Writer implementation must override '_write_edge_data'
NotImplementedError: Writer implementation must override '_construct_import_call'
NotImplementedError: Writer implementation must override '_get_import_script_name'
"""
def __init__(
self,
translator: Translator,
deduplicator: Deduplicator,
output_directory: Optional[str] = None,
strict_mode: bool = False,
*args,
**kwargs,
):
"""Abstract class for writing node and edge representations to disk.
Args:
translator (Translator): Instance of :py:class:`Translator` to enable translation of
nodes and manipulation of properties.
deduplicator (Deduplicator): Instance of :py:class:`Deduplicator` to enable deduplication
of nodes and edges.
output_directory (str, optional): Path for exporting CSV files. Defaults to None.
strict_mode (bool, optional): Whether to enforce source, version, and license properties. Defaults to False.
strict_mode (bool, optional): Whether to enforce source, version, and license properties. Defaults to False.
"""
self.translator = translator
self.deduplicator = deduplicator
self.strict_mode = strict_mode
self.output_directory = output_directory
if os.path.exists(self.output_directory):
if kwargs.get("write_to_file", True):
logger.warning(
f"Output directory `{self.output_directory}` already exists. "
"If this is not planned, file consistency may be compromised."
)
else:
logger.info(f"Creating output directory `{self.output_directory}`.")
os.makedirs(self.output_directory)
@abstractmethod
def _write_node_data(
self,
nodes: Iterable[
Union[BioCypherNode, BioCypherEdge, BioCypherRelAsNode]
],
) -> bool:
"""Implement how to output.write nodes to disk.
Args:
nodes (Iterable): An iterable of BioCypherNode / BioCypherEdge / BioCypherRelAsNode objects.
Returns:
bool: The return value. True for success, False otherwise.
"""
raise NotImplementedError(
"Writer implementation must override 'write_nodes'"
)
@abstractmethod
def _write_edge_data(
self,
edges: Iterable[
Union[BioCypherNode, BioCypherEdge, BioCypherRelAsNode]
],
) -> bool:
"""Implement how to output.write edges to disk.
Args:
edges (Iterable): An iterable of BioCypherNode / BioCypherEdge / BioCypherRelAsNode objects.
Returns:
bool: The return value. True for success, False otherwise.
"""
raise NotImplementedError(
"Writer implementation must override 'write_edges'"
)
@abstractmethod
def _construct_import_call(self) -> str:
"""
Function to construct the import call detailing folder and
individual node and edge headers and data files, as well as
delimiters and database name. Built after all data has been
processed to ensure that nodes are called before any edges.
Returns:
str: command for importing the output files into a DBMS.
"""
raise NotImplementedError(
"Writer implementation must override '_construct_import_call'"
)
@abstractmethod
def _get_import_script_name(self) -> str:
"""Returns the name of the import script.
Returns:
str: The name of the import script (ending in .sh)
"""
raise NotImplementedError(
"Writer implementation must override '_get_import_script_name'"
)
def write_nodes(
self, nodes, batch_size: int = int(1e6), force: bool = False
):
"""Wrapper for writing nodes.
Args:
nodes (BioCypherNode): a list or generator of nodes in
:py:class:`BioCypherNode` format
batch_size (int): The batch size for writing nodes.
force (bool): Whether to force writing nodes even if their type is
not present in the schema.
Returns:
bool: The return value. True for success, False otherwise.
"""
passed = self._write_node_data(nodes)
if not passed:
logger.error("Error while writing node data.")
return False
return True
def write_edges(
self, edges, batch_size: int = int(1e6), force: bool = False
):
"""Wrapper for writing edges.
Args:
nodes (BioCypherNode): a list or generator of nodes in
:py:class:`BioCypherNode` format
batch_size (int): The batch size for writing nodes.
force (bool): Whether to force writing nodes even if their type is
not present in the schema.
Returns:
bool: The return value. True for success, False otherwise.
"""
passed = self._write_edge_data(edges)
if not passed:
logger.error("Error while writing edge data.")
return False
return True
def write_import_call(self):
"""
Function to output.write the import call detailing folder and
individual node and edge headers and data files, as well as
delimiters and database name, to the export folder as txt.
Returns:
str: The path of the file holding the import call.
"""
file_path = os.path.join(
self.output_directory, self._get_import_script_name()
)
logger.info(
f"Writing {self.__class__.__name__} import call to `{file_path}`."
)
with open(file_path, "w", encoding="utf-8") as f:
f.write(self._construct_import_call())
return file_path
biocypher/output/write/graph/__init__.py
View File
Whitespace-only changes.
biocypher/output/write/graph/_arangodb.py
+241
View File
@@ -0,0 +1,241 @@
import os
from biocypher._logger import logger
from biocypher.output.write.graph._neo4j import _Neo4jBatchWriter
class _ArangoDBBatchWriter(_Neo4jBatchWriter):
"""
Class for writing node and edge representations to disk using the format
specified by ArangoDB for the use of "arangoimport". Output files are
similar to Neo4j, but with a different header format.
"""
def _get_default_import_call_bin_prefix(self):
"""
Method to provide the default string for the import call bin prefix.
Returns:
str: The default location for the neo4j admin import location
"""
return ""
def _get_import_script_name(self) -> str:
"""
Returns the name of the neo4j admin import script
Returns:
str: The name of the import script (ending in .sh)
"""
return "arangodb-import-call.sh"
def _write_node_headers(self):
"""
Writes single CSV file for a graph entity that is represented
as a node as per the definition in the `schema_config.yaml`,
containing only the header for this type of node.
Returns:
bool: The return value. True for success, False otherwise.
"""
# load headers from data parse
if not self.node_property_dict:
logger.error(
"Header information not found. Was the data parsed first?",
)
return False
for label, props in self.node_property_dict.items():
# create header CSV with ID, properties, labels
_id = "_key"
# translate label to PascalCase
pascal_label = self.translator.name_sentence_to_pascal(label)
header = f"{pascal_label}-header.csv"
header_path = os.path.join(
self.outdir,
header,
)
# check if file already exists
if os.path.exists(header_path):
logger.warning(
f"File {header_path} already exists. Overwriting."
)
# concatenate key:value in props
props_list = []
for k in props.keys():
props_list.append(f"{k}")
# create list of lists and flatten
# removes need for empty check of property list
out_list = [[_id], props_list]
out_list = [val for sublist in out_list for val in sublist]
with open(header_path, "w", encoding="utf-8") as f:
# concatenate with delimiter
row = self.delim.join(out_list)
f.write(row)
# add collection from schema config
collection = self.translator.ontology.mapping.extended_schema[
label
].get("db_collection_name", None)
# add file path to neo4 admin import statement
# do once for each part file
parts = self.parts.get(label, [])
if not parts:
raise ValueError(
f"No parts found for node label {label}. "
f"Check that the data was parsed first.",
)
for part in parts:
import_call_header_path = os.path.join(
self.import_call_file_prefix,
header,
)
import_call_parts_path = os.path.join(
self.import_call_file_prefix,
part,
)
self.import_call_nodes.add(
(
import_call_header_path,
import_call_parts_path,
collection,
)
)
return True
def _write_edge_headers(self):
"""
Writes single CSV file for a graph entity that is represented
as an edge as per the definition in the `schema_config.yaml`,
containing only the header for this type of edge.
Returns:
bool: The return value. True for success, False otherwise.
"""
# load headers from data parse
if not self.edge_property_dict:
logger.error(
"Header information not found. Was the data parsed first?",
)
return False
for label, props in self.edge_property_dict.items():
# translate label to PascalCase
pascal_label = self.translator.name_sentence_to_pascal(label)
# paths
header = f"{pascal_label}-header.csv"
header_path = os.path.join(
self.outdir,
header,
)
parts = f"{pascal_label}-part.*"
# check for file exists
if os.path.exists(header_path):
logger.warning(
f"Header file {header_path} already exists. Overwriting."
)
# concatenate key:value in props
props_list = []
for k in props.keys():
props_list.append(f"{k}")
out_list = ["_from", "_key", *props_list, "_to"]
with open(header_path, "w", encoding="utf-8") as f:
# concatenate with delimiter
row = self.delim.join(out_list)
f.write(row)
# add collection from schema config
if not self.translator.ontology.mapping.extended_schema.get(label):
for (
_,
v,
) in self.translator.ontology.mapping.extended_schema.items():
if v.get("label_as_edge") == label:
collection = v.get("db_collection_name", None)
break
else:
collection = self.translator.ontology.mapping.extended_schema[
label
].get("db_collection_name", None)
# add file path to neo4 admin import statement (import call path
# may be different from actual output path)
header_import_call_path = os.path.join(
self.import_call_file_prefix,
header,
)
parts_import_call_path = os.path.join(
self.import_call_file_prefix,
parts,
)
self.import_call_edges.add(
(
header_import_call_path,
parts_import_call_path,
collection,
)
)
return True
def _construct_import_call(self) -> str:
"""
Function to construct the import call detailing folder and
individual node and edge headers and data files, as well as
delimiters and database name. Built after all data has been
processed to ensure that nodes are called before any edges.
Returns:
str: a bash command for neo4j-admin import
"""
import_call = (
f"{self.import_call_bin_prefix}arangoimp "
f"--type csv "
f'--separator="{self.escaped_delim}" '
)
if self.quote == "'":
import_call += f'--quote="{self.quote}" '
else:
import_call += f"--quote='{self.quote}' "
node_lines = ""
# node import calls: one line per node type
for header_path, parts_path, collection in self.import_call_nodes:
line = (
f"{import_call} "
f"--headers-file {header_path} "
f"--file= {parts_path} "
)
if collection:
line += f"--create-collection --collection {collection} "
node_lines += f"{line}\n"
edge_lines = ""
# edge import calls: one line per edge type
for header_path, parts_path, collection in self.import_call_edges:
import_call += f'--relationships="{header_path},{parts_path}" '
return node_lines + edge_lines
biocypher/output/write/graph/_neo4j.py
+502
View File
@@ -0,0 +1,502 @@
import os
import glob
import pandas as pd
from biocypher._logger import logger
from biocypher.output.write._batch_writer import parse_label, _BatchWriter
class _Neo4jBatchWriter(_BatchWriter):
"""
Class for writing node and edge representations to disk using the
format specified by Neo4j for the use of admin import. Each batch
writer instance has a fixed representation that needs to be passed
at instantiation via the :py:attr:`schema` argument. The instance
also expects an ontology adapter via :py:attr:`ontology_adapter` to be able
to convert and extend the hierarchy.
This class inherits from the abstract class "_BatchWriter" and implements the
Neo4j-specific methods:
- _write_node_headers
- _write_edge_headers
- _construct_import_call
- _write_array_string
"""
def __init__(self, *args, **kwargs):
"""
Constructor.
Check the version of Neo4j and adds a command scope if version >= 5.
Returns:
_Neo4jBatchWriter: An instance of the writer.
"""
# Should read the configuration and setup import_call_bin_prefix.
super().__init__(*args, **kwargs)
def _get_default_import_call_bin_prefix(self):
"""
Method to provide the default string for the import call bin prefix.
Returns:
str: The default location for the neo4j admin import location
"""
return "bin/"
def _write_array_string(self, string_list):
"""
Abstract method to output.write the string representation of an array into a .csv file
as required by the neo4j admin-import.
Args:
string_list (list): list of ontology strings
Returns:
str: The string representation of an array for the neo4j admin import
"""
string = self.adelim.join(string_list)
return f"{self.quote}{string}{self.quote}"
def _write_node_headers(self):
"""
Writes single CSV file for a graph entity that is represented
as a node as per the definition in the `schema_config.yaml`,
containing only the header for this type of node.
Returns:
bool: The return value. True for success, False otherwise.
"""
# load headers from data parse
if not self.node_property_dict:
logger.error(
"Header information not found. Was the data parsed first?",
)
return False
for label, props in self.node_property_dict.items():
_id = ":ID"
##MeDaX dev remark:
##From Fhir data we get case sensitive labels. E.g. 'Procedure' and 'procedure' are two distinct node types.
##Because we are converting Resources to more specific node classes using their "resourceType" attribute.
# translate label to PascalCase
pascal_label = self.translator.name_sentence_to_pascal(
parse_label(label)
)
header = f"{pascal_label}-header.csv"
header_path = os.path.join(
self.outdir,
header,
)
parts = f"{pascal_label}-part.*"
existing_header = False
# check if file already exists
if os.path.exists(header_path):
logger.warning(
f"Header file `{header_path}` already exists. Overwriting.",
)
with open(header_path, "r", encoding="utf-8") as existing:
existing_header = existing.read().strip().split(self.delim)
# concatenate key:value in props
props_list = []
for k, v in props.items():
if v in ["int", "long", "integer"]:
props_list.append(f"{k}:long")
elif v in ["int[]", "long[]", "integer[]"]:
props_list.append(f"{k}:long[]")
elif v in ["float", "double", "dbl"]:
props_list.append(f"{k}:double")
elif v in ["float[]", "double[]"]:
props_list.append(f"{k}:double[]")
elif v in ["bool", "boolean"]:
# TODO Neo4j boolean support / spelling?
props_list.append(f"{k}:boolean")
elif v in ["bool[]", "boolean[]"]:
props_list.append(f"{k}:boolean[]")
elif v in ["str[]", "string[]"]:
props_list.append(f"{k}:string[]")
else:
props_list.append(f"{k}")
# create list of lists and flatten
out_list = [[_id], props_list, [":LABEL"]]
out_list = [val for sublist in out_list for val in sublist]
with open(header_path, "w", encoding="utf-8") as f:
# Check if header file already exists and has different columns
if os.path.exists(header_path):
if existing_header:
#existing_header = existing.read().strip().split(self.delim)
# Compare existing and new headers
if set(existing_header) != set(out_list):
# Get part files associated with this header
base_name = os.path.basename(header_path).replace("-header.csv", "")
part_files = glob.glob(os.path.join(os.path.dirname(header_path), f"{base_name}-part*.csv"))
# Find the highest numbered part file without full sorting
highest_part = None
highest_number = -1
for part_file in part_files:
try:
# Extract number from filename (assuming format like "part123.csv")
file_name = os.path.basename(part_file)
number_part = file_name.split("part")[1].split(".")[0]
number = int(number_part)
if number > highest_number:
highest_number = number
highest_part = part_file
except (IndexError, ValueError):
# Skip files that don't match the expected pattern
continue
# Update each part file with the new columns
for part_file in part_files:
if part_file == highest_part:
print(f"Skipping the highest part file: {highest_part}")
continue
try:
#print("exi: ", existing_header)
#print("out: ", out_list)
df = self.adapt_csv_to_new_header(existing_header, out_list, part_file)
# Read the file without headers
# Write back to file WITHOUT including the header
df.to_csv(part_file, sep=self.delim, index=False, header=False)
print(f"Updated {part_file} with new columns in correct positions")
except Exception as e:
print(f"Error updating {part_file}: {e}")
# Write the new header
row = self.delim.join(out_list)
f.write(row)
# add file path to neo4 admin import statement (import call file
# path may be different from actual file path)
import_call_header_path = os.path.join(
self.import_call_file_prefix,
header,
)
import_call_parts_path = os.path.join(
self.import_call_file_prefix,
parts,
)
self.import_call_nodes.add(
(import_call_header_path, import_call_parts_path)
)
return True
def _write_edge_headers(self):
"""
Writes single CSV file for a graph entity that is represented
as an edge as per the definition in the `schema_config.yaml`,
containing only the header for this type of edge.
Returns:
bool: The return value. True for success, False otherwise.
"""
# load headers from data parse
if not self.edge_property_dict:
logger.error(
"Header information not found. Was the data parsed first?",
)
return False
for label, props in self.edge_property_dict.items():
# translate label to PascalCase
pascal_label = self.translator.name_sentence_to_pascal(
parse_label(label)
)
# paths
header = f"{pascal_label}-header.csv"
header_path = os.path.join(
self.outdir,
header,
)
parts = f"{pascal_label}-part.*"
# check for file exists
if os.path.exists(header_path):
logger.warning(
f"File {header_path} already exists. Overwriting."
)
# concatenate key:value in props
props_list = []
for k, v in props.items():
if v in ["int", "long", "integer"]:
props_list.append(f"{k}:long")
elif v in ["int[]", "long[]", "integer[]"]:
props_list.append(f"{k}:long[]")
elif v in ["float", "double"]:
props_list.append(f"{k}:double")
elif v in ["float[]", "double[]"]:
props_list.append(f"{k}:double[]")
elif v in [
"bool",
"boolean",
]: # TODO does Neo4j support bool?
props_list.append(f"{k}:boolean")
elif v in ["bool[]", "boolean[]"]:
props_list.append(f"{k}:boolean[]")
elif v in ["str[]", "string[]"]:
props_list.append(f"{k}:string[]")
else:
props_list.append(f"{k}")
skip_id = False
schema_label = None
if label in ["IS_SOURCE_OF", "IS_TARGET_OF", "IS_PART_OF"]:
skip_id = True
elif not self.translator.ontology.mapping.extended_schema.get(
label
):
# find label in schema by label_as_edge
for (
k,
v,
) in self.translator.ontology.mapping.extended_schema.items():
if v.get("label_as_edge") == label:
schema_label = k
break
else:
schema_label = label
out_list = [":START_ID"]
if schema_label:
if (
self.translator.ontology.mapping.extended_schema.get(
schema_label
).get("use_id")
== False
):
skip_id = True
if not skip_id:
out_list.append("id")
out_list.extend(props_list)
out_list.extend([":END_ID", ":TYPE"])
existing_header = False
# check if file already exists
if os.path.exists(header_path):
logger.warning(
f"Header file `{header_path}` already exists. Overwriting.",
)
with open(header_path, "r", encoding="utf-8") as existing:
existing_header = existing.read().strip().split(self.delim)
with open(header_path, "w", encoding="utf-8") as f:
# Check if header file already exists and has different columns
if os.path.exists(header_path):
if existing_header:
#existing_header = existing.read().strip().split(self.delim)
# Compare existing and new headers
if set(existing_header) != set(out_list):
# Get part files associated with this header
base_name = os.path.basename(header_path).replace("-header.csv", "")
part_files = glob.glob(os.path.join(os.path.dirname(header_path), f"{base_name}-part*.csv"))
# Find the highest numbered part file without full sorting
highest_part = None
highest_number = -1
for part_file in part_files:
try:
# Extract number from filename (assuming format like "part123.csv")
file_name = os.path.basename(part_file)
number_part = file_name.split("part")[1].split(".")[0]
number = int(number_part)
if number > highest_number:
highest_number = number
highest_part = part_file
except (IndexError, ValueError):
# Skip files that don't match the expected pattern
continue
# Update each part file with the new columns
for part_file in part_files:
if part_file == highest_part:
print(f"Skipping the highest part file: {highest_part}")
continue
try:
print("exi: ", existing_header)
print("out: ", out_list)
df = self.adapt_csv_to_new_header(existing_header, out_list, part_file)
# Read the file without headers
# Write back to file WITHOUT including the header
df.to_csv(part_file, sep=self.delim, index=False, header=False)
print(f"Updated {part_file} with new columns in correct positions")
except Exception as e:
print(f"Error updating {part_file}: {e}")
# Write the new header
row = self.delim.join(out_list)
f.write(row)
# add file path to neo4 admin import statement (import call file
# path may be different from actual file path)
import_call_header_path = os.path.join(
self.import_call_file_prefix,
header,
)
import_call_parts_path = os.path.join(
self.import_call_file_prefix,
parts,
)
self.import_call_edges.add(
(import_call_header_path, import_call_parts_path)
)
return True
def _get_import_script_name(self) -> str:
"""
Returns the name of the neo4j admin import script
Returns:
str: The name of the import script (ending in .sh)
"""
return "neo4j-admin-import-call.sh"
def _construct_import_call(self) -> str:
"""
Function to construct the import call detailing folder and
individual node and edge headers and data files, as well as
delimiters and database name. Built after all data has been
processed to ensure that nodes are called before any edges.
Returns:
str: a bash command for neo4j-admin import
"""
import_call_neo4j_v4 = self._get_import_call(
"import", "--database=", "--force="
)
import_call_neo4j_v5 = self._get_import_call(
"database import full", "", "--overwrite-destination="
)
neo4j_version_check = f"version=$({self._get_default_import_call_bin_prefix()}neo4j-admin --version | cut -d '.' -f 1)"
import_script = f"#!/bin/bash\n{neo4j_version_check}\nif [[ $version -ge 5 ]]; then\n\t{import_call_neo4j_v5}\nelse\n\t{import_call_neo4j_v4}\nfi"
return import_script
def _get_import_call(
self, import_cmd: str, database_cmd: str, wipe_cmd: str
) -> str:
"""Get parametrized import call for Neo4j 4 or 5+.
Args:
import_cmd (str): The import command to use.
database_cmd (str): The database command to use.
wipe_cmd (str): The wipe command to use.
Returns:
str: The import call.
"""
import_call = f"{self.import_call_bin_prefix}neo4j-admin {import_cmd} "
import_call += f"{database_cmd}{self.db_name} "
import_call += f'--delimiter="{self.escaped_delim}" '
import_call += f'--array-delimiter="{self.escaped_adelim}" '
if self.quote == "'":
import_call += f'--quote="{self.quote}" '
else:
import_call += f"--quote='{self.quote}' "
if self.wipe:
import_call += f"{wipe_cmd}true "
if self.skip_bad_relationships:
import_call += "--skip-bad-relationships=true "
if self.skip_duplicate_nodes:
import_call += "--skip-duplicate-nodes=true "
# append node import calls
for header_path, parts_path in self.import_call_nodes:
import_call += f'--nodes="{header_path},{parts_path}" '
# append edge import calls
for header_path, parts_path in self.import_call_edges:
import_call += f'--relationships="{header_path},{parts_path}" '
return import_call
def adapt_csv_to_new_header(self, old_header, new_header, csv_file_path):
"""
Adapt a CSV table to a new header structure, placing new columns in their correct positions.
Parameters:
old_header (list): The original header columns
new_header (list): The new header columns
csv_file_path (str): Path to the CSV file
Returns:
pandas.DataFrame: CSV data with the new header structure
"""
# Step 1: Read the CSV data without headers
df = pd.read_csv(csv_file_path, sep=self.delim, header=None)
# Step 2: If the file is empty, return empty DataFrame with new headers
if df.empty:
return pd.DataFrame(columns=new_header)
# Step 3: If column count doesn't match old_header length, handle the mismatch
if len(df.columns) != len(old_header):
print(f"Warning: CSV columns count ({len(df.columns)}) doesn't match the provided old header count ({len(old_header)})")
# If file has fewer columns than old_header, pad with NaN
if len(df.columns) < len(old_header):
for i in range(len(df.columns), len(old_header)):
df[i] = None
# If file has more columns than old_header, truncate
else:
df = df.iloc[:, :len(old_header)]
# Step 4: Assign old header names to the dataframe
df.columns = old_header
# Step 5: Create a new DataFrame with the correct structure
new_df = pd.DataFrame(columns=new_header)
# Step 6: For each column in the new header, find its position in the old header
for new_col_idx, new_col in enumerate(new_header):
if new_col in old_header:
# If column exists in old header, copy data
new_df[new_col] = df[new_col]
else:
# If new column, add empty column
new_df[new_col] = None
# Step 7: Ensure columns are in the exact order of new_header
new_df = new_df[new_header]
return new_df
biocypher/output/write/graph/_networkx.py
+76
View File
@@ -0,0 +1,76 @@
import pickle
import networkx as nx
from biocypher._logger import logger
from biocypher.output.write._writer import _Writer
from biocypher.output.write.relational._csv import _PandasCSVWriter
class _NetworkXWriter(_Writer):
"""
Class for writing node and edges to a networkx DiGraph.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.csv_writer = _PandasCSVWriter(*args, write_to_file=False, **kwargs)
self.G = nx.DiGraph()
def _construct_import_call(self) -> str:
"""Function to construct the Python code to load all node and edge csv files again into Pandas dfs.
Returns:
str: Python code to load the csv files into Pandas dfs.
"""
logger.info(
f"Writing networkx {self.G} to pickle file networkx_graph.pkl."
)
with open(f"{self.output_directory}/networkx_graph.pkl", "wb") as f:
pickle.dump(self.G, f)
import_call = "import pickle\n"
import_call += "with open('./networkx_graph.pkl', 'rb') as f:\n\tG_loaded = pickle.load(f)"
return import_call
def _get_import_script_name(self) -> str:
"""Function to return the name of the import script."""
return "import_networkx.py"
def _write_node_data(self, nodes) -> bool:
passed = self.csv_writer._write_entities_to_file(nodes)
self.add_to_networkx()
return passed
def _write_edge_data(self, edges) -> bool:
passed = self.csv_writer._write_entities_to_file(edges)
self.add_to_networkx()
return passed
def add_to_networkx(self) -> bool:
all_dfs = self.csv_writer.stored_dfs
node_dfs = [
df
for df in all_dfs.values()
if df.columns.str.contains("node_id").any()
]
edge_dfs = [
df
for df in all_dfs.values()
if df.columns.str.contains("source_id").any()
and df.columns.str.contains("target_id").any()
]
for df in node_dfs:
nodes = df.set_index("node_id").to_dict(orient="index")
self.G.add_nodes_from(nodes.items())
for df in edge_dfs:
edges = df.set_index(["source_id", "target_id"]).to_dict(
orient="index"
)
self.G.add_edges_from(
(
(source, target, attrs)
for (source, target), attrs in edges.items()
)
)
return True
biocypher/output/write/graph/_rdf.py
+515
View File
@@ -0,0 +1,515 @@
#!/usr/bin/env python
#
# Copyright 2021, Heidelberg University Clinic
#
# File author(s): Loes van den Biggelaar
# Sebastian Lobentanzer
#
# Distributed under MIT licence, see the file `LICENSE`.
#
"""
BioCypher 'offline' module. Handles the writing of node and edge representations
suitable for import into a DBMS.
"""
from types import GeneratorType
from typing import Union
import os
from rdflib import DC, RDF, RDFS, SKOS, DCTERMS, Graph, Literal, Namespace
from rdflib.namespace import (
_NAMESPACE_PREFIXES_CORE,
_NAMESPACE_PREFIXES_RDFLIB,
)
from biocypher._create import BioCypherEdge, BioCypherNode
from biocypher._logger import logger
from biocypher.output.write._batch_writer import _BatchWriter
class _RDFWriter(_BatchWriter):
"""
Class to write BioCypher's property graph into an RDF format using
rdflib and all the extensions it supports (RDF/XML, N3, NTriples,
N-Quads, Turtle, TriX, Trig and JSON-LD). By default the conversion
is done keeping only the minimum information about node and edges,
skipping all properties.
"""
def _get_import_script_name(self) -> str:
"""
Returns the name of the RDF admin import script.
This function applicable for RDF export.
Returns:
str: The name of the import script (ending in .sh)
"""
return "rdf-import-call.sh"
def _get_default_import_call_bin_prefix(self):
"""
Method to provide the default string for the import call bin prefix.
Returns:
str: The default location for the RDF admin import location
"""
return "bin/"
def _is_rdf_format_supported(self, rdf_format: str) -> bool:
"""
Function to check if the specified RDF format is supported.
Args:
rdf_format (str): The RDF format to check.
Returns:
bool: Returns True if rdf format supported, False otherwise.
"""
supported_formats = [
"xml",
"n3",
"turtle",
"nt",
"pretty-xml",
"trix",
"trig",
"nquads",
"json-ld",
]
if rdf_format not in supported_formats:
logger.error(
f"{rdf_format}; Incorrect or unsupported RDF format, use one of the following: "
f'"xml", "n3", "turtle", "nt", "pretty-xml", "trix", "trig", "nquads", "json-ld" ',
)
return False
else:
# RDF graph does not support 'ttl' format, only 'turtle' format. however, the preferred file extension is always '.ttl'
if self.rdf_format == "turtle":
self.extension = "ttl"
elif self.rdf_format == "ttl":
self.rdf_format = "turtle"
self.extension = "ttl"
else:
self.extension = self.rdf_format
return True
def _write_single_edge_list_to_file(
self,
edge_list: list,
label: str,
prop_dict: dict,
):
"""
This function takes one list of biocypher edges and writes them
to an RDF file with the given format.
Args:
edge_list (list): list of BioCypherEdges to be written
label (str): the label (type) of the edge
prop_dict (dict): properties of node class passed from parsing
function and their types
Returns:
bool: The return value. True for success, False otherwise.
"""
if not all(isinstance(n, BioCypherEdge) for n in edge_list):
logger.error("Edges must be passed as type BioCypherEdge.")
return False
# translate label to PascalCase
label_pascal = self.translator.name_sentence_to_pascal(label)
# create file name
file_name = os.path.join(
self.outdir, f"{label_pascal}.{self.extension}"
)
# write data in graph
graph = Graph()
self._init_namespaces(graph)
for edge in edge_list:
rdf_subject = edge.get_source_id()
rdf_object = edge.get_target_id()
rdf_predicate = edge.get_id()
rdf_properties = edge.get_properties()
if rdf_predicate == None:
rdf_predicate = rdf_subject + rdf_object
edge_label = self.translator.name_sentence_to_pascal(
edge.get_label()
)
edge_uri = self.rdf_namespaces["biocypher"][edge_label]
graph.add((edge_uri, RDF.type, RDFS.Class))
graph.add(
(
self.rdf_namespaces["biocypher"][rdf_predicate],
RDF.type,
edge_uri,
)
)
graph.add(
(
self.rdf_namespaces["biocypher"][rdf_predicate],
self.rdf_namespaces["biocypher"]["subject"],
self.subject_to_uri(rdf_subject),
)
)
graph.add(
(
self.rdf_namespaces["biocypher"][rdf_predicate],
self.rdf_namespaces["biocypher"]["object"],
self.subject_to_uri(rdf_object),
)
)
# add properties to the transformed edge --> node
for key, value in rdf_properties.items():
# only write value if it exists.
if value:
self.add_property_to_graph(graph, rdf_predicate, value, key)
graph.serialize(destination=file_name, format=self.rdf_format)
logger.info(
f"Writing {len(edge_list)} entries to {label_pascal}.{self.rdf_format}",
)
return True
def add_property_to_graph(
self,
graph: Graph,
rdf_subject: str,
rdf_object: str,
rdf_predicate: str,
):
"""
Function to add the properties to an RDF node. It takes the graph, the subject, object, and predicate of the RDF triple.
It checks if the property is a list and adds it to the graph accordingly. otherwise it checks if the string represents a list.
If it does, it transforms it to a list and adds it to the graph. if not, it adds the property to the graph as a literal.
If the property is neither a list or string, it will also be added as a literal.
Args:
graph (RDFLib.Graph): The RDF graph to add the nodes to.
rdf_subject (str): The subject of the RDF triple.
rdf_object (str): The object of the RDF triple.
rdf_predicate (str): The predicate of the RDF triple.
Returns:
None
"""
if isinstance(rdf_object, list):
for obj in rdf_object:
graph.add(
(
self.subject_to_uri(rdf_subject),
self.property_to_uri(rdf_predicate),
Literal(obj),
)
)
elif isinstance(rdf_object, str):
if rdf_object.startswith("[") and rdf_object.endswith("]"):
self.add_property_to_graph(
graph,
rdf_subject,
self.transform_string_to_list(rdf_object),
rdf_predicate,
)
else:
graph.add(
(
self.subject_to_uri(rdf_subject),
self.property_to_uri(rdf_predicate),
Literal(rdf_object),
)
)
else:
graph.add(
(
self.subject_to_uri(rdf_subject),
self.property_to_uri(rdf_predicate),
Literal(rdf_object),
)
)
def transform_string_to_list(self, string_list: str) -> list:
"""
Function to transform a string representation of a list into a list.
Args:
string_list (str): The string representation of the list.
Returns:
list: The list representation of the input string.
"""
return (
string_list.replace("[", "")
.replace("]", "")
.replace("'", "")
.split(", ")
)
def _write_single_node_list_to_file(
self,
node_list: list,
label: str,
prop_dict: dict,
labels: str,
):
"""
This function takes a list of BioCypherNodes and writes them
to an RDF file in the specified format.
Args:
node_list (list): A list of BioCypherNodes to be written.
label (str): The label (type) of the nodes.
prop_dict (dict): A dictionary of properties and their types for the node class.
Returns:
bool: True if the writing is successful, False otherwise.
"""
if not all(isinstance(n, BioCypherNode) for n in node_list):
logger.error("Nodes must be passed as type BioCypherNode.")
return False
# translate label to PascalCase
label_pascal = self.translator.name_sentence_to_pascal(label)
# create file name
file_name = os.path.join(
self.outdir, f"{label_pascal}.{self.extension}"
)
# write data in graph
graph = Graph()
self._init_namespaces(graph)
for n in node_list:
rdf_subject = n.get_id()
rdf_object = n.get_label()
properties = n.get_properties()
class_name = self.translator.name_sentence_to_pascal(rdf_object)
graph.add(
(
self.rdf_namespaces["biocypher"][class_name],
RDF.type,
RDFS.Class,
)
)
graph.add(
(
self.subject_to_uri(rdf_subject),
RDF.type,
self.rdf_namespaces["biocypher"][class_name],
)
)
for key, value in properties.items():
# only write value if it exists.
if value:
self.add_property_to_graph(graph, rdf_subject, value, key)
graph.serialize(destination=file_name, format=self.rdf_format)
logger.info(
f"Writing {len(node_list)} entries to {label_pascal}.{self.rdf_format}",
)
return True
def write_nodes(
self, nodes, batch_size: int = int(1e6), force: bool = False
) -> bool:
"""
Wrapper for writing nodes in RDF format. It calls the _write_node_data() function, specifying the node data.
Args:
nodes (list or generator): A list or generator of nodes in BioCypherNode format.
batch_size (int): The number of nodes to write in each batch.
force (bool): Flag to force the writing even if the output file already exists.
Returns:
bool: True if the writing is successful, False otherwise.
"""
# check if specified output format is correct
passed = self._is_rdf_format_supported(self.rdf_format)
if not passed:
logger.error("Error while writing node data, wrong RDF format")
return False
# write node data using _write_node_data method
passed = self._write_node_data(nodes, batch_size, force)
if not passed:
logger.error("Error while writing node data.")
return False
return True
def write_edges(
self,
edges: Union[list, GeneratorType],
batch_size: int = int(1e6),
) -> bool:
"""
Wrapper for writing edges in RDF format. It calls _write_edge_data()
functions specifying it's edge data.
Args:
edges (BioCypherEdge): a list or generator of edges in
:py:class:`BioCypherEdge` format
batch_size (int): The number of edges to write in each batch.
Returns:
bool: The return value. True for success, False otherwise.
"""
# check if specified output format is correct
passed = self._is_rdf_format_supported(self.rdf_format)
if not passed:
logger.error("Error while writing edge data, wrong RDF format")
return False
# write edge data using _write_edge_data method
passed = self._write_edge_data(edges, batch_size=batch_size)
if not passed:
logger.error("Error while writing edge data.")
return False
return True
def _construct_import_call(self) -> bool:
"""
Function to write the import call.
This function is not applicable for RDF.
Returns:
bool: The return value. True for success, False otherwise.
"""
return ""
def _write_array_string(self, string_list):
"""
Abstract method to write the string representation of an array into a .csv file
as required by the RDF admin-import.
This function is not applicable for RDF.
Args:
string_list (list): list of ontology strings
Returns:
str: The string representation of an array for the neo4j admin import
"""
return True
def _write_node_headers(self):
"""
Abstract method that takes care of importing properties of a graph entity that is represented
as a node as per the definition in the `schema_config.yaml`
This function is not applicable for RDF.
Returns:
bool: The return value. True for success, False otherwise.
"""
return True
def _write_edge_headers(self):
"""
Abstract method to write a database import-file for a graph entity that is represented
as an edge as per the definition in the `schema_config.yaml`,
containing only the header for this type of edge.
This function is not applicable for RDF.
Returns:
bool: The return value. True for success, False otherwise.
"""
return True
def subject_to_uri(self, subject: str) -> str:
"""
Converts the subject to a proper URI using the available namespaces.
If the conversion fails, it defaults to the biocypher prefix.
Args:
subject (str): The subject to be converted to a URI.
Returns:
str: The corresponding URI for the subject.
"""
try:
_pref, _id = subject.split(":")
if _pref in self.rdf_namespaces.keys():
return self.rdf_namespaces[_pref][_id]
else:
return self.rdf_namespaces["biocypher"][subject]
except ValueError:
return self.rdf_namespaces["biocypher"][subject]
def property_to_uri(self, property_name: str) -> dict[str, str]:
"""
Converts a property name to its corresponding URI.
This function takes a property name and searches for its corresponding URI in various namespaces.
It first checks the core namespaces for rdflib, including owl, rdf, rdfs, xsd, and xml.
Args:
property_name (str): The property name to be converted to a URI.
Returns:
str: The corresponding URI for the input property name.
"""
# These namespaces are core for rdflib; owl, rdf, rdfs, xsd and xml
for namespace in _NAMESPACE_PREFIXES_CORE.values():
if property_name in namespace:
return namespace[property_name]
# If the property name is not found in the core namespaces, search in the SKOS, DC, and DCTERMS namespaces
for namespace in [SKOS, DC, DCTERMS]:
if property_name in namespace:
return namespace[property_name]
# If the property name is still not found, try other namespaces from rdflib.
for namespace in _NAMESPACE_PREFIXES_RDFLIB.values():
if property_name in namespace:
return namespace[property_name]
# If the property name is "licence", it recursively calls the function with "license" as the input.
if property_name == "licence":
return self.property_to_uri("license")
# TODO: add an option to search trough manually implemented namespaces
# If the input is not found in any of the namespaces, it returns the corresponding URI from the biocypher namespace.
# TODO: give a warning and try to prevent this option altogether
return self.rdf_namespaces["biocypher"][property_name]
def _init_namespaces(self, graph: Graph):
"""
Initializes the namespaces for the RDF graph. These namespaces are used to convert nodes to URIs.
This function adds the biocypher standard namespace to the `rdf_namespaces` attribute of the class.
If `rdf_namespaces` is empty, it sets it to the biocypher standard namespace. Otherwise, it merges
the biocypher standard namespace with the namespaces defined in the biocypher_config.yaml.
Args:
graph (RDFLib.Graph): The RDF graph to bind the namespaces to.
Returns:
None
"""
# add biocypher standard to self.rdf_namespaces
biocypher_standard = {"biocypher": "https://biocypher.org/biocypher#"}
if not self.rdf_namespaces:
self.rdf_namespaces = biocypher_standard
else:
self.rdf_namespaces = self.rdf_namespaces | biocypher_standard
for key, value in self.rdf_namespaces.items():
namespace = Namespace(value)
self.rdf_namespaces[key] = namespace
graph.bind(key, namespace)
biocypher/output/write/relational/__init__.py
View File
Whitespace-only changes.
biocypher/output/write/relational/_csv.py
+76
View File
@@ -0,0 +1,76 @@
from more_itertools import peekable
from biocypher._logger import logger
from biocypher.output.write._writer import _Writer
from biocypher.output.in_memory._pandas import Pandas
class _PandasCSVWriter(_Writer):
"""
Class for writing node and edge representations to a CSV file.
"""
def __init__(self, *args, write_to_file: bool = True, **kwargs):
kwargs["write_to_file"] = write_to_file
super().__init__(*args, **kwargs)
self.in_memory_dfs = {}
self.stored_dfs = {}
self.pandas_in_memory = Pandas(
translator=self.translator,
deduplicator=self.deduplicator,
)
self.delimiter = kwargs.get("delimiter")
if not self.delimiter:
self.delimiter = ","
self.write_to_file = write_to_file
def _construct_import_call(self) -> str:
"""Function to construct the Python code to load all node and edge csv files again into Pandas dfs.
Returns:
str: Python code to load the csv files into Pandas dfs.
"""
import_call = "import pandas as pd\n\n"
for df_name in self.stored_dfs.keys():
import_call += f"{df_name} = pd.read_csv('./{df_name}.csv', header=0, index_col=0)\n"
return import_call
def _get_import_script_name(self) -> str:
"""Function to return the name of the import script."""
return "import_pandas_csv.py"
def _write_node_data(self, nodes) -> bool:
passed = self._write_entities_to_file(nodes)
return passed
def _write_edge_data(self, edges) -> bool:
passed = self._write_entities_to_file(edges)
return passed
def _write_entities_to_file(self, entities: iter) -> bool:
"""Function to output.write the entities to a CSV file.
Args:
entities (iterable): An iterable of BioCypherNode / BioCypherEdge / BioCypherRelAsNode objects.
"""
entities = peekable(entities)
entity_list = self.pandas_in_memory._separate_entity_types(entities)
for entity_type, entities in entity_list.items():
self.in_memory_dfs[
entity_type
] = self.pandas_in_memory._add_entity_df(entity_type, entities)
for entity_type in self.in_memory_dfs.keys():
entity_df = self.in_memory_dfs[entity_type]
if " " in entity_type or "." in entity_type:
entity_type = entity_type.replace(" ", "_").replace(".", "_")
if self.write_to_file:
logger.info(
f"Writing {entity_df.shape[0]} entries to {entity_type}.csv."
)
entity_df.to_csv(
f"{self.output_directory}/{entity_type}.csv",
sep=self.delimiter,
)
self.stored_dfs[entity_type] = entity_df
self.in_memory_dfs = {}
return True
biocypher/output/write/relational/_postgresql.py
+320
View File
@@ -0,0 +1,320 @@
import os
import glob
from biocypher._logger import logger
from biocypher.output.write._batch_writer import _BatchWriter
class _PostgreSQLBatchWriter(_BatchWriter):
"""
Class for writing node and edge representations to disk using the
format specified by PostgreSQL for the use of "COPY FROM...". Each batch
writer instance has a fixed representation that needs to be passed
at instantiation via the :py:attr:`schema` argument. The instance
also expects an ontology adapter via :py:attr:`ontology_adapter` to be able
to convert and extend the hierarchy.
This class inherits from the abstract class "_BatchWriter" and implements the
PostgreSQL-specific methods:
- _write_node_headers
- _write_edge_headers
- _construct_import_call
- _write_array_string
"""
DATA_TYPE_LOOKUP = {
"str": "VARCHAR", # VARCHAR needs limit
"int": "INTEGER",
"long": "BIGINT",
"float": "NUMERIC",
"double": "NUMERIC",
"dbl": "NUMERIC",
"boolean": "BOOLEAN",
"str[]": "VARCHAR[]",
"string[]": "VARCHAR[]",
}
def __init__(self, *args, **kwargs):
self._copy_from_csv_commands = set()
super().__init__(*args, **kwargs)
def _get_default_import_call_bin_prefix(self):
"""
Method to provide the default string for the import call bin prefix.
Returns:
str: The default location for the psql command
"""
return ""
def _get_data_type(self, string) -> str:
try:
return self.DATA_TYPE_LOOKUP[string]
except KeyError:
logger.info(
'Could not determine data type {string}. Using default "VARCHAR"'
)
return "VARCHAR"
def _write_array_string(self, string_list) -> str:
"""
Abstract method to output.write the string representation of an array into a .csv file
as required by the postgresql COPY command, with '{','}' brackets and ',' separation.
Args:
string_list (list): list of ontology strings
Returns:
str: The string representation of an array for postgres COPY
"""
string = ",".join(string_list)
string = f'"{{{string}}}"'
return string
def _get_import_script_name(self) -> str:
"""
Returns the name of the psql import script
Returns:
str: The name of the import script (ending in .sh)
"""
return f"{self.db_name}-import-call.sh"
def _adjust_pascal_to_psql(self, string):
string = string.replace(".", "_")
string = string.lower()
return string
def _write_node_headers(self):
"""
Writes single CSV file for a graph entity that is represented
as a node as per the definition in the `schema_config.yaml`,
containing only the header for this type of node.
Returns:
bool: The return value. True for success, False otherwise.
"""
# load headers from data parse
if not self.node_property_dict:
logger.error(
"Header information not found. Was the data parsed first?",
)
return False
for label, props in self.node_property_dict.items():
# create header CSV with ID, properties, labels
# translate label to PascalCase
pascal_label = self.translator.name_sentence_to_pascal(label)
parts = f"{pascal_label}-part*.csv"
parts_paths = os.path.join(self.outdir, parts)
parts_paths = glob.glob(parts_paths)
parts_paths.sort()
# adjust label for import to psql
pascal_label = self._adjust_pascal_to_psql(pascal_label)
table_create_command_path = os.path.join(
self.outdir,
f"{pascal_label}-create_table.sql",
)
# check if file already exists
if os.path.exists(table_create_command_path):
logger.warning(
f"File {table_create_command_path} already exists. Overwriting.",
)
# concatenate key:value in props
columns = ["_ID VARCHAR"]
for col_name, col_type in props.items():
col_type = self._get_data_type(col_type)
col_name = self._adjust_pascal_to_psql(col_name)
columns.append(f"{col_name} {col_type}")
columns.append("_LABEL VARCHAR[]")
with open(table_create_command_path, "w", encoding="utf-8") as f:
command = ""
if self.wipe:
command += f"DROP TABLE IF EXISTS {pascal_label};\n"
# table creation requires comma separation
command += (
f'CREATE TABLE {pascal_label}({",".join(columns)});\n'
)
f.write(command)
for parts_path in parts_paths:
# if import_call_file_prefix is set, replace actual path
# with prefix
if self.import_call_file_prefix != self.outdir:
parts_path = parts_path.replace(
self.outdir,
self.import_call_file_prefix,
)
self._copy_from_csv_commands.add(
f"\\copy {pascal_label} FROM '{parts_path}' DELIMITER E'{self.delim}' CSV;"
)
# add file path to import statement
# if import_call_file_prefix is set, replace actual path
# with prefix
if self.import_call_file_prefix != self.outdir:
table_create_command_path = table_create_command_path.replace(
self.outdir,
self.import_call_file_prefix,
)
self.import_call_nodes.add(table_create_command_path)
return True
def _write_edge_headers(self):
"""
Writes single CSV file for a graph entity that is represented
as an edge as per the definition in the `schema_config.yaml`,
containing only the header for this type of edge.
Returns:
bool: The return value. True for success, False otherwise.
"""
# load headers from data parse
if not self.edge_property_dict:
logger.error(
"Header information not found. Was the data parsed first?",
)
return False
for label, props in self.edge_property_dict.items():
# translate label to PascalCase
pascal_label = self.translator.name_sentence_to_pascal(label)
parts_paths = os.path.join(self.outdir, f"{pascal_label}-part*.csv")
parts_paths = glob.glob(parts_paths)
parts_paths.sort()
# adjust label for import to psql
pascal_label = self._adjust_pascal_to_psql(pascal_label)
table_create_command_path = os.path.join(
self.outdir,
f"{pascal_label}-create_table.sql",
)
# check for file exists
if os.path.exists(table_create_command_path):
logger.warning(
f"File {table_create_command_path} already exists. Overwriting.",
)
# concatenate key:value in props
columns = []
for col_name, col_type in props.items():
col_type = self._get_data_type(col_type)
col_name = self._adjust_pascal_to_psql(col_name)
if col_name == "_ID":
# should ideally never happen
raise ValueError(
"Column name '_ID' is reserved for internal use, "
"denoting the relationship ID. Please choose a "
"different name for your column."
)
columns.append(f"{col_name} {col_type}")
# create list of lists and flatten
# removes need for empty check of property list
out_list = [
"_START_ID VARCHAR",
"_ID VARCHAR",
*columns,
"_END_ID VARCHAR",
"_TYPE VARCHAR",
]
with open(table_create_command_path, "w", encoding="utf-8") as f:
command = ""
if self.wipe:
command += f"DROP TABLE IF EXISTS {pascal_label};\n"
# table creation requires comma separation
command += (
f'CREATE TABLE {pascal_label}({",".join(out_list)});\n'
)
f.write(command)
for parts_path in parts_paths:
# if import_call_file_prefix is set, replace actual path
# with prefix
if self.import_call_file_prefix != self.outdir:
parts_path = parts_path.replace(
self.outdir,
self.import_call_file_prefix,
)
self._copy_from_csv_commands.add(
f"\\copy {pascal_label} FROM '{parts_path}' DELIMITER E'{self.delim}' CSV;"
)
# add file path to import statement
# if import_call_file_prefix is set, replace actual path
# with prefix
if self.import_call_file_prefix != self.outdir:
table_create_command_path = table_create_command_path.replace(
self.outdir,
self.import_call_file_prefix,
)
self.import_call_edges.add(table_create_command_path)
return True
def _construct_import_call(self) -> str:
"""
Function to construct the import call detailing folder and
individual node and edge headers and data files, as well as
delimiters and database name. Built after all data has been
processed to ensure that nodes are called before any edges.
Returns:
str: a bash command for postgresql import
"""
import_call = ""
# create tables
# At this point, csv files of nodes and edges do not require differentiation
for import_file_path in [
*self.import_call_nodes,
*self.import_call_edges,
]:
import_call += f'echo "Setup {import_file_path}..."\n'
if {self.db_password}:
# set password variable inline
import_call += f"PGPASSWORD={self.db_password} "
import_call += (
f"{self.import_call_bin_prefix}psql -f {import_file_path}"
)
import_call += f" --dbname {self.db_name}"
import_call += f" --host {self.db_host}"
import_call += f" --port {self.db_port}"
import_call += f" --user {self.db_user}"
import_call += '\necho "Done!"\n'
import_call += "\n"
# copy data to tables
for command in self._copy_from_csv_commands:
table_part = command.split(" ")[3]
import_call += f'echo "Importing {table_part}..."\n'
if {self.db_password}:
# set password variable inline
import_call += f"PGPASSWORD={self.db_password} "
import_call += f'{self.import_call_bin_prefix}psql -c "{command}"'
import_call += f" --dbname {self.db_name}"
import_call += f" --host {self.db_host}"
import_call += f" --port {self.db_port}"
import_call += f" --user {self.db_user}"
import_call += '\necho "Done!"\n'
import_call += "\n"
return import_call
biocypher/output/write/relational/_sqlite.py
+51
View File
@@ -0,0 +1,51 @@
from biocypher.output.write.relational._postgresql import _PostgreSQLBatchWriter
class _SQLiteBatchWriter(_PostgreSQLBatchWriter):
"""
Class for writing node and edge representations to a SQLite database.
It uses the _PostgreSQLBatchWriter class under the hood, which already
implements the logic to write the nodes/edges to a relational DBMS.
Only the import bash script differs between PostgreSQL and SQLite
and is therefore implemented in this class.
- _construct_import_call
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _construct_import_call(self) -> str:
"""
Function to construct the import call detailing folder and
individual node and edge headers and data files, as well as
delimiters and database name. Built after all data has been
processed to ensure that nodes are called before any edges.
Returns:
str: a bash command for sqlite import
"""
import_call = ""
# create tables
# At this point, csv files of nodes and edges do not require differentiation
for import_file_path in [
*self.import_call_nodes,
*self.import_call_edges,
]:
import_call += f'echo "Setup {import_file_path}..."\n'
import_call += f"{self.import_call_bin_prefix}sqlite3 {self.db_name} < {import_file_path}"
import_call += '\necho "Done!"\n'
import_call += "\n"
for command in self._copy_from_csv_commands:
table_name = command.split(" ")[1]
table_part = command.split(" ")[3].replace("'", "")
import_call += f'echo "Importing {table_part}..."\n'
separator = self.delim
import_part = f".import {table_part} {table_name}"
import_call += f"{self.import_call_bin_prefix}sqlite3 -separator $'{separator}' {self.db_name} \"{import_part}\""
import_call += '\necho "Done!"\n'
import_call += "\n"
return import_call
config/ODM_schema.yaml
-215
View File
@@ -1,215 +0,0 @@
# BioCypher ODM Import Schema Configuration with BioLink Model Integration
# Node types extracted from ODM generators mapped to BioLink classes
study:
represented_as: node
preferred_id: oid
label_in_input: Study
is_a: biolink:ClinicalTrial
properties:
OID: str
Name: str
Description: str
Protocol: str
ModelID: str
studyevent:
represented_as: node
preferred_id: oid
label_in_input: StudyEvent
is_a: biolink:ClinicalEntity
properties:
OID: str
Name: str
Type: str
StudyOID: str
ModelID: str
form:
represented_as: node
preferred_id: oid
label_in_input: Form
is_a: biolink:InformationContentEntity
properties:
OID: str
Name: str
StudyOID: str
ModelID: str
itemgroup:
represented_as: node
preferred_id: oid
label_in_input: ItemGroup
is_a: biolink:InformationContentEntity
properties:
OID: str
Name: str
StudyOID: str
ModelID: str
item:
represented_as: node
preferred_id: oid
label_in_input: Item
is_a: biolink:ClinicalFinding
properties:
OID: str
Name: str
DataType: str
Question: str
StudyOID: str
ModelID: str
alias:
represented_as: node
preferred_id: [context, name]
label_in_input: Alias
is_a: biolink:InformationContentEntity
properties:
Context: str
Name: str
ItemOID: str
StudyOID: str
ModelID: str
measurementunit:
represented_as: node
preferred_id: oid
label_in_input: MeasurementUnit
is_a: biolink:Unit
properties:
OID: str
Name: str
Symbol: str
basicdefinitions:
represented_as: node
preferred_id: [studyoid, modelid]
label_in_input: BasicDefinitions
is_a: biolink:InformationContentEntity
properties:
StudyOID: str
ModelID: str
rangecheck:
represented_as: node
preferred_id: [comparator, constraint, checkvalue]
label_in_input: RangeCheck
is_a: biolink:ClinicalFinding
properties:
Comparator: str
Constraint: str
CheckValue: str
codelist:
represented_as: node
preferred_id: oid
label_in_input: CodeList
is_a: biolink:InformationContentEntity
properties:
OID: str
Name: str
DataType: str
StudyOID: str
ModelID: str
codelistitem:
represented_as: node
preferred_id: codedvalue
label_in_input: CodeListItem
is_a: biolink:ConceptualEntity
properties:
CodedValue: str
Decode: str
# Edge types extracted from ODM generators mapped to BioLink predicates
study_has_studyevent:
represented_as: edge
label_in_input: STUDY_HAS_STUDYEVENT
source: study
target: studyevent
is_a: biolink:related_to
itemgroup_has_item:
represented_as: edge
label_in_input: ITEMGROUP_HAS_ITEM
source: itemgroup
target: item
is_a: biolink:has_part
form_has_itemgroup:
represented_as: edge
label_in_input: FORM_HAS_ITEMGROUP
source: form
target: itemgroup
is_a: biolink:has_part
studyevent_has_form:
represented_as: edge
label_in_input: STUDYEVENT_HAS_FORM
source: studyevent
target: form
is_a: biolink:has_part
item_has_alias:
represented_as: edge
label_in_input: ITEM_HAS_ALIAS
source: item
target: alias
is_a: biolink:same_as
itemgroup_has_alias:
represented_as: edge
label_in_input: ITEMGROUP_HAS_ALIAS
source: itemgroup
target: alias
is_a: biolink:same_as
item_has_measurementunit:
represented_as: edge
label_in_input: ITEM_HAS_MEASUREMENTUNIT
source: item
target: measurementunit
is_a: biolink:has_attribute
has_basedef:
represented_as: edge
label_in_input: HAS_BASEDEF
source: study
target: basicdefinitions
is_a: biolink:has_part
basedef_has_measurementunit:
represented_as: edge
label_in_input: BASEDEF_HAS_MEASUREMENTUNIT
source: basicdefinitions
target: measurementunit
is_a: biolink:has_part
item_has_rangecheck:
represented_as: edge
label_in_input: ITEM_HAS_RANGECHECK
source: item
target: rangecheck
is_a: biolink:has_attribute
item_has_codelist:
represented_as: edge
label_in_input: ITEM_HAS_CODELIST
source: item
target: codelist
is_a: biolink:has_attribute
codelist_has_codelistitem:
represented_as: edge
label_in_input: CODELIST_HAS_CODELISTITEM
source: codelist
target: codelistitem
is_a: biolink:has_part
composite:
represented_as: edge
label_in_input: COMPOSITE
source: alias
target: alias
is_a: biolink:related_to
config/biocypher_config.yaml
+1 -1
View File
@@ -5,7 +5,7 @@ biocypher:
offline: true
#debug: true
output_directory: /neo4j_import #comment if you want to debug, so that bc creates a new folder for each run in /biocypher-out
schema_config_path: config/automated_schema.yaml #config/automated_schema.yaml
schema_config_path: config/automated_schema.yaml
head_ontology:
url: config/head_ontology/biolink-model.owl.ttl
config/head_ontology/biolink-model.owl.ttl
+7508 -17715
View File
File diff suppressed because it is too large. Load diff
config/manual_schema_config.yaml
+2 -2
View File
@@ -141,8 +141,8 @@ Patient:
id: str
birthDate: str
procedure:
is_a: named thing
Procedure:
# is_a: Procedure
represented_as: node
preferred_id: fhir_id
label_in_input: Procedure
docker-compose.yml
Executable → Regular
+13 -235
View File
@@ -1,18 +1,6 @@
services:
neo4j-certs-perm-fix:
image: busybox
command: ["chmod", "-R", "755", "/certs"]
volumes:
- ./neo4j-certs:/certs
networks:
- medax-network
neo4j:
image: neo4j:5.7
restart: unless-stopped
depends_on:
neo4j-certs-perm-fix:
condition: service_completed_successfully
environment:
- NEO4J_AUTH=${NEO4J_AUTH:-neo4j/password}
- NEO4J_PLUGINS=["apoc"]
@@ -21,46 +9,10 @@ services:
- NEO4J_apoc_import_file_enabled=true
- NEO4J_apoc_import_file_use__neo4j__config=true
- SHARED_PATH=/neo4j_import
- NO_PROXY=${NO_PROXY},10.0.0.0/8
- DISABLE_IMPORT=${DISABLE_IMPORT:-false}
- ENABLE_BOLT_TLS=${ENABLE_BOLT_TLS:-false}
command: >
bash -c '
if [ "$${DISABLE_IMPORT:-false}" = "false" ]; then
echo "running cmd from docker compose" &&
# Copy plugin files if they exist
if [ -f /init_files/semspect_neo4j-plugin-8.2.0.jar ]; then
echo "Copying Semspect plugin..." &&
cp /init_files/semspect_neo4j-plugin-8.2.0.jar /var/lib/neo4j/plugins/
fi
if [ -f /init_files/semspect.lic ]; then
echo "Copying Semspect license..." &&
cp /init_files/semspect.lic /var/lib/neo4j/plugins/
fi
# Copy neo4j.conf if it exists
ls
ls /init_files/
if [ -f /init_files/neo4j.conf ]; then
echo "Copying Neo4j configuration..." &&
cp /init_files/neo4j.conf /var/lib/neo4j/conf/
fi
if [ "$${ENABLE_BOLT_TLS:-false}" != "true" ]; then
echo "ENABLE_BOLT_TLS is not true - disabling required bolt TLS (no certs mounted in ./neo4j-certs locally)." &&
sed -i \
-e "s/^server.bolt.tls_level=REQUIRED/#server.bolt.tls_level=REQUIRED/" \
-e "s/^dbms.ssl.policy.bolt.enabled=true/#dbms.ssl.policy.bolt.enabled=true/" \
/var/lib/neo4j/conf/neo4j.conf
fi
echo "<----------------"
# Set proper permissions
#chown -R neo4j:neo4j /var/lib/neo4j/plugins
#chown -R neo4j:neo4j /var/lib/neo4j/conf
echo "Import loop is enabled."
# Drop a stale trigger from an aborted run so old data is not imported
# before python_app has cleared and regenerated /neo4j_import
rm -f /neo4j_import/ready-to-import
#neo4j start &&
while true; do
if [ -f /neo4j_import/ready-to-import ]; then
echo "Starting import process..."
@@ -70,35 +22,10 @@ services:
touch /neo4j_import/import-complete &&
chmod 777 /neo4j_import/import-complete
neo4j start
echo "The container is running. CTRL+C will end the bash command and thus, the neo4j container"
echo "The container is running. STR+C will end the bash command and thus, the neo4j container"
fi
sleep 10
done
else
cp /init_files/neo4j.conf /var/lib/neo4j/conf/
if [ "$${ENABLE_BOLT_TLS:-false}" != "true" ]; then
echo "ENABLE_BOLT_TLS is not true - disabling required bolt TLS (no certs mounted in ./neo4j-certs locally)." &&
sed -i \
-e "s/^server.bolt.tls_level=REQUIRED/#server.bolt.tls_level=REQUIRED/" \
-e "s/^dbms.ssl.policy.bolt.enabled=true/#dbms.ssl.policy.bolt.enabled=true/" \
/var/lib/neo4j/conf/neo4j.conf
fi
if [ "$${DISABLE_IMPORT:-false}" = "load" ]; then
if [ ! -f /neo4j_import/neo4j-admin-import-call.sh ]; then
echo "DISABLE_IMPORT is load but /neo4j_import/neo4j-admin-import-call.sh is missing - skipping import." >&2
else
echo "DISABLE_IMPORT is load - Loading admin import data from neo4j_import." &&
neo4j stop &&
bash /neo4j_import/neo4j-admin-import-call.sh &&
touch /neo4j_import/import-complete &&
chmod 777 /neo4j_import/import-complete
fi
fi
echo "Import loop is disabled. Keeping Neo4j container alive..."
neo4j start
echo "The container is running. CTRL+C will end the bash command and thus, the neo4j container"
tail -f /dev/null
fi
'
healthcheck:
test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"]
@@ -106,28 +33,18 @@ services:
timeout: 5s
retries: 5
ports:
- "8080:7474" # direct access — remove in production
- "8081:7687" # remove in production
#- "127.0.0.1:8080:7474" #production
#- "10.66.83.60:8081:7687"
- "8080:7474"
- "8081:7687"
volumes:
- neo4j_data:/data
- neo4j_logs:/logs
- ./neo4j_import:/neo4j_import
- neo4j_data:/neo4j_data
- neo4j_logs:/neo4j_logs
- neo4j_import:/neo4j_import
- ${INPUT_DATA_PATH:-./data}:/input_data
- ./init_scripts:/init_scripts
- ./init_files:/init_files
- ./init-scripts:/init-scripts
- ./importData:/importData
- ./neo4j-certs:/var/lib/neo4j/certificates
networks:
- medax-network
python_app:
depends_on:
loader:
condition: service_completed_successfully
required: false
build:
context: .
dockerfile: Dockerfile
@@ -141,158 +58,19 @@ services:
- NEO4J_URI=bolt://neo4j:7687
- NEO4J_USER=${NEO4J_USER:-neo4j}
- NEO4J_PASSWORD=${NEO4J_PASSWORD:-password}
- DISABLE_IMPORT=${DISABLE_IMPORT:-false}
- INPUT_DATA_PATH=/input_data
- POETRY_VIRTUALENVS_CREATE=false
- NEO4J_dbms_directories_import=/neo4j_import
volumes:
- ./neo4j_import:/neo4j_import
- neo4j_import:/neo4j_import
- ${INPUT_DATA_PATH:-./data}:/input_data
- ./importData:/importData # Share the import data directory
# depends_on:
# neo4j:
# condition: service_healthy
networks:
- medax-network
# depends_on:
# neo4j:
# condition: service_healthy
oauth2-proxy:
profiles:
- server
image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0
container_name: oauth2-proxy
restart: unless-stopped
extra_hosts:
- "keycloak-test.miracum.med.uni-greifswald.de:10.66.82.11"
- "host.docker.internal:host-gateway"
command:
- --config=/etc/oauth2-proxy.cfg
ports:
# - "80:80"
- "443:443"
volumes:
- ./oauth_conf/oauth2-proxy.cfg:/etc/oauth2-proxy.cfg:ro
- ./oauth_templates:/templates:ro
- ./oauth-certs:/etc/oauth-certs:ro
- ./logs:/var/log/oauth2-proxy
depends_on:
- neo4j
networks:
- medax-network
nginx:
profiles:
- server
image: nginx:latest
container_name: neo4j-nginx
## ports:
## - "8088:80"
# - "7687:7687"
volumes:
- ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro
- ./logs:/var/log/nginx
depends_on:
- neo4j
networks:
- medax-network
neo4j-proxy:
profiles:
- proxy
image: quay.io/oauth2-proxy/oauth2-proxy:latest
restart: on-failure
environment:
OAUTH2_PROXY_PROVIDER: keycloak-oidc
OAUTH2_PROXY_CLIENT_ID: ${NEO4J_OAUTH_CLIENT_ID:-neo4j-client}
OAUTH2_PROXY_CLIENT_SECRET: ${NEO4J_OAUTH_SECRET}
OAUTH2_PROXY_OIDC_ISSUER_URL: ${KEYCLOAK_ISSUER_URL}
OAUTH2_PROXY_UPSTREAMS: http://neo4j:7474/
OAUTH2_PROXY_HTTP_ADDRESS: 0.0.0.0:4180
OAUTH2_PROXY_REDIRECT_URL: ${NEO4J_PROXY_REDIRECT_URL:-http://localhost:8082/oauth2/callback}
OAUTH2_PROXY_COOKIE_SECRET: ${NEO4J_COOKIE_SECRET}
OAUTH2_PROXY_EMAIL_DOMAINS: "*"
OAUTH2_PROXY_SKIP_PROVIDER_BUTTON: "true"
OAUTH2_PROXY_INSECURE_OIDC_SKIP_ISSUER_VERIFICATION: "true"
OAUTH2_PROXY_COOKIE_SECURE: "false"
extra_hosts:
- "localhost:host-gateway"
ports:
- "8082:4180"
depends_on:
- neo4j
#- keycloak # enable if a local keycloak is set up
keycloak:
profiles:
- keycloak
image: quay.io/keycloak/keycloak:latest
command: start-dev
restart: on-failure
environment:
KC_BOOTSTRAP_ADMIN_USERNAME: admin
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
KC_HOSTNAME: http://localhost:4040
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
ports:
- "4040:8080"
volumes:
- keycloak_data:/opt/keycloak/data
blaze:
profiles:
- blaze
image: samply/blaze:latest
ports:
- "8090:8080"
environment:
JAVA_TOOL_OPTIONS: "-Xmx2g"
ENFORCE_REFERENTIAL_INTEGRITY: "false"
volumes:
- blaze-data:/app/data
healthcheck:
test: curl -f http://localhost:8080/fhir/metadata || exit 1
interval: 5s
timeout: 10s
retries: 30
networks:
- medax-network
loader:
profiles:
- blaze
image: curlimages/curl:latest
depends_on:
blaze:
condition: service_healthy
volumes:
- ./testData/POLAR_mock:/bundles:ro #adapt first part to whereever the testfiles are stored
entrypoint: /bin/sh
command: >
-c '
for f in /bundles/*.json; do
echo "Loading $$f ...";
curl -s -X POST http://blaze:8080/fhir \
-H "Content-Type: application/fhir+json" \
-d @"$$f" > /dev/null;
echo "";
done;
echo "Done loading all bundles."
'
networks:
- medax-network
# Define named volumes
volumes:
neo4j_data:
neo4j_logs:
neo4j_import:
keycloak_data:
blaze-data:
networks:
medax-network:
driver: bridge
entrypoint.sh
+21 -46
View File
@@ -1,63 +1,38 @@
#!/bin/bash
set -euo pipefail
set -e
case "${DISABLE_IMPORT:-false}" in
false) ;;
true|load)
echo "DISABLE_IMPORT is ${DISABLE_IMPORT} - skipping ETL pipeline (neo4j handles the rest)"
exit 0
;;
*)
echo "Invalid DISABLE_IMPORT='${DISABLE_IMPORT}' - expected false, true or load" >&2
exit 1
;;
esac
# Wait for a file to appear, with a timeout so the container can't hang forever.
# Usage: wait_for_file <path> [timeout_seconds]
wait_for_file() {
local file="$1"
local timeout="${2:-300}"
local waited=0
while [ ! -f "$file" ]; do
if [ "$waited" -ge "$timeout" ]; then
echo "Timed out after ${timeout}s waiting for $(basename "$file")" >&2
exit 1
fi
echo "Waiting for $(basename "$file")..."
sleep 5
waited=$((waited + 5))
done
}
chmod -R 777 /neo4j_import #make the dir accessible for both the python app an neo4j
# Start from an empty import dir: BioCypher appends new -partNNN files next to
# old ones and the admin import globs all of them, so leftovers would be
# imported again. Stale marker files would also break the handshake below.
echo "Clearing /neo4j_import..."
find /neo4j_import -mindepth 1 -delete
# Make the import dir accessible to both the Python app and Neo4j
chmod -R 777 /neo4j_import
#echo "Waiting for Neo4j to be ready..."
#echo "Waiting for Neo4j to be ready... ..."
#python wait-for-neo4j.py
#if [ $? -ne 0 ]; then
# echo "Failed to connect to Neo4j"
# exit 1
#fi
echo "Running Python data processing script..."
#python import_fhir_to_nx_diGraph.py
python multiple_adapters.py
poetry run python import_fhir_to_nx_diGraph.py
echo "Running Neo4j import..."
# Give downstream services a moment before touching the database
# Wait a bit before attempting database operations
sleep 5
# Wait for the shell script to finish writing its data
wait_for_file /neo4j_import/shell-scipt-complete
while [ ! -f /neo4j_import/shell-scipt-complete ]; do
echo "Waiting for shell-script file"
sleep 5
done
# Signal that data is prepared and ready to import
# Create a signal file that we've prepared the data
touch /neo4j_import/ready-to-import
chmod 777 /neo4j_import/ready-to-import
chmod -R 777 /neo4j_import/ready-to-import
# Wait for the import to complete
# Wait for import to complete by monitoring a completion file
echo "Waiting for Neo4j import to complete..."
wait_for_file /neo4j_import/import-complete
while [ ! -f /neo4j_import/import-complete ]; do
echo "Waiting for import-complete file"
sleep 5
done
echo "Database setup complete!"
fhirImport.py
+4 -7
View File
@@ -7,7 +7,7 @@ from requests.auth import HTTPBasicAuth
# Load environment variables from .env file
load_dotenv()
def get_bundle(url, search):
def getBundle(url: str, search: str):
headers = {
'Accept': 'application/fhir+json',
'Content-Type': 'application/fhir+json'
@@ -45,21 +45,18 @@ def get_bundle(url, search):
response = requests.get(
link,
headers=headers,
auth=auth,
timeout=300
auth=auth
)
else:
response = requests.get(
link,
headers=headers,
timeout=300
)
return response
def get_patient_everything(id: str):
def getPatientEverything(id: str):
search = '/Patient/' + id + '/$everything?'
return get_bundle(None, search)
return getBundle(None, search)
# Example usage
graphCreation/create_graph.py
+6 -5
View File
@@ -12,17 +12,13 @@ def add_nodes_from_dict(graph, parent_node, current_dict):
# Recurse into the nested dictionary
add_nodes_from_dict(graph, new_node, value)
elif isinstance(value, list):
# if list doesn't contain any nested dictionaries, make it a value in the node
if any(isinstance(item, dict) for item in value)==False:
graph.nodes[parent_node][key] = value
else:
process_dictionaries(value, parent_node, key, graph)
else:
# For non-dict and non-list values, add them as attributes to the parent node
graph.nodes[parent_node][key] = value
def process_dictionaries(value, parent_node, key, graph):
# Process each dictionary in the list
for index, item in enumerate(value):
if isinstance(item, dict):
@@ -34,6 +30,11 @@ def process_dictionaries(value, parent_node, key, graph):
graph.add_edge(parent_node, item_node, edge_type=key)
add_nodes_from_dict(graph, item_node, item)
else:
# For non-dict and non-list values, add them as attributes to the parent node
graph.nodes[parent_node][key] = value
def add_json_to_networkx(json_data, bundle_name, graph):
if not isinstance(graph, nx.DiGraph):
raise ValueError("The provided graph must be a networkx.DiGraph")
graphCreation/node_typing.py
+5 -3
View File
@@ -21,8 +21,8 @@ def set_resource_type(graph):
resource_type = node.resource_type
if resource_type:
# Dynamically create a new class based on the resource_type
new_resource_class = create_resource_class(resource_type)
new_node = new_resource_class(resource_type)
NewResourceClass = create_resource_class(resource_type)
new_node = NewResourceClass(resource_type)
nodes_to_replace.append((node, new_node, data))
else:
print(f"Warning: Node {node} is a resource but has no resource_type")
@@ -35,4 +35,6 @@ def set_resource_type(graph):
for succ in graph.successors(old_node):
graph.add_edge(new_node, succ)
graph.remove_node(old_node)
"""
for node, data in graph.nodes(data=True):
print(node, data) """
graphCreation/process_references.py
+15 -9
View File
@@ -1,6 +1,4 @@
import networkx as nx
import os
def parse_synthea_reference(ref):
if not ref.startswith('#'):
@@ -17,17 +15,17 @@ def parse_synthea_reference(ref):
def process_references(graph):
is_synthea = os.getenv('IS_SYNTHEA')
print("--------------->", is_synthea)
isSynthea = False
nodes_with_reference = [[n, attr['reference']] for n, attr in graph.nodes(data=True) if 'reference' in attr]
directly_referenced_nodes = []
indirectly_referenced_nodes = []
dummy_references = []
if is_synthea and is_synthea.upper() == 'TRUE':
if isSynthea:
nodes_with_mock_reference = []
@@ -62,16 +60,19 @@ def process_references(graph):
graph.add_edge(node_from, node_to, edge_type='reference', reference_type=ref_type)
else:
#for node, data in graph.nodes(data=True):
# if "id" in data:
# if not "resourceType" in data:
# print("FAILS AT: ", data, node)
id_to_node = {data["resourceType"]+'/'+data["id"]: node for node, data in graph.nodes(data=True) if ("id" in data and "resourceType" in data)}
for i in nodes_with_reference:
ref_id=i[1]
if ref_id in id_to_node.keys():
#print("everything is here?", ref_id)
directly_referenced_nodes.append([i[0], id_to_node[ref_id]])
else:
#print("ALL WE KNOW ABOUT DUMMY NODES: ", ref_id)
dummy_references.append([i[0], ref_id])
for i in directly_referenced_nodes:
@@ -81,12 +82,15 @@ def process_references(graph):
graph.add_edge(node_from, node_to, edge_type='reference', reference_type=ref_type)
for i in dummy_references:
ref_type = i[1][:i[1].find('/')]
#print(i)
node_to='dummy_' + i[1]
graph.add_node(node_to, label='dummy', unique_id=i[1], edge_label=ref_type)
graph.add_node(node_to, label='dummy', unique_id=i[1])
node_from=list(graph.predecessors(i[0]))[0]
ref_type=graph.nodes[i[0]]['label']
graph.add_edge(node_from, node_to, edge_type='reference', reference_type=ref_type)
#graph.remove_nodes_from([i[0] for i in nodes_with_reference])
graph.remove_nodes_from([i[0] for i in directly_referenced_nodes])
graph.remove_nodes_from([i[0] for i in indirectly_referenced_nodes])
graph.remove_nodes_from([i[0] for i in dummy_references])
@@ -94,3 +98,5 @@ def process_references(graph):
nodes_to_remove = [n for n, attr in graph.nodes(data=True) if attr.get('label') in ['root', 'entry', 'request']]
graph.remove_nodes_from(nodes_to_remove)
#graph.remove_nodes_from(list(nx.isolates(graph)))
import_fhir_to_nx_diGraph.py
+74 -98
View File
@@ -17,72 +17,47 @@ from graphCreation.property_convolution import property_convolution
from schema_config_generation import write_automated_schema
from fhirImport import get_patient_everything, get_bundle
is_test = os.getenv('TEST_MODE')
test_depth = os.getenv('TEST_DEPTH')
from fhirImport import getPatientEverything, getBundle
def load_multiple_fhir_patients(n):
init_load = True
#graph = nx.DiGraph()
init = True
initLoad = True
ids = []
#get n ids
init_ids = True
while len(ids) < n and init_ids:
if init_load:
is_complex = os.getenv('COMPLEX_PATIENTS')
selected_patients_file = os.getenv('SELECTED_PATIENTS_FILE')
#print("> /dev/null")
if selected_patients_file and os.path.isfile(selected_patients_file):
print(f"-- Loading patient IDs from file: {selected_patients_file} --")
with open(selected_patients_file, 'r') as f:
for line in f:
patient_id = line.strip()
if patient_id:
ids.append(patient_id)
print(f"Loaded {len(ids)} patient IDs from file.")
n = len(ids)
break # Exit the while loop since we got IDs from the file
elif is_complex and is_complex.upper() != 'TRUE':
bundle = get_bundle(None, '/Patient?_count=' + str(n))
nextIds = True
while len(ids) < n and nextIds:
if initLoad:
complex = os.getenv('COMPLEX_PATIENTS')
if complex and complex.upper() != 'TRUE':
bundle = getBundle(None, '/Patient?_count=' + str(n))
else:
#print("-- Looking for complex patients --")
print("-- Looking for gout patients --")
bundle = get_bundle(None, '/Patient?_has:Observation:subject:status=final&_count=' + str(n))
#bundle = get_bundle(None, '/Patient?_has:Condition:patient:code=M10.00,M10.01,M10.02,M10.03,M10.04,M10.05,M10.06,M10.07,M10.08,M10.09,M10.10,M10.11,M10.12,M10.13,M10.14,M10.15,M10.16,M10.17,M10.18,M10.19,M10.20,M10.21,M10.22,M10.23,M10.24,M10.25,M10.26,M10.27,M10.28,M10.29,M10.30,M10.31,M10.32,M10.33,M10.34,M10.35,M10.36,M10.37,M10.38,M10.39,M10.40,M10.41,M10.42,M10.43,M10.44,M10.45,M10.46,M10.47,M10.48,M10.49,M10.90,M10.91,M10.92,M10.93,M10.94,M10.95,M10.96,M10.97,M10.98,M10.99&_count=' + str(n))
print("Got patient ids")
init_load = False
bundle = getBundle(None, '/Patient?_has:Observation:subject:status=final&_count=' + str(n))
initLoad = False
else:
print("NEXT LINK: ", next_link, flush=True)
bundle = get_bundle(next_link, None)
print("NEXT LINK: ", nextLink, flush=True)
bundle = getBundle(nextLink, None)
#print(bundle.json())
if 'entry' not in bundle.json():
if not 'entry' in bundle.json():
print("ERROR -- No data found in the fhir bundle. Check the request and if the server is up and responding")
sys.exit(1)
for entry in bundle.json()['entry']:
ids.append(entry['resource']['id'])
init_ids = False
nextIds = False
for l in bundle.json()['link']:
if l['relation'] == "next":
next_link = l['url']
init_ids = True
nextLink = l['url']
nextIds = True
if len(ids) < n:
n = len(ids)
print("####GRABBED ", n , " IDs from the DB####", flush=True)
bundles_to_graph(ids, n)
batchSize = int(os.getenv('BATCH_SIZE'))
def bundles_to_graph(ids, n):
init = True
batch_size = int(os.getenv('BATCH_SIZE'))
c = 0
print(len(ids))
@@ -91,7 +66,9 @@ def bundles_to_graph(ids, n):
for id in ids:
c += 1
bundle = get_patient_everything(id).json()
bundle = getPatientEverything(id).json()
bundle = replace_single_quotes(bundle) ### maybe not needed for german data
if init:
graph = nx.DiGraph()
@@ -102,15 +79,19 @@ def bundles_to_graph(ids, n):
if c % 50 == 0:
print("---------- ", c, " patients loaded ----------", flush=True)
if c % batch_size == 0 or c == n:
if c % batchSize == 0 or c == n:
print(c, " patients imported, reducing graph", flush = True)
process_references(graph)
property_convolution(graph)
run_biocypher(graph)
lastChunk = False
if n == c:
lastChunk = True
runBioCypher(graph, lastChunk)
init = True
#print("Full batch:", c)
print(graph)
del graph
gc.collect
def replace_single_quotes(obj):
if isinstance(obj, str): # If it's a string, replace single quotes
@@ -126,23 +107,15 @@ def main():
## create networkX and run improvement scripts
print("Creating the graph...", flush=True)
n_patients = int(os.getenv('NUMBER_OF_PATIENTS'))
load_multiple_fhir_patients(n_patients)
#write the import script -- we are creating our own script since BC would only consider the last batch as an input
print("CREATING THE SCRIPT")
generate_neo4j_import_script()
with open('/neo4j_import/shell-scipt-complete', 'w') as f:
f.write('Import completed successfully')
print("FHIR import completed successfully")
nPatients = int(os.getenv('NUMBER_OF_PATIENTS'))
load_multiple_fhir_patients(nPatients)
def run_biocypher(nx_graph):
def runBioCypher(nxGraph, final):
#get lists of node and edge types
print("Generate auto schema...", flush=True)
write_automated_schema(nx_graph, 'config/automated_schema.yaml', 'config/manual_schema_config.yaml')
write_automated_schema(nxGraph, 'config/automated_schema.yaml', 'config/manual_schema_config.yaml')
# create Biocypher driver
@@ -151,65 +124,69 @@ def run_biocypher(nx_graph):
)
#bc.show_ontology_structure() #very extensive
#BioCypher preperation
def node_generator():
for node in nxGraph.nodes():
bc.write_nodes(node_generator(nx_graph))
bc.write_edges(edge_generator(nx_graph))
def node_generator(nx_graph):
for node in nx_graph.nodes():
label = nx_graph.nodes[node].get('label')
if(label == 'dummy' or label == 'Dummy'):
print("Skipped dummy: ", nx_graph.nodes[node])
continue
label = nxGraph.nodes[node].get('label')
if label == "resource":
label = nx_graph.nodes[node].get('resourceType')
label = nxGraph.nodes[node].get('resourceType')
nxGraph.nodes[node]['label'] = label.capitalize()
label = label.capitalize()
nx_graph.nodes[node]['label'] = label
if(nx_graph.nodes[node].get('label') in ['search', 'meta', 'link', 'Search', 'Meta', 'Link']):
unq_id = nxGraph.nodes[node].get('unique_id', False)
if(nxGraph.nodes[node].get('label') in ['search', 'meta', 'link']):
#print("skipped a node: ", nxGraph.nodes[node].get('label'))
continue
label = nxGraph.nodes[node].get('label')
if(label == 'dummy'):
#print("SKIPPED dummy node: ", unq_id)
continue
yield(
nx_graph.nodes[node].get('unique_id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
nxGraph.nodes[node].get('unique_id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
label,
nx_graph.nodes[node] # get properties
nxGraph.nodes[node] # get properties
)
def edge_generator(nx_graph):
for edge in nx_graph.edges(data = True):
def edge_generator():
for edge in nxGraph.edges(data = True):
source, target, attributes = edge
s_label = nx_graph.nodes[source].get('label')
if s_label == 'resource':
s_label = nx_graph.nodes[source].get('resourceType')
elif s_label == 'dummy' or s_label == 'Dummy':
s_label = nx_graph.nodes[source].get('edge_label')
t_label = nx_graph.nodes[target].get('label')
if t_label == 'resource':
t_label = nx_graph.nodes[target].get('resourceType')
elif t_label == 'dummy' or t_label == 'Dummy':
t_label = nx_graph.nodes[target].get('edge_label')
label = s_label.capitalize() + '_to_' + t_label.capitalize()
sLabel = nxGraph.nodes[source].get('label')
if sLabel == 'resource':
sLabel = nxGraph.nodes[source].get('resourceType')
tLabel = nxGraph.nodes[target].get('label')
if tLabel == 'resource':
tLabel = nxGraph.nodes[target].get('resourceType')
label = sLabel.capitalize() + '_to_' + tLabel
yield(
attributes.get('id', str(uuid.uuid4())), # Edge ID (if exists, otherwise use nx internal id)
nx_graph.nodes[source].get('unique_id', source),
nx_graph.nodes[target].get('unique_id', target),
nxGraph.nodes[source].get('unique_id', source),
nxGraph.nodes[target].get('unique_id', target),
label,
attributes # All edge attributes
)
#import nodes
bc.write_nodes(node_generator())
bc.write_edges(edge_generator())
#write the import script -- we are creating our own script since BC would only consider the last batch as an input
if final:
print("CREATING THE SCRIPT")
generate_neo4j_import_script()
with open('/neo4j_import/shell-scipt-complete', 'w') as f:
f.write('Import completed successfully')
print("FHIR import completed successfully")
def generate_neo4j_import_script(directory_path="/neo4j_import/", output_file="neo4j-admin-import-call.sh"):
"""
@@ -228,14 +205,13 @@ def generate_neo4j_import_script(directory_path="/neo4j_import/", output_file="n
# Dictionary to store entity types (nodes and relationships)
entity_types = {}
# Find all header files and use them to identify entity types
for filename in all_files:
if '-header.csv' in filename:
entity_name = filename.split('-header.csv')[0]
# Check if it's a relationship (contains "To" and "Association")
is_relationship = ("To" in entity_name and "Association" in entity_name) or "_has_" in entity_name or "Has_" in entity_name
is_relationship = "To" in entity_name and "Association" in entity_name
# Store in entity_types dictionary
if is_relationship:
import_nx_diGraph.py
+82 -35
View File
@@ -3,11 +3,15 @@ import networkx as nx
import json
import os
import uuid
#from networkx_based import create_graph
from graphCreation import create_graph
#from networkx_based.process_references import process_references
from graphCreation.process_references import process_references
#from networkx_based.property_convolution import property_convolution
from graphCreation.property_convolution import property_convolution
from schema_config_generation import write_automated_schema
#from networkx_based.node_typing import set_ressource_type
from graphCreation.node_typing import set_resource_type
@@ -15,7 +19,7 @@ from graphCreation.node_typing import set_resource_type
def load_multiple_fhir_bundles(directory_path):
graph = nx.DiGraph()
init = True
#limit = 2
# Iterate over all files in the directory
for filename in os.listdir(directory_path):
if filename.endswith('.json'): # Assuming FHIR bundles are in JSON format
@@ -24,15 +28,20 @@ def load_multiple_fhir_bundles(directory_path):
bundle_json = json.load(f)
#fix all strings to to enable ' in neo4j
fixed_quotes = replace_single_quotes(bundle_json)
fixedQuotes = replace_single_quotes(bundle_json)
if init:
#print(bundle_json, filename, graph)
create_graph.json_to_networkx(fixed_quotes, filename, graph)
create_graph.json_to_networkx(fixedQuotes, filename, graph)
init = False
else:
create_graph.add_json_to_networkx(fixed_quotes, filename, graph)
create_graph.add_json_to_networkx(fixedQuotes, filename, graph)
print("Imported: ", filename)
#if limit == 0:
# return graph
#limit = limit - 1
return graph
def replace_single_quotes(obj):
@@ -49,24 +58,61 @@ def main():
#get a list of nodes that should be imported
## create networkX and run improvement scripts
print("Creating the graph...", flush=True)
nx_graph = load_multiple_fhir_bundles('./testData/') # 'mockData' for unit test data, 'testData' for Synthea files
print(nx_graph)
nxGraph = load_multiple_fhir_bundles('./testData/') # 'mockData' for unit test data, 'testData' for Synthea files
print(nxGraph)
print("Reducing references...", flush=True)
process_references(nx_graph)
print(nx_graph)
process_references(nxGraph)
print(nxGraph)
print("Convolute references...", flush=True)
property_convolution(nx_graph)
print(nx_graph)
property_convolution(nxGraph)
print(nxGraph)
#Set types of all resource nodes to resource_type
#set_resource_type(nxGraph)
#get lists of node and edge types
""" all_nLabels = set()
all_eLabels = set()
for node, attrs in nxGraph.nodes(data=True):
for attr_name, attr_value in attrs.items():
if attr_name == "label":
all_nLabels.add(attr_value)
for nt in all_nLabels:
print(nt)
print("-" * 50)
for u, v, attrs in nxGraph.edges(data=True):
u_label = nxGraph.nodes[u]['label']
if u_label == "resource":
u_label = nxGraph.nodes[u]['resourceType']
v_label = nxGraph.nodes[v]['label']
if v_label == "resource":
v_label = nxGraph.nodes[v]['resourceType']
all_eLabels.add(u_label + " to " + v_label)
for et in all_eLabels:
print(et)
print("-" * 50)
print("...end")
return """
print("Generate auto schema...")
write_automated_schema(nx_graph, 'config/automated_schema.yaml', None)
write_automated_schema(nxGraph, 'config/automated_schema.yaml')
# create Biocypher driver
bc = BioCypher(
biocypher_config_path="config/biocypher_config.yaml",
#schema_config_path="/config/manual_schema_config.yaml"
)
bc.show_ontology_structure()
@@ -74,40 +120,40 @@ def main():
#BioCypher preperation
## node generator: extract id, label and property dictionary
def node_generator():
for node in nx_graph.nodes():
for node in nxGraph.nodes():
""" #single qoutes break neo4j import, e.g. 'CHILDREN'S Hospital'
checkDisplay = nx_graph.nodes[node].get('display')
checkDisplay = nxGraph.nodes[node].get('display')
if checkDisplay:
checkDisplay = checkDisplay.replace("'", "''")
nx_graph.nodes[node]['display'] = checkDisplay
#print("------->", nx_graph.nodes[node].get('display'))
nxGraph.nodes[node]['display'] = checkDisplay
#print("------->", nxGraph.nodes[node].get('display'))
checkName = nx_graph.nodes[node].get('name')
checkName = nxGraph.nodes[node].get('name')
if checkName:
checkName = checkName.replace("'", "''")
nx_graph.nodes[node]['name'] = checkName
#print("------->", nx_graph.nodes[node].get('name')) """
nxGraph.nodes[node]['name'] = checkName
#print("------->", nxGraph.nodes[node].get('name')) """
label = nx_graph.nodes[node].get('label')
label = nxGraph.nodes[node].get('label')
if label == "resource":
label = nx_graph.nodes[node].get('resourceType')
label = nxGraph.nodes[node].get('resourceType')
'''
elif label == 'identifier':
label = nx_graph.nodes[node].get('system')
label = nxGraph.nodes[node].get('system')
print('/' in label)
if '/' in label:
lastSlash = label.rfind('/') + 1
label = label[lastSlash:] + '-ID'
elif label == 'telecom':
label = nx_graph.nodes[node].get('system')
label = nxGraph.nodes[node].get('system')
print('/' in label)
if '/' in label:
lastSlash = label.rfind('/') + 1
label = 'telecom-' + label[lastSlash:]
elif label == 'address':
extension = nx_graph.nodes[node].get('extension_url')
extension = nxGraph.nodes[node].get('extension_url')
print("EX!: ", extension)
if extension:
lastSlash = extension.rfind('/') + 1
@@ -115,31 +161,32 @@ def main():
'''
yield(
nx_graph.nodes[node].get('id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
nxGraph.nodes[node].get('id', node), #remark: this returns the node id if this attribute exists. otherwise it returns node which equals the identifier that is used by nx
label,
nx_graph.nodes[node] # get properties
nxGraph.nodes[node] # get properties
)
def edge_generator():
for edge in nx_graph.edges(data = True):
for edge in nxGraph.edges(data = True):
source, target, attributes = edge
s_label = nx_graph.nodes[source].get('label')
if s_label == 'resource':
s_label = nx_graph.nodes[source].get('resourceType')
t_label = nx_graph.nodes[target].get('label')
if t_label == 'resource':
t_label = nx_graph.nodes[target].get('resourceType')
label = s_label + '_to_' + t_label
sLabel = nxGraph.nodes[source].get('label')
if sLabel == 'resource':
sLabel = nxGraph.nodes[source].get('resourceType')
tLabel = nxGraph.nodes[target].get('label')
if tLabel == 'resource':
tLabel = nxGraph.nodes[target].get('resourceType')
label = sLabel + '_to_' + tLabel
yield(
attributes.get('id', str(uuid.uuid4())), # Edge ID (if exists, otherwise use nx internal id)
nx_graph.nodes[source].get('id', source),
nx_graph.nodes[target].get('id', target),
nxGraph.nodes[source].get('id', source),
nxGraph.nodes[target].get('id', target),
label,
attributes # All edge attributes
)
#import nodes
bc.write_nodes(node_generator())
bc.write_edges(edge_generator())
init_files/neo4j.conf
-377
View File
@@ -1,377 +0,0 @@
#*****************************************************************
# Neo4j configuration
#
# For more details and a complete list of settings, please see
# https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/
#*****************************************************************
# The name of the default database
#initial.dbms.default_database=neo4j
# Paths of directories in the installation.
#server.directories.data=data
#server.directories.plugins=plugins
#server.directories.logs=logs
#server.directories.lib=lib
#server.directories.run=run
#server.directories.licenses=licenses
#server.directories.transaction.logs.root=data/transactions
# This setting constrains all `LOAD CSV` import files to be under the `import` directory. Remove or comment it out to
# allow files to be loaded from anywhere in the filesystem; this introduces possible security problems. See the
# `LOAD CSV` section of the manual for details.
server.directories.import=import
# Whether requests to Neo4j are authenticated.
# To disable authentication, uncomment this line
dbms.security.auth_enabled=false
#********************************************************************
# Memory Settings
#********************************************************************
#
# Memory settings are specified kilobytes with the 'k' suffix, megabytes with
# 'm' and gigabytes with 'g'.
# If Neo4j is running on a dedicated server, then it is generally recommended
# to leave about 2-4 gigabytes for the operating system, give the JVM enough
# heap to hold all your transaction state and query context, and then leave the
# rest for the page cache.
# Java Heap Size: by default the Java heap size is dynamically calculated based
# on available system resources. Uncomment these lines to set specific initial
# and maximum heap size.
#server.memory.heap.initial_size=512m
#server.memory.heap.max_size=512m
# The amount of memory to use for mapping the store files.
# The default page cache memory assumes the machine is dedicated to running
# Neo4j, and is heuristically set to 50% of RAM minus the Java heap size.
#server.memory.pagecache.size=10g
# Limit the amount of memory that all of the running transaction can consume.
# The default value is 70% of the heap size limit.
#dbms.memory.transaction.total.max=256m
# Limit the amount of memory that a single transaction can consume.
# By default there is no limit.
#db.memory.transaction.max=16m
# Transaction state location. It is recommended to use ON_HEAP.
# db.tx_state.memory_allocation=ON_HEAP
#*****************************************************************
# Network connector configuration
#*****************************************************************
# With default configuration Neo4j only accepts local connections.
# To accept non-local connections, uncomment this line:
#server.default_listen_address=0.0.0.0
# You can also choose a specific network interface, and configure a non-default
# port for each connector, by setting their individual listen_address.
# The address at which this server can be reached by its clients. This may be the server's IP address or DNS name, or
# it may be the address of a reverse proxy which sits in front of the server. This setting may be overridden for
# individual connectors below.
#server.default_advertised_address=localhost
# You can also choose a specific advertised hostname or IP address, and
# configure an advertised port for each connector, by setting their
# individual advertised_address.
# By default, encryption is turned off.
# To turn on encryption, an ssl policy for the connector needs to be configured
# Read more in SSL policy section in this file for how to define a SSL policy.
# Bolt connector
server.bolt.enabled=true
#server.bolt.tls_level=DISABLED
#server.bolt.listen_address=:7687
server.bolt.advertised_address=localhost:8081
#maybe has to be changed in production
# HTTP Connector. There can be zero or one HTTP connectors.
server.http.enabled=true
#server.http.listen_address=:7474
#server.http.advertised_address=:7474
# HTTPS Connector. There can be zero or one HTTPS connectors.
server.https.enabled=false
#server.https.listen_address=:7473
#server.https.advertised_address=:7473
# Number of Neo4j worker threads.
#server.threads.worker_count=
#*****************************************************************
# SSL policy configuration
#*****************************************************************
# Each policy is configured under a separate namespace, e.g.
# dbms.ssl.policy.<scope>.*
# <scope> can be any of 'bolt', 'https', 'cluster' or 'backup'
#
# The scope is the name of the component where the policy will be used
# Each component where the use of an ssl policy is desired needs to declare at least one setting of the policy.
# Allowable values are 'bolt', 'https', 'cluster' or 'backup'.
# E.g if bolt and https connectors should use the same policy, the following could be declared
# dbms.ssl.policy.bolt.base_directory=certificates/default
# dbms.ssl.policy.https.base_directory=certificates/default
# However, it's strongly encouraged to not use the same key pair for multiple scopes.
#
# N.B: Note that a connector must be configured to support/require
# SSL/TLS for the policy to actually be utilized.
#
# see: dbms.connector.*.tls_level
# SSL settings (dbms.ssl.policy.<scope>.*)
# .base_directory Base directory for SSL policies paths. All relative paths within the
# SSL configuration will be resolved from the base dir.
#
# .private_key A path to the key file relative to the '.base_directory'.
#
# .private_key_password The password for the private key.
#
# .public_certificate A path to the public certificate file relative to the '.base_directory'.
#
# .trusted_dir A path to a directory containing trusted certificates.
#
# .revoked_dir Path to the directory with Certificate Revocation Lists (CRLs).
#
# .verify_hostname If true, the server will verify the hostname that the client uses to connect with. In order
# for this to work, the server public certificate must have a valid CN and/or matching
# Subject Alternative Names.
#
# .client_auth How the client should be authorized. Possible values are: 'none', 'optional', 'require'.
#
# .tls_versions A comma-separated list of allowed TLS versions. By default only TLSv1.2 is allowed.
#
# .trust_all Setting this to 'true' will ignore the trust truststore, trusting all clients and servers.
# Use of this mode is discouraged. It would offer encryption but no security.
#
# .ciphers A comma-separated list of allowed ciphers. The default ciphers are the defaults of
# the JVM platform.
# Bolt SSL configuration
#dbms.ssl.policy.bolt.enabled=true
#dbms.ssl.policy.bolt.base_directory=certificates/bolt
#dbms.ssl.policy.bolt.private_key=private.key
#dbms.ssl.policy.bolt.public_certificate=public.crt
#dbms.ssl.policy.bolt.client_auth=NONE
# Https SSL configuration
#dbms.ssl.policy.https.enabled=true
#dbms.ssl.policy.https.base_directory=certificates/https
#dbms.ssl.policy.https.private_key=private.key
#dbms.ssl.policy.https.public_certificate=public.crt
#dbms.ssl.policy.https.client_auth=NONE
# Cluster SSL configuration
#dbms.ssl.policy.cluster.enabled=true
#dbms.ssl.policy.cluster.base_directory=certificates/cluster
#dbms.ssl.policy.cluster.private_key=private.key
#dbms.ssl.policy.cluster.public_certificate=public.crt
# Backup SSL configuration
#dbms.ssl.policy.backup.enabled=true
#dbms.ssl.policy.backup.base_directory=certificates/backup
#dbms.ssl.policy.backup.private_key=private.key
#dbms.ssl.policy.backup.public_certificate=public.crt
#*****************************************************************
# Logging configuration
#*****************************************************************
# To enable HTTP logging, uncomment this line
#dbms.logs.http.enabled=true
# To enable GC Logging, uncomment this line
#server.logs.gc.enabled=true
# GC Logging Options
# see https://docs.oracle.com/en/java/javase/11/tools/java.html#GUID-BE93ABDC-999C-4CB5-A88B-1994AAAC74D5
#server.logs.gc.options=-Xlog:gc*,safepoint,age*=trace
# Number of GC logs to keep.
#server.logs.gc.rotation.keep_number=5
# Size of each GC log that is kept.
#server.logs.gc.rotation.size=20m
#*****************************************************************
# Miscellaneous configuration
#*****************************************************************
# Determines if Cypher will allow using file URLs when loading data using
# `LOAD CSV`. Setting this value to `false` will cause Neo4j to fail `LOAD CSV`
# clauses that load data from the file system.
#dbms.security.allow_csv_import_from_file_urls=true
# Value of the Access-Control-Allow-Origin header sent over any HTTP or HTTPS
# connector. This defaults to '*', which allows broadest compatibility. Note
# that any URI provided here limits HTTP/HTTPS access to that URI only.
#dbms.security.http_access_control_allow_origin=*
# Value of the HTTP Strict-Transport-Security (HSTS) response header. This header
# tells browsers that a webpage should only be accessed using HTTPS instead of HTTP.
# It is attached to every HTTPS response. Setting is not set by default so
# 'Strict-Transport-Security' header is not sent. Value is expected to contain
# directives like 'max-age', 'includeSubDomains' and 'preload'.
#dbms.security.http_strict_transport_security=
# Retention policy for transaction logs needed to perform recovery and backups.
#db.tx_log.rotation.retention_policy=2 days
# Whether or not any database on this instance are read_only by default.
# If false, individual databases may be marked as read_only using dbms.database.read_only.
# If true, individual databases may be marked as writable using dbms.databases.writable.
#dbms.databases.default_to_read_only=false
# Comma separated list of JAX-RS packages containing JAX-RS resources, one
# package name for each mountpoint. The listed package names will be loaded
# under the mountpoints specified. Uncomment this line to mount the
# org.neo4j.examples.server.unmanaged.HelloWorldResource.java from
# neo4j-server-examples under /examples/unmanaged, resulting in a final URL of
# http://localhost:7474/examples/unmanaged/helloworld/{nodeId}
#server.unmanaged_extension_classes=org.neo4j.examples.server.unmanaged=/examples/unmanaged
# A comma separated list of procedures and user defined functions that are allowed
# full access to the database through unsupported/insecure internal APIs.
#dbms.security.procedures.unrestricted=my.extensions.example,my.procedures.*
# A comma separated list of procedures to be loaded by default.
# Leaving this unconfigured will load all procedures found.
#dbms.security.procedures.allowlist=apoc.coll.*,apoc.load.*,gds.*
#********************************************************************
# JVM Parameters
#********************************************************************
# G1GC generally strikes a good balance between throughput and tail
# latency, without too much tuning.
server.jvm.additional=-XX:+UseG1GC
# Have common exceptions keep producing stack traces, so they can be
# debugged regardless of how often logs are rotated.
server.jvm.additional=-XX:-OmitStackTraceInFastThrow
# Make sure that `initmemory` is not only allocated, but committed to
# the process, before starting the database. This reduces memory
# fragmentation, increasing the effectiveness of transparent huge
# pages. It also reduces the possibility of seeing performance drop
# due to heap-growing GC events, where a decrease in available page
# cache leads to an increase in mean IO response time.
# Try reducing the heap memory, if this flag degrades performance.
server.jvm.additional=-XX:+AlwaysPreTouch
# Trust that non-static final fields are really final.
# This allows more optimizations and improves overall performance.
# NOTE: Disable this if you use embedded mode, or have extensions or dependencies that may use reflection or
# serialization to change the value of final fields!
server.jvm.additional=-XX:+UnlockExperimentalVMOptions
server.jvm.additional=-XX:+TrustFinalNonStaticFields
# Disable explicit garbage collection, which is occasionally invoked by the JDK itself.
server.jvm.additional=-XX:+DisableExplicitGC
# Restrict size of cached JDK buffers to 1 KB
server.jvm.additional=-Djdk.nio.maxCachedBufferSize=1024
# More efficient buffer allocation in Netty by allowing direct no cleaner buffers.
server.jvm.additional=-Dio.netty.tryReflectionSetAccessible=true
# Exits JVM on the first occurrence of an out-of-memory error. Its preferable to restart VM in case of out of memory errors.
# server.jvm.additional=-XX:+ExitOnOutOfMemoryError
# Expand Diffie Hellman (DH) key size from default 1024 to 2048 for DH-RSA cipher suites used in server TLS handshakes.
# This is to protect the server from any potential passive eavesdropping.
server.jvm.additional=-Djdk.tls.ephemeralDHKeySize=2048
# This mitigates a DDoS vector.
server.jvm.additional=-Djdk.tls.rejectClientInitiatedRenegotiation=true
# Enable remote debugging
#server.jvm.additional=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
# This filter prevents deserialization of arbitrary objects via java object serialization, addressing potential vulnerabilities.
# By default this filter whitelists all neo4j classes, as well as classes from the hazelcast library and the java standard library.
# These defaults should only be modified by expert users!
# For more details (including filter syntax) see: https://openjdk.java.net/jeps/290
#server.jvm.additional=-Djdk.serialFilter=java.**;org.neo4j.**;com.neo4j.**;com.hazelcast.**;net.sf.ehcache.Element;com.sun.proxy.*;org.openjdk.jmh.**;!*
# Increase the default flight recorder stack sampling depth from 64 to 256, to avoid truncating frames when profiling.
server.jvm.additional=-XX:FlightRecorderOptions=stackdepth=256
# Allow profilers to sample between safepoints. Without this, sampling profilers may produce less accurate results.
server.jvm.additional=-XX:+UnlockDiagnosticVMOptions
server.jvm.additional=-XX:+DebugNonSafepoints
# Open modules for neo4j to allow internal access
server.jvm.additional=--add-opens=java.base/java.nio=ALL-UNNAMED
server.jvm.additional=--add-opens=java.base/java.io=ALL-UNNAMED
server.jvm.additional=--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
# Disable logging JMX endpoint.
server.jvm.additional=-Dlog4j2.disable.jmx=true
# Limit JVM metaspace and code cache to allow garbage collection. Used by cypher for code generation and may grow indefinitely unless constrained.
# Useful for memory constrained environments
#server.jvm.additional=-XX:MaxMetaspaceSize=1024m
#server.jvm.additional=-XX:ReservedCodeCacheSize=512m
# Allow big methods to be JIT compiled.
# Useful for big queries and big expressions where cypher code generation can create large methods.
#server.jvm.additional=-XX:-DontCompileHugeMethods
#********************************************************************
# Wrapper Windows NT/2000/XP Service Properties
#********************************************************************
# WARNING - Do not modify any of these properties when an application
# using this configuration file has been installed as a service.
# Please uninstall the service before modifying this section. The
# service can then be reinstalled.
# Name of the service
server.windows_service_name=neo4j
#********************************************************************
# Other Neo4j system properties
#********************************************************************
dbms.security.procedures.unrestricted=apoc.*
db.tx_log.rotation.retention_policy=100M size
server.memory.pagecache.size=512M
server.default_listen_address=0.0.0.0
#### SEMSPECT ####
server.unmanaged_extension_classes=de.derivo.semspect.server.neo4japp.server=/semspect
#dbms.security.http_auth_allowlist=/,/semspect.*
dbms.security.http_auth_allowlist=/,/browser.*,/semspect.*
server.directories.logs=/logs
server.config.strict_validation.enabled=false
dbms.security.auth_enabled=false
## new advertise
server.default_listen_address=0.0.0.0
server.bolt.enabled=true
server.bolt.listen_address=:7687
server.bolt.advertised_address=srvdizsrvmedaxdev.med.uni-greifswald.de:8081
server.bolt.tls_level=REQUIRED
dbms.ssl.policy.bolt.enabled=true
dbms.ssl.policy.bolt.base_directory=certificates/bolt
dbms.ssl.policy.bolt.private_key=private.key
dbms.ssl.policy.bolt.public_certificate=public.crt
dbms.ssl.policy.bolt.client_auth=NONE
server.http.enabled=true
server.http.listen_address=:7474
server.http.advertised_address=srvdizsrvmedaxdev.med.uni-greifswald.de:443
mdm2neo4j
-1
Submodule mdm2neo4j deleted from 3f91b1847d.
medax_softwareplan.svg
-4177
View File
File diff suppressed because it is too large. Load diff
Side by Side

Before

Width:  |  Height:  |  Size: 367 KiB

multiple_adapters.py
-63
View File
@@ -1,63 +0,0 @@
import sys
import os
from biocypher import BioCypher
from schema_config_generation import write_automated_schema
# Add submodule to path BEFORE any other imports
sys.path.insert(0, '/app/mdm2neo4j/src') # Absolute path in Docker
# OR
#sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'mdm2neo4j')) # Relative path
# Now your imports should work
import mdm2neo4j.src.xml_processor.xml_processor as xp
from import_fhir_to_nx_diGraph import generate_neo4j_import_script
from import_fhir_to_nx_diGraph import load_multiple_fhir_patients
#import mdm2neo4j.src.xml_processor.xml_processor as xp
def main():
# create Biocypher driver
bc = BioCypher(
biocypher_config_path="config/biocypher_config.yaml",
)
#bc.show_ontology_structure() #very extensive
#BioCypher preperation
## create networkX and run improvement scripts
adapter_mode = 0
xml_file_path = "./odm_models"
write_automated_schema(None, 'config/automated_schema.yaml', ['config/manual_schema_config.yaml', 'config/ODM_schema.yaml'])
if(adapter_mode == 0 or adapter_mode == -1):
n_patients = int(os.getenv('NUMBER_OF_PATIENTS'))
print("--- load ", n_patients, " fhir patients ---")
load_multiple_fhir_patients(n_patients)
if(adapter_mode == 1 or adapter_mode == -1):
#bc.show_ontology_structure() #very extensive
bc.write_nodes(xp.parse_xml_generate_nodes(xml_file_path))
bc.write_edges(xp.parse_xml_generate_edges(xml_file_path))
print("CREATING THE SCRIPT", flush=True)
generate_neo4j_import_script()
with open('/neo4j_import/shell-scipt-complete', 'w') as f:
f.write('Import completed successfully')
print("FHIR import completed successfully")
if __name__ == "__main__":
main()
pipeline.puml
+34 -84
View File
@@ -1,94 +1,44 @@
@startuml Server Architecture
@startuml "MeDaX pipeline"
left to right direction
actor admin
database "fhir server" as fhir
node "docker compose" as compose{
node "python app" as pyApp {
[scripts]
[nodeGenerator] as ngen
[edgeGenerator] as egen
[BioCypher] as BC
file "generated Schema" as gSchema
file "manual Schema" as mSchema
!define RECTANGLE class
' Define styles
skinparam rectangle {
BackgroundColor<<zone>> LightBlue
BorderColor<<zone>> Navy
BackgroundColor<<server>> LightGreen
BorderColor<<server>> DarkGreen
BackgroundColor<<admin>> LightYellow
BorderColor<<admin>> Orange
BackgroundColor<<client>> LightPink
BorderColor<<client>> Red
BackgroundColor<<database>> LightCyan
BorderColor<<database>> DarkCyan
BackgroundColor<<component>> Wheat
BorderColor<<component>> Brown
}
' Zone 2 container
rectangle "Zone 2" <<zone>> {
' Admin DIZ (provider)
actor "Admin DIZ\n(Provider)" <<admin>> as AdminD
' BLAZE Server
rectangle "BLAZE Server" <<server>> as BlazeServer {
note "FHIR data source for\nthe pipeline" as n1
mSchema --> scripts : input
scripts --> gSchema : generates
scripts --> ngen : generates
scripts --> egen : generates
gSchema --> BC : input
ngen--> BC : input
egen--> BC : input
}
node "neo4j app" as neoApp{
database "neo4j GDB" as neoDB
[web server] as neoServer
' Server M with internal components
rectangle "Server MeDaX" <<server>> as ServerM {
' Web Interface
rectangle "Web Interface" <<component>> as WebInterface
' Pipeline components
rectangle "Pipeline" <<component>> as Pipeline {
rectangle "Python Scripts" <<component>> as PythonScripts
collections "Input Files" <<files>> as InputFiles
note top of PythonScripts : 1. Python scripts fetch data\n2. Create input files\n3. Load into GDB
' Graph Database
database "Graph Database\n(GDB)" <<database>> as GraphDB
neoDB --> neoServer
}
folder "admin files" as afiles {
file nodes
file edges
file "import script" as iscript
}
}
rectangle WTSDIZ <<client>> as DIZclient
admin -[dashed]-> compose : triggers
BC --> afiles : exports
fhir --> scripts : http request
afiles --> neoApp : input
}
' Admin MeDaX
rectangle "MeDaX Admins" <<admin>> as AdminGroup {
actor "Admin 1" as Admin1
actor "Admin 2" as Admin2
}
' Clients
rectangle "Clients" <<client>> as Clients {
rectangle "Clients 1 - MeDaX" as Client1
rectangle "Clients 2 - Allgemeinmedizin" as Client2
}
' Hidden alignment helpers
BlazeServer -[hidden]right- ServerM
BlazeServer -[hidden]down- AdminD
' Pipeline flow within Server M
PythonScripts -down-> InputFiles : creates
InputFiles -down-> GraphDB : loads into
' BLAZE Server connection
PythonScripts -up-> BlazeServer : requests data
' Web Interface connection
WebInterface -up-> GraphDB : connected to
' External relationships
AdminD -up-> ServerM : manages, secures & installs
AdminGroup -up-> DIZclient : access via VPN
DIZclient -left-> ServerM : ssh connection
Clients -up-> DIZclient : access via VPN
DIZclient -left-> WebInterface : access
actor user
user --> neoServer : uses
neoApp --> pyApp : kills
@enduml
poetry.lock
Generated
+3315
View File
File diff suppressed because it is too large. Load diff
pyproject.toml
+55 -66
View File
@@ -1,8 +1,7 @@
[project]
name = "medax-pipeline"
version = "1.0.1"
description = "An ETL pipeline to transfer FHIR data into a graph database"
[tool.poetry]
name = "MeDaX pipeline"
version = "1.0.0"
description = "A unifying framework for biomedical research knowledge graphs"
authors = [
"Ilya Mazien",
"Tom Gebhardt",
@@ -12,82 +11,72 @@ authors = [
"Dagmar Waltemath",
"Judith Wodke"
]
license = "MIT"
readme = "README.md"
requires-python = ">=3.10"
packages = [
{ include = "biocypher" }
]
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Natural Language :: English",
"Topic :: Scientific/Engineering :: Medical-Informatics",
]
dependencies = [
"pyyaml>=5.0",
"biocypher>=0.16.0,<1.0.0",
"more-itertools",
"appdirs",
"treelib==1.6.4",
"rdflib>=6.2.0,<7.0.0",
"networkx>=3.0,<4.0",
"stringcase>=1.2.0,<2.0.0",
"neo4j-utils==0.0.7",
"pandas>=2.0.1,<3.0.0",
"pooch>=1.7.0,<2.0.0",
"tqdm>=4.65.0,<5.0.0",
"requests>=2.31.0,<3.0.0",
"python-dotenv>=1.0.0,<2.0.0",
"beautifulsoup4>=4.12.3,<5.0.0",
"lxml>=5.1.0,<6.0.0",
"Topic :: Scientific/Engineering :: Bio-Informatics"
]
repository = "https://github.com/biocypher/biocypher"
readme = "README.md"
[project.urls]
Homepage = "https://www.medizin.uni-greifswald.de/medizininformatik/research/current-projects/medax/"
Repository = "https://git.uni-greifswald.de/MILA_public/medax_pipeline"
# biocypher is pulled from a feature branch via uv's source override
[tool.uv.sources]
#biocypher = { git = "https://github.com/biocypher/biocypher.git", branch = "feat/dynamic-node-property-headers" } #enable this to test the feature branch again
[tool.poetry.dependencies]
python = "^3.9"
PyYAML = ">=5.0"
more_itertools = "*"
appdirs = "*"
treelib = "1.6.4"
rdflib = "^6.2.0"
networkx = "^3.0"
stringcase = "^1.2.0"
neo4j-utils = "0.0.7"
pandas = "^2.0.1"
pooch = "^1.7.0"
tqdm = "^4.65.0"
[tool.poetry.group.dev.dependencies]
sphinx = ">=5.0.0"
sphinx-design = "^0.3.0"
sphinx-rtd-theme = ">=1.0.0"
sphinx-last-updated-by-git = ">=0.3"
sphinx-autodoc-typehints = ">=1.18.0"
myst-parser = "^0.18.0"
yapf = "^0.32.0"
pytest = ">=6.0"
tox = ">=3.20.1"
pre-commit = ">=2.17.0"
bump2version = "*"
coverage = ">=6.0"
pytest-cov = "^3.0.0"
hypothesis = "^6.50.1"
isort = "^5.10.1"
ipython = "^8.7.0"
ipykernel = "^6.23.1"
sphinxext-opengraph = "^0.8.2"
coverage-badge = "^1.1.0"
nbsphinx = "^0.9.2"
black = "^23.9.1"
flake8 = "^6.1.0"
[tool.uv]
package = false
[build-system]
requires = ["poetry-core<2.0.0"]
build-backend = "poetry.core.masonry.api"
# Dev dependencies (PEP 735 dependency groups, natively understood by uv)
[dependency-groups]
dev = [
"sphinx>=5.0.0",
"sphinx-design>=0.3.0,<0.4.0",
"sphinx-rtd-theme>=1.0.0",
"sphinx-last-updated-by-git>=0.3",
"sphinx-autodoc-typehints>=1.18.0",
"myst-parser>=0.18.0,<0.19.0",
"yapf>=0.32.0,<0.33.0",
"pytest>=6.0",
"tox>=3.20.1",
"pre-commit>=2.17.0",
"bump2version",
"coverage>=6.0",
"pytest-cov>=3.0.0,<4.0.0",
"hypothesis>=6.50.1,<7.0.0",
"isort>=5.10.1,<6.0.0",
"ipython>=8.7.0,<9.0.0",
"ipykernel>=6.23.1,<7.0.0",
"sphinxext-opengraph>=0.8.2,<0.9.0",
"coverage-badge>=1.1.0,<2.0.0",
"nbsphinx>=0.9.2,<0.10.0",
"black>=23.9.1,<24.0.0",
"flake8>=6.1.0,<7.0.0",
]
# --- Tool configs below are build-tool-agnostic and carried over unchanged ---
[tool.poetry.urls]
"Bug Tracker" = "https://github.com/biocypher/biocypher/issues"
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "--ignore=mdm2neo4j"
log_cli = true
log_level = "INFO"
markers = [
@@ -123,9 +112,9 @@ line_length = 80
multi_line_output = 3
include_trailing_comma = true
use_parentheses = true
known_num = "numpy,pandas"
known_num="numpy,pandas"
sections = "FUTURE,STDLIB,THIRDPARTY,NUM,FIRSTPARTY,LOCALFOLDER"
no_lines_before = "LOCALFOLDER"
no_lines_before="LOCALFOLDER"
balanced_wrapping = true
force_grid_wrap = 0
length_sort = "1"
@@ -137,7 +126,7 @@ ignore = ["E203", "D200", "D202", "D401", "D105", "W504"]
per-file-ignores = [
"docs/source/conf.py:D100",
"tests/*:D100,D101,D102",
"*/__init__.py:F401",
"*/__init__.py:F401"
]
max-line-length = 80
count = true
reproduce_header_misalignment.py
-105
View File
@@ -1,105 +0,0 @@
"""
Reproduction: vanilla BioCypher corrupts already-written part files when a later
batch of the same node type introduces a new property.
Run:
pip install biocypher
python reproduce_header_misalignment.py
Tested against BioCypher 0.15.1. Self-contained: writes its own config files
into a temp dir, needs no Neo4j (offline CSV writer only).
Background
----------
MeDaX ingests data in successive batches and extends the schema on the fly:
when a node/edge type or attribute first appears, it is added to the schema so
no information is dropped. BioCypher's Neo4j batch writer rewrites the *header*
file on every write_nodes() call, but leaves the *part files already on disk*
untouched. As soon as a new attribute widens/reorders the header, the rows in
earlier part files no longer line up with it -> silent column misalignment on
neo4j-admin import.
Exact location upstream:
biocypher/output/write/graph/_neo4j.py
_Neo4jBatchWriter._write_node_headers() (and ._write_edge_headers())
-> the `if os.path.exists(header_path): logger.warning("... Overwriting.")`
branch rewrites the header but never reconciles existing -part*.csv files.
"""
import glob
import os
import sys
import tempfile
from biocypher import BioCypher
workdir = tempfile.mkdtemp(prefix="bc_repro_")
os.chdir(workdir)
with open("schema_config.yaml", "w") as fh:
fh.write(
"patient:\n"
" represented_as: node\n"
" input_label: patient\n"
" is_a: entity\n" # no fixed `properties:` -> inferred from data per batch
)
with open("biocypher_config.yaml", "w") as fh:
fh.write(
"biocypher:\n"
" dbms: neo4j\n"
" offline: true\n"
" strict_mode: false\n"
" schema_config_path: schema_config.yaml\n"
" head_ontology:\n"
" url: https://raw.githubusercontent.com/biolink/biolink-model/v3.2.1/biolink-model.owl.ttl\n"
" root_node: entity\n"
)
bc = BioCypher(
biocypher_config_path="biocypher_config.yaml",
schema_config_path="schema_config.yaml",
output_directory="out",
)
# Batch 1: patients described by {name, age}
bc.write_nodes(
[
("p1", "patient", {"name": "Alice", "age": 30}),
("p2", "patient", {"name": "Bob", "age": 41}),
],
force=True,
)
# Batch 2: same node type, now with an extra `diagnosis` attribute.
bc.write_nodes(
[("p3", "patient", {"name": "Carol", "age": 52, "diagnosis": "E11"})],
force=True,
)
# --- Inspect what ended up on disk -----------------------------------------
header_path = "out/Patient-header.csv"
header_cols = open(header_path).read().strip().split(";")
n_header = len(header_cols)
print("HEADER :", header_cols, f"({n_header} columns)")
ok = True
for part in sorted(glob.glob("out/Patient-part*.csv")):
for row in open(part).read().splitlines():
n_row = len(row.split(";"))
flag = "" if n_row == n_header else " <-- MISALIGNED"
if n_row != n_header:
ok = False
print(f"{os.path.basename(part)}: {n_row} columns{flag} {row}")
print()
if ok:
print("PASS: every part file matches the header.")
sys.exit(0)
else:
print(
"FAIL: a part file written before the schema grew no longer matches the\n"
" header. On import, its values land under the wrong columns\n"
" (e.g. the node id ends up under `diagnosis`, :LABEL ends up empty)."
)
sys.exit(1)
requirements.txt
+2
View File
@@ -0,0 +1,2 @@
requests==2.31.0
python-dotenv==1.0.0
schema_config_generation.py
+74 -129
View File
@@ -3,82 +3,82 @@
from pathlib import Path
#import networkx as nx
import yaml
from collections import defaultdict
#extract all node types and generate basic yaml config part for nodes
def write_automated_schema(graph, file_path, manual_schema_path):
schema_data = {
def write_automated_schema(graph, filePath, mSchemaPath):
schemaData = {
'nodes': {},
'edges': {}
}
if Path(file_path).exists():
print("-- Using existing schema --")
schema_data = load_manual_schema(file_path)
elif isinstance(manual_schema_path, list):
print("-- Using two schema files --")
schema_data = schema_merger(manual_schema_path[0], manual_schema_path[1])
elif manual_schema_path:
print("-- Using the manual schema --")
schema_data = load_manual_schema(manual_schema_path)
if Path(filePath).exists():
schemaData = loadManualSchema(filePath)
elif mSchemaPath:
print("using the manual schema")
schemaData = loadManualSchema(mSchemaPath)
if graph is not None:
for node in graph.nodes():
label = graph.nodes[node].get('label')
if label == 'resource':
label = graph.nodes[node].get('resourceType')
label = label.capitalize()
if label not in schema_data['nodes']:
schema_data['nodes'][label] = {}
if not label in schemaData['nodes']:
schemaData['nodes'][label] = {}
if 'properties' not in schema_data['nodes'][label] or schema_data['nodes'][label]['properties'] is None:
schema_data['nodes'][label]['properties'] = {}
if not 'properties' in schemaData['nodes'][label]:
schemaData['nodes'][label]['properties'] = {}
for k in graph.nodes[node].keys():
schema_data['nodes'][label]['properties'][k] = 'str'
#print(k, '----- ', graph.nodes[node][k])
#if k != 'label':
schemaData['nodes'][label]['properties'][k] = 'str'
#schema_data['nodes'][label]['properties'].update(graph.nodes[node].keys())
#schemaData['nodes'][label]['properties'].update(graph.nodes[node].keys())
file=open(file_path, 'w')
file=open(filePath, 'w')
for n in schema_data['nodes']:
for n in schemaData['nodes']:
temp = n+':\n'
if 'is_a' in schema_data['nodes'][n]:
is_a_value = schema_data['nodes'][n]['is_a']
temp += ' is_a: ' + (', '.join(is_a_value) if isinstance(is_a_value, list) else str(is_a_value)) + '\n'
if 'is_a' in schemaData['nodes'][n]:
temp += ' is_a: ' + schemaData['nodes'][n]['is_a'] + '\n'
else:
temp += ' is_a: named thing\n'
if 'represented_as' in schema_data['nodes'][n]:
represented_as_value = schema_data['nodes'][n]['represented_as']
temp += ' represented_as: ' + (', '.join(represented_as_value) if isinstance(represented_as_value, list) else str(represented_as_value)) + '\n'
if 'represented_as' in schemaData['nodes'][n]:
temp += ' represented_as: ' + schemaData['nodes'][n]['represented_as'] + '\n'
else:
temp += ' represented_as: node\n'
if 'label_in_input' in schema_data['nodes'][n]:
label_in_input_value = schema_data['nodes'][n]['label_in_input']
temp += ' label_in_input: ' + (', '.join(label_in_input_value) if isinstance(label_in_input_value, list) else str(label_in_input_value)) + '\n'
if 'preferred_id' in schema_data['nodes'][n]:
preferred_id_value = schema_data['nodes'][n]['preferred_id']
temp += ' preferred_id: ' + (', '.join(preferred_id_value) if isinstance(preferred_id_value, list) else str(preferred_id_value)) + '\n'
if 'label_in_input' in schemaData['nodes'][n]:
temp += ' label_in_input: ' + schemaData['nodes'][n]['label_in_input'] + '\n'
if 'preferred_id' in schemaData['nodes'][n]:
temp += ' preferred_id: ' + schemaData['nodes'][n]['preferred_id'] + '\n'
else:
temp += ' preferred_id: fhir_id\n'
temp += ' label_in_input: ' + n + '\n'
temp += ' properties:\n'
# get property values from schema_data if exists
#print("---------->", str(schema_data['nodes'][n]))
if schema_data['nodes'][n]['properties'] is not None:
for p_key in schema_data['nodes'][n]['properties']:
prop_value = schema_data['nodes'][n]['properties'][p_key]
temp += ' ' + p_key + ': ' + (', '.join(prop_value) if isinstance(prop_value, list) else str(prop_value)) + '\n'
#elif schema_data['nodes']['properties']:
#print("----> ", schema_data['nodes']['properties'])
# get property values from schemaData if exists
for pKey in schemaData['nodes'][n]['properties']:
temp += ' ' + pKey + ': ' + schemaData['nodes'][n]['properties'][pKey] + '\n'
#elif schemaData['nodes']['properties']:
#print("----> ", schemaData['nodes']['properties'])
""" else:
for attr in schema_data['nodes'][n]:
for attr in schemaData['nodes'][n]:
temp += ' ' + attr + ': str\n' """
temp += '\n'
@@ -90,63 +90,61 @@ def write_automated_schema(graph, file_path, manual_schema_path):
#extract all relationship types and generate basic yaml config part for relationships
#if not edgeTypes: edgeTypes = set()
if graph is not None:
for u, v, a in graph.edges(data=True):
#edge_label = graph[u][v].get('edge_type', '')
source_label = graph.nodes[u].get('label')
target_label = graph.nodes[v].get('label')
if source_label == 'resource':
source_label = graph.nodes[u].get('resourceType', str(u))
elif source_label == 'dummy' or source_label == 'Dummy':
source_label = graph.nodes[u].get('edge_label', str(u))
if target_label == 'resource':
target_label = graph.nodes[v].get('resourceType', str(v))
elif target_label == 'dummy' or target_label == 'Dummy':
target_label = graph.nodes[v].get('edge_label', str(v))
source_label = source_label.capitalize()
target_label = target_label.capitalize()
#target_label = target_label.capitalize()
CONST_ASSOCIATION = ' association'
if source_label + ' to ' + target_label + CONST_ASSOCIATION in schema_data['edges']:
if source_label + ' to ' + target_label + ' association' in schemaData['edges']:
# add missing attributes
continue
elif source_label + ' derived from ' + target_label + CONST_ASSOCIATION in schema_data['edges']:
elif source_label + ' derived from ' + target_label + ' association' in schemaData['edges']:
continue
elif source_label + ' has member ' + target_label + CONST_ASSOCIATION in schema_data['edges']:
elif source_label + ' has member ' + target_label + ' association' in schemaData['edges']:
continue
elif source_label + ' reasoned by ' + target_label + CONST_ASSOCIATION in schema_data['edges']:
elif source_label + ' reasoned by ' + target_label + ' association' in schemaData['edges']:
continue
elif source_label + ' is ' + target_label + CONST_ASSOCIATION in schema_data['edges']:
elif source_label + ' is ' + target_label + ' association' in schemaData['edges']:
continue
else:
schema_data['edges'][source_label + ' to ' + target_label + ' association'] = {
#schemaData['edges'][source_label + ' to ' + target_label + ' association'] = set()
schemaData['edges'][source_label + ' to ' + target_label + ' association'] = {
'is_a': 'association',
'represented_as': 'edge',
'label_in_input': source_label + '_to_' + target_label,
'properties': a
}
for label in schema_data['edges']:
for label in schemaData['edges']:
temp = '' + label + ':\n'
for key in schema_data['edges'][label]:
for key in schemaData['edges'][label]:
if key == 'properties':
if schema_data['edges'][label][key] is not None:
temp += ' properties:\n'
for prop in schema_data['edges'][label][key]:
prop_value = schema_data['edges'][label][key][prop]
temp += ' ' + prop + ': ' + (', '.join(prop_value) if isinstance(prop_value, list) else str(prop_value)) + '\n'
for prop in schemaData['edges'][label][key]:
temp += ' ' + prop + ': ' + schemaData['edges'][label][key][prop] + '\n'
else:
field_value = schema_data['edges'][label][key]
temp += ' ' + key + ': ' + (', '.join(field_value) if isinstance(field_value, list) else str(field_value)) + '\n'
temp+= ' ' + key + ': ' + schemaData['edges'][label][key] + '\n'
temp += '\n'
file.write(temp)
file.close()
def load_manual_schema(path):
schema_data = {
def loadManualSchema(path):
schemaData = {
'nodes': {},
'edges': {}
}
@@ -157,87 +155,34 @@ def load_manual_schema(path):
data = yaml.safe_load(file)
for label, attrs in data.items():
cLabel = label#.capitalize() # less n, e and p with capitalize
if label != 'Title':
cLabel = label.capitalize()
if not label == 'Title':
if attrs["represented_as"] == 'node':
if not hasattr(schema_data['nodes'], cLabel):
schema_data['nodes'][cLabel] = set()
if not hasattr(schemaData['nodes'], cLabel):
schemaData['nodes'][cLabel] = set()
#assuming uniqueness in schema file here. If the same node type exits twice, it will be overwritten.
schema_data['nodes'][cLabel] = attrs
schemaData['nodes'][cLabel] = attrs
#for a in attrs:
#print(v)
""" for k, v in attrs:
if not k == ''
schema_data['nodes'][label][k] = v """
schemaData['nodes'][label][k] = v """
else:
if not hasattr(schema_data['edges'], cLabel):
schema_data['edges'][cLabel] = set()
if not hasattr(schemaData['edges'], cLabel):
schemaData['edges'][cLabel] = set()
#assuming uniqueness in schema file here. If the same node type exits twice, it will be overwritten.
schema_data['edges'][cLabel] = attrs
schemaData['edges'][cLabel] = attrs
return schemaData
return schema_data
def schema_merger(schema_path1, schema_path2):
schema1 = load_manual_schema(schema_path1)
schema2 = load_manual_schema(schema_path2)
merged = {
'nodes': {**schema1['nodes'], **schema2['nodes']},
'edges': {**schema1['edges'], **schema2['edges']}
}
return merged
def schema_diff_report(manual_path, auto_path):
#manual_path = "config/manual_schema_config.yaml"
#auto_path = "config/automated_schema.yaml"
man_schema = load_manual_schema(manual_path)
automated_schema = load_manual_schema(auto_path)
print("\nNODES:\n")
for label, attributes in automated_schema['nodes'].items():
if 'nodes' in man_schema and label in man_schema['nodes']: # node already exists
print("Found in man schema: ", label)
for prop, val in attributes.items():
if prop in man_schema['nodes'][label]: # property exists
if isinstance(val, dict):
for attr, attr_val in val.items():
if attr not in man_schema['nodes'][label][prop]:
print("Added the property " + attr + " to " + prop + " of " + label)
elif attr_val != man_schema['nodes'][label][prop][attr]:
print("Updated property value of " + label + "." + prop + "." + attr + "to: " + attr_val)
elif val != man_schema['nodes'][label][prop]: # property value is not the same
print("Updated property value of " + label + "." + prop + " to: " + val)
else:
print(label + "." + prop + " has not changed")
else:
print(label + ": added property " + prop + ". Value: " + val)
else: # node not found in manual schema
print('Resource type added: ' + label)
print("\nEDGES:\n")
for label, attributes in automated_schema['edges'].items():
if 'edges' in man_schema and label in man_schema['edges']: # node already exists
print("Found in man schema: ", label)
for prop, val in attributes.items():
if prop in man_schema['edges'][label]: # property exists
if isinstance(val, dict):
for attr, attr_val in val.items():
if attr not in man_schema['edges'][label][prop]:
print("Added the property " + attr + " to " + prop)
elif attr_val != man_schema['edges'][label][prop][attr]:
print("Updated property value of " + label + "." + prop + "." + attr + "to: " + attr_val)
elif val != man_schema['edges'][label][prop]: # property value is not the same
print("Updated property value of " + label + "." + prop + " to: " + val)
else:
print(label + "." + prop + " has not changed")
else:
print(label + ": added property " + prop + ". Value: " + val)
else: # node not found in manual schema
print('Resource type added: ' + label)
selected_patients.txt
-250
View File
@@ -1,250 +0,0 @@
52466995
52467955
52466558
5
13
14
22
30
37
57
110
118
134
170
177
id1568758349858
233
567
768
1006
1009
1021
1029
1059
1091
1190
1194
1200
1201
1203
1208
1222
1316
1526
1654
2000
patient
24739
25420
30163
30358
31678
38100
39254
40052
40625
42024
45276
45280
49006
50427
123a
example
cf-1572285491892
51779
52865
53254
53373
55190
55193
55810
55812
59530
69050
76107
141547
141549
206342
206686
221706
221709
257273
Patient1
pat1
pat2
579296
579367
581516
591093
591164
591221
591229
591245
591252
591253
591264
591286
591309
591352
591360
591372
591378
591427
591626
591645
591653
591661
591695
591702
591723
591727
591744
591788
591829
591841
591874
591956
591975
592014
592030
592215
592228
592274
592275
592294
592295
592318
592350
592353
592473
592474
592485
592502
592529
592530
592580
592621
592733
592736
592738
592740
592741
592760
592761
592766
592773
592808
592809
592817
592824
592841
592845
592911
592912
592913
592922
592940
592943
592944
592955
592956
592995
593004
593043
593160
593165
593166
593170
593171
593178
593186
593190
593210
593218
593271
593321
593380
594843
595184
595251
595262
595271
595280
595284
595606
596328
596336
596341
596357
596380
596388
596407
596438
596439
596467
596490
596492
596495
596496
596498
596500
596505
596506
596510
596513
596517
596518
596520
596522
596544
596545
596546
596547
596555
596557
596561
596562
596563
596568
596570
596571
596574
596575
596599
596615
596634
596638
596644
596805
596813
596857
597041
597063
597064
597065
597067
597068
597069
597083
597110
597118
597172
597176
597178
597226
597291
597296
597305
597306
597309
597395
597521
597879
597991
599548
602313
611763
618626
618670
618750
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0001.json
-22482
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0002.json
-7582
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0003.json
-5263
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0004.json
-4918
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0005.json
-6355
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0006.json
-3735
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0007.json
-3331
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0008.json
-11215
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0009.json
-5250
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0010.json
-5637
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0011.json
-6131
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0012.json
-5656
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0013.json
-7614
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0014.json
-16014
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKB-UKB-0015.json
-11151
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0001.json
-28963
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0002.json
-51605
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0003.json
-1802
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0004.json
-2876
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0005.json
-3299
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0006.json
-3671
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0007.json
-4895
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0008.json
-1930
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0009.json
-3091
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0010.json
-3263
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0011.json
-11681
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0012.json
-1641
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0013.json
-959
View File
@@ -1,959 +0,0 @@
{
"resourceType": "Bundle",
"type": "transaction",
"entry": [ {
"fullUrl": "Patient/UKE-0013",
"resource": {
"resourceType": "Patient",
"id": "UKE-0013",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-person/StructureDefinition/Patient" ]
},
"identifier": [ {
"use": "usual",
"type": {
"coding": [ {
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "MR"
} ]
},
"system": "https://UKE.de/pid",
"value": "UKE-0013"
} ],
"name": [ {
"use": "official",
"family": "l",
"given": [ "i" ]
} ],
"gender": "male",
"birthDate": "1943-01-01",
"address": [ {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
} ],
"generalPractitioner": [ {
"display": "Barmer"
} ]
},
"request": {
"method": "PUT",
"url": "Patient/UKE-0013"
}
}, {
"fullUrl": "Encounter/UKE-0013-E-1",
"resource": {
"resourceType": "Encounter",
"id": "UKE-0013-E-1",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-fall/StructureDefinition/KontaktGesundheitseinrichtung" ]
},
"identifier": [ {
"type": {
"coding": [ {
"system": "http://terminology.hl7.org/CodeSystem/v2-0203",
"code": "VN"
} ]
},
"_system": {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
},
"value": "UKE-0013-E-1",
"assigner": {
"identifier": {
"system": "https://www.medizininformatik-initiative.de/fhir/core/NamingSystem/org-identifier",
"value": "UKE"
}
}
} ],
"status": "finished",
"class": {
"system": "http://terminology.hl7.org/CodeSystem/v3-ActCode",
"code": "IMP",
"display": "inpatient encounter"
},
"type": [ {
"coding": [ {
"code": "einrichtungskontakt",
"display": "Einrichtungskontakt"
} ]
} ],
"subject": {
"reference": "Patient/UKE-0013"
},
"period": {
"start": "2020-06-13T00:00:00+02:00",
"end": "2020-06-18T00:00:00+02:00"
},
"diagnosis": [ {
"condition": {
"reference": "Condition/UKE-0013-E-1-D-1"
},
"use": {
"coding": [ {
"system": "http://terminology.hl7.org/CodeSystem/diagnosis-role",
"code": "AD",
"display": "Admission diagnosis"
} ]
}
} ]
},
"request": {
"method": "PUT",
"url": "Encounter/UKE-0013-E-1"
}
}, {
"fullUrl": "Encounter/UKE-0013-E-1-A-1",
"resource": {
"resourceType": "Encounter",
"id": "UKE-0013-E-1-A-1",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-fall/StructureDefinition/KontaktGesundheitseinrichtung" ]
},
"status": "finished",
"class": {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
},
"type": [ {
"coding": [ {
"code": "abteilungskontakt",
"display": "Abteilungskontakt"
} ]
} ],
"serviceType": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/dkgev/Fachabteilungsschluessel",
"code": "0300",
"display": "Kardiologie"
} ]
},
"subject": {
"reference": "Patient/UKE-0013"
},
"period": {
"start": "2020-06-13T00:00:00+02:00",
"end": "2020-06-18T00:00:00+02:00"
},
"diagnosis": [ {
"condition": {
"_reference": {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
}
}
} ],
"partOf": {
"reference": "Encounter/UKE-0013-E-1"
}
},
"request": {
"method": "PUT",
"url": "Encounter/UKE-0013-E-1-A-1"
}
}, {
"fullUrl": "Condition/UKE-0013-E-1-D-1",
"resource": {
"resourceType": "Condition",
"id": "UKE-0013-E-1-D-1",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-diagnose/StructureDefinition/Diagnose" ]
},
"identifier": [ {
"value": "UKE-0013-E-1-D-1"
} ],
"code": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/bfarm/icd-10-gm",
"version": "2020",
"code": "I25.22"
} ],
"text": "Myokardinfarkt in Vergangenheit"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"recordedDate": "2020-06-13T00:00:00+02:00"
},
"request": {
"method": "PUT",
"url": "Condition/UKE-0013-E-1-D-1"
}
}, {
"fullUrl": "Medication/Medication-154102257",
"resource": {
"resourceType": "Medication",
"id": "Medication-154102257",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/Medication" ]
},
"identifier": [ {
"value": "Medication-154102257"
} ],
"code": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/ifa/pzn",
"code": "05484267"
}, {
"system": "http://fhir.de/CodeSystem/bfarm/atc",
"code": "L03AA02"
} ],
"text": "Filgastrim"
},
"ingredient": [ {
"itemCodeableConcept": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/ask",
"_code": {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
}
} ]
},
"strength": {
"numerator": {
"value": 48,
"unit": "Mio I.E./0,5ml",
"system": "http://unitsofmeasure.org",
"code": "Mio I.E./0,5ml"
},
"denominator": {
"value": 1,
"system": "http://XXX",
"code": "Fertigspritzen"
}
}
} ]
},
"request": {
"method": "PUT",
"url": "Medication/Medication-154102257"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-1",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-1",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-154102257"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-13T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Fertigspritzen",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-1"
}
}, {
"fullUrl": "Medication/Medication--132842199",
"resource": {
"resourceType": "Medication",
"id": "Medication--132842199",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/Medication" ]
},
"identifier": [ {
"value": "Medication--132842199"
} ],
"code": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/ifa/pzn",
"code": "12636016"
}, {
"system": "http://fhir.de/CodeSystem/bfarm/atc",
"code": "B01AF01"
} ],
"text": "Rivaroxaban"
},
"ingredient": [ {
"itemCodeableConcept": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/ask",
"_code": {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
}
} ]
},
"strength": {
"numerator": {
"value": 10,
"unit": "milligram",
"system": "http://unitsofmeasure.org",
"code": "mg"
},
"denominator": {
"value": 1,
"system": "http://XXX",
"code": "Tabletten"
}
}
} ]
},
"request": {
"method": "PUT",
"url": "Medication/Medication--132842199"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-2",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-2",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication--132842199"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-13T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Tabletten",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-2"
}
}, {
"fullUrl": "Medication/Medication-1928216850",
"resource": {
"resourceType": "Medication",
"id": "Medication-1928216850",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/Medication" ]
},
"identifier": [ {
"value": "Medication-1928216850"
} ],
"code": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/ifa/pzn",
"code": "06882768"
}, {
"system": "http://fhir.de/CodeSystem/bfarm/atc",
"code": "N02BB02"
} ],
"text": "Metamizol natrium-1-Wasser"
},
"ingredient": [ {
"itemCodeableConcept": {
"coding": [ {
"system": "http://fhir.de/CodeSystem/ask",
"_code": {
"extension": [ {
"url": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
"valueCode": "unknown"
} ]
}
} ]
},
"strength": {
"numerator": {
"value": 1000,
"unit": "milligram",
"system": "http://unitsofmeasure.org",
"code": "mg"
},
"denominator": {
"value": 1,
"system": "http://XXX",
"code": "Injektionslösung"
}
}
} ]
},
"request": {
"method": "PUT",
"url": "Medication/Medication-1928216850"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-3",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-3",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-1928216850"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-13T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 4,
"unit": "Injektionslösung",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-3"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-4",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-4",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-154102257"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-14T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Fertigspritzen",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-4"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-5",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-5",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication--132842199"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-14T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Tabletten",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-5"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-6",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-6",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-1928216850"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-14T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 4,
"unit": "Injektionslösung",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-6"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-7",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-7",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-154102257"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-15T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Fertigspritzen",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-7"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-8",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-8",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication--132842199"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-15T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Tabletten",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-8"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-9",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-9",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-1928216850"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-15T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 4,
"unit": "Injektionslösung",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-9"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-10",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-10",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-154102257"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-16T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Fertigspritzen",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-10"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-11",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-11",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication--132842199"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-16T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Tabletten",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-11"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-12",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-12",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-1928216850"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-16T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 4,
"unit": "Injektionslösung",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-12"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-13",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-13",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-154102257"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-17T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Fertigspritzen",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-13"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-14",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-14",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication--132842199"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-17T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Tabletten",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-14"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-15",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-15",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-1928216850"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-17T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 4,
"unit": "Injektionslösung",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-15"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-16",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-16",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-154102257"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-18T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Fertigspritzen",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-16"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-17",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-17",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication--132842199"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-18T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 1,
"unit": "Tabletten",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-17"
}
}, {
"fullUrl": "MedicationStatement/UKE-0013-E-1-MS-18",
"resource": {
"resourceType": "MedicationStatement",
"id": "UKE-0013-E-1-MS-18",
"meta": {
"profile": [ "https://www.medizininformatik-initiative.de/fhir/core/modul-medikation/StructureDefinition/MedicationStatement" ]
},
"status": "active",
"medicationReference": {
"reference": "Medication/Medication-1928216850"
},
"subject": {
"reference": "Patient/UKE-0013"
},
"context": {
"reference": "Encounter/UKE-0013-E-1"
},
"effectiveDateTime": "2020-06-18T00:00:00+02:00",
"dosage": [ {
"doseAndRate": [ {
"doseQuantity": {
"value": 4,
"unit": "Injektionslösung",
"system": "http://unitsofmeasure.org",
"code": "1"
}
} ]
} ]
},
"request": {
"method": "PUT",
"url": "MedicationStatement/UKE-0013-E-1-MS-18"
}
} ]
}
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0014.json
-14480
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0015.json
-11505
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0016.json
-11729
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0017.json
-19022
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0018.json
-65261
View File
File diff suppressed because it is too large. Load diff
testData/POLAR/POLAR_Testdaten_Original_UKE-UKE-0019.json
-10508
View File
File diff suppressed because it is too large. Load diff
Loaded 100 of 137 files, more files were not shown because too many files have changed in this diff. Show more