vantage6-node#

Node class#

ServerContext class#

class ServerContext(instance_name, environment='prod', system_folders=True)#

Server context

Parameters:

  • instance_name (str) – Name of the configuration instance, corresponds to the filename of the configuration file.

  • environment (str, optional) – DTAP environment to load from the configuration file, by default S_ENV

  • system_folders (bool, optional) – System wide or user configuration, by default S_FOL

INST_CONFIG_MANAGER#

alias of ServerConfigurationManager

classmethod available_configurations(system_folders=True)#

Find all available server configurations in the default folders.

Parameters:

system_folders (bool, optional) – System wide or user configuration, by default S_FOL

Returns:

The first list contains validated configuration files, the second list contains invalid configuration files.

Return type:

Tuple[List, List]

classmethod config_exists(instance_name, environment='prod', system_folders=True)#

Check if a configuration file exists.

Parameters:

  • instance_name (str) – Name of the configuration instance, corresponds to the filename of the configuration file.

  • environment (str, optional) – DTAP environment that needs to be present, by default S_ENV

  • system_folders (bool, optional) – System wide or user configuration, by default S_FOL

Returns:

Whether the configuration file exists or not

Return type:

bool

property docker_container_name: str#

Unique name of the docker container.

Returns:

Unique docker container name

Return type:

str

classmethod from_external_config_file(path, environment='prod', system_folders=True)#

Create a server context from an external configuration file. External means that the configuration file is not located in the default folders but its location is specified by the user.

Parameters:

  • path (str) – Path of the configuration file

  • environment (str, optional) – DTAP environment to be loaded, by default S_ENV

  • system_folders (bool, optional) – System wide or user configuration, by default S_FOL

Returns:

Server context object

Return type:

ServerContext

get_database_uri()#

Obtain the database uri from the environment or the configuration. The VANTAGE6_DB_URI environment variable is used by the Docker container, but can also be set by the user.

Returns:

string representation of the database uri

Return type:

str

NodeContext class#

class NodeContext(instance_name, environment='application', system_folders=False, config_file=None)#

Node context

See DockerNodeContext for the node instance mounts when running as a dockerized service.

Parameters:

  • instance_name (str) – Name of the configuration instance, corresponds to the filename of the configuration file.

  • environment (str, optional) – DTAP environment to be loaded, by default N_ENV

  • system_folders (bool, optional) – _description_, by default N_FOL

  • config_file (str, optional) – _description_, by default None

INST_CONFIG_MANAGER#

alias of NodeConfigurationManager

classmethod available_configurations(system_folders=False)#

Find all available server configurations in the default folders.

Parameters:

system_folders (bool, optional) – System wide or user configuration, by default N_FOL

Returns:

The first list contains validated configuration files, the second list contains invalid configuration files.

Return type:

Tuple[List, List]

classmethod config_exists(instance_name, environment='application', system_folders=False)#

Check if a configuration file exists.

Parameters:

  • instance_name (str) – Name of the configuration instance, corresponds to the filename of the configuration file.

  • environment (str, optional) – DTAP environment that needs to be present, by default N_ENV

  • system_folders (bool, optional) – System wide or user configuration, by default N_FOL

Returns:

Whether the configuration file exists or not

Return type:

bool

property databases#

Dictionary of local databases that are available for this node.

Returns:

dictionary with database names as keys and their corresponding paths as values.

Return type:

dict

property docker_container_name: str#

Unique Docker container name of the node.

Returns:

Unique Docker container name

Return type:

str

property docker_network_name: str#

Private Docker network name which is unique for this node.

Returns:

Docker network name

Return type:

str

docker_temporary_volume_name(run_id)#

Docker volume in which temporary data is stored. Temporary data is linked to a specific run. Multiple algorithm containers can have the same run id, and therefore the share same temporary volume.

Parameters:

run_id (int) – run id provided by the server

Returns:

Docker volume name

Return type:

str

property docker_volume_name: str#

Docker volume in which task data is stored. In case a file based database is used, this volume contains the database file as well.

Returns:

Docker volume name

Return type:

str

property docker_vpn_volume_name: str#

Docker volume in which the VPN configuration is stored.

Returns:

Docker volume name

Return type:

str

classmethod from_external_config_file(path, environment='application', system_folders=False)#

Create a node context from an external configuration file. External means that the configuration file is not located in the default folders but its location is specified by the user.

Parameters:

  • path (str) – Path of the configuration file

  • environment (str, optional) – DTAP environment to be loaded, by default N_ENV

  • system_folders (bool, optional) – System wide or user configuration, by default N_FOL

Returns:

Node context object

Return type:

NodeContext

get_database_uri(label='default')#

Obtain the database URI for a specific database.

Parameters:

label (str, optional) – Database label, by default “default”

Returns:

URI to the database

Return type:

str

static type_data_folder(system_folders=False)#

Obtain OS specific data folder where to store node specific data.

Parameters:

system_folders (bool, optional) – System wide or user configuration, by default N_FOL

Returns:

Path to the data folder

Return type:

Path

DockerNodeContext class#

class DockerNodeContext(*args, **kwargs)#

Node context for the dockerized version of the node.

static instance_folders(instance_type, instance_name, system_folders)#

Log, data and config folders are always mounted. The node manager should take care of this.

set_folders(instance_type, instance_name, system_folders)#

In case of the dockerized version we do not want to use user specified directories within the container.

DockerBaseManager class#

class DockerBaseManager(isolated_network_mgr)#

Base class for docker-using classes. Contains simple methods that are used by multiple derived classes

get_isolated_netw_ip(container)#

Get address of a container in the isolated network

Parameters:

container (Container) – Docker container whose IP address should be obtained

Returns:

IP address of a container in isolated network

Return type:

str

DockerManager class#

class DockerManager(ctx, isolated_network_mgr, vpn_manager, tasks_dir)#

Wrapper for the docker-py module.

This classes manages tasks related to Docker, such as logging in to docker registries, managing input/output files, logs etc. Results can be retrieved through get_result() which returns the first available algorithm result.

cleanup()#

Stop all active tasks and delete the isolated network

Note: the temporary docker volumes are kept as they may still be used by a master container

Return type:

None

cleanup_tasks()#

Stop all active tasks

Returns:

List of information on tasks that have been killed

Return type:

List[KilledResult]

create_volume(volume_name)#

Create a temporary volume for a single run.

A single run can consist of multiple algorithm containers. It is important to note that all algorithm containers having the same run_id have access to this container.

Parameters:

volume_name (str) – Name of the volume to be created

Return type:

None

get_result()#

Returns the oldest (FIFO) finished docker container.

This is a blocking method until a finished container shows up. Once the container is obtained and the results are read, the container is removed from the docker environment.

Returns:

result of the docker image

Return type:

Result

is_docker_image_allowed(docker_image_name)#

Checks the docker image name.

Against a list of regular expressions as defined in the configuration file. If no expressions are defined, all docker images are accepted.

Parameters:

docker_image_name (str) – uri to the docker image

Returns:

Whether docker image is allowed or not

Return type:

bool

is_running(result_id)#

Check if a container is already running for <result_id>.

Parameters:

result_id (int) – result_id of the algorithm container to be found

Returns:

Whether or not algorithm container is running already

Return type:

bool

kill_selected_tasks(org_id, kill_list=None)#

Kill tasks specified by a kill list, if they are currently running on this node

Parameters:

  • org_id (int) – The organization id of this node

  • kill_list (List[ToBeKilled]) – A list of info about tasks that should be killed.

Returns:

List with information on killed tasks

Return type:

List[KilledResult]

kill_tasks(org_id, kill_list=None)#

Kill tasks currently running on this node.

Parameters:

  • org_id (int) – The organization id of this node

  • kill_list (List[ToBeKilled] (optional)) – A list of info on tasks that should be killed. If the list is not specified, all running algorithm containers will be killed.

Returns:

List of dictionaries with information on killed tasks

Return type:

List[KilledResult]

Link a docker container to the isolated docker network

Parameters:

  • container_name (str) – Name of the docker container to be linked to the network

  • config_alias (str) – Alias of the docker container defined in the config file

Return type:

None

login_to_registries(registries=[])#

Login to the docker registries

Parameters:

registries (list) – list of registries to login to

Return type:

None

run(result_id, task_info, image, docker_input, tmp_vol_name, token, database)#

Checks if docker task is running. If not, creates DockerTaskManager to run the task

Parameters:

  • result_id (int) – Server result identifier

  • task_info (Dict) – Dictionary with task information

  • image (str) – Docker image name

  • docker_input (bytes) – Input that can be read by docker container

  • tmp_vol_name (str) – Name of temporary docker volume assigned to the algorithm

  • token (str) – Bearer token that the container can use

  • database (str) – Name of the Database to use

Returns:

Description of each port on the VPN client that forwards traffic to the algo container. None if VPN is not set up.

Return type:

List[Dict] or None

class Result(result_id: int, task_id: int, logs: str, data: str, status: str, parent_id: Optional[int])#

Data class to store the result of the docker image.

result_id#

ID of the current algorithm run

Type:

int

logs#

Logs attached to current algorithm run

Type:

str

data#

Output data of the algorithm

Type:

str

status_code#

Status code of the algorithm run

Type:

int

property data#

Alias for field number 3

property logs#

Alias for field number 2

property parent_id#

Alias for field number 5

property result_id#

Alias for field number 0

property status#

Alias for field number 4

property task_id#

Alias for field number 1

VPNManager class#

class VPNManager(isolated_network_mgr, node_name, vpn_volume_name, vpn_subnet, alpine_image=None, vpn_client_image=None, network_config_image=None)#

Setup a VPN client in a Docker container and configure the network so that the VPN container can forward traffic to and from algorithm containers.

connect_vpn()#

Start VPN client container and configure network to allow algorithm-to-algoritm communication

Return type:

None

exit_vpn(cleanup_host_rules=True)#

Gracefully shutdown the VPN and clean up

Parameters:

cleanup_host_rules (bool, optional) – Whether or not to clear host configuration rules. Should be True if they have been created at the time this function runs.

Return type:

None

forward_vpn_traffic(helper_container, algo_image_name)#

Setup rules so that traffic is properly forwarded between the VPN container and the algorithm container (and its helper container)

Parameters:

  • algo_helper_container (Container) – Helper algorithm container

  • algo_image_name (str) – Name of algorithm image that is run

Returns:

Description of each port on the VPN client that forwards traffic to the algo container. None if VPN is not set up.

Return type:

List[Dict] or None

get_vpn_ip()#

Get VPN IP address in VPN server namespace

Returns:

IP address assigned to VPN client container by VPN server

Return type:

str

has_connection()#

Return True if VPN connection is active

Return type:

bool

is_isolated_interface(ip_interface, vpn_ip_isolated_netw)#

Return True if a network interface is the isolated network interface. Identify this based on the IP address of the VPN client in the isolated network

Parameters:

  • ip_interface (dict) – IP interface obtained by executing ip –json addr command

  • vpn_ip_isolated_netw (str) – IP address of VPN container in isolated network

Returns:

True if this is the interface describing the isolated network

Return type:

boolean