Inheritance diagram for Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf:

Public Member Functions
public	default_options ()

public	pipeline_create_commands ()

public	pipeline_wide_parameters ()

public	resource_classes ()

public	pipeline_analyses ()

public	beekeeper_extra_cmdline_options ()

public	hive_meta_table ()

public	pre_options ()

public	dbconn_2_url ()

public	pipeline_url ()

public	db_cmd ()

public	print_debug ()

public	process_pipeline_name ()

public	default_pipeline_name ()

public	process_options ()

public	overridable_pipeline_create_commands ()

public	is_analysis_topup ()

public	run_pipeline_create_commands ()

public	add_objects_from_config ()

public	useful_commands_legend ()

Public Member Functions inherited from Bio::EnsEMBL::Hive::DependentOptions
public	new ()

public	use_cases ()

public	load_cmdline_options ()

public	root ()

public	is_fully_substituted_string ()

public	is_fully_substituted_structure ()

public	hash_leaves ()

public	o ()

public	substitute ()

public	merge_from_rules ()

public	process_options ()

Detailed Description

Synopsis

    # Example 1: specifying only the mandatory option:
init_pipeline.pl Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf -password <mypass>
    # Example 2: specifying the mandatory options as well as overriding some defaults:
init_pipeline.pl Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf -host <myhost> -dbname <mydbname> -password <mypass>

Description

    Generic configuration module for all Hive pipelines with loader functionality.
    All other Hive PipeConfig modules should inherit from this module and will probably need to redefine some or all of the following interface methods:

          default_options:                  returns a hash of (possibly multilevel) defaults for the options on which depend the rest of the configuration

          pipeline_create_commands:         returns a list of strings that will be executed as system commands needed to create and set up the pipeline database

          pipeline_wide_parameters:         returns a hash of pipeline-wide parameter names and their values

          resource_classes:                 returns a hash of resource class definitions

          pipeline_analyses:                returns a list of hash structures that define analysis objects bundled with definitions of corresponding jobs, rules and resources

          beekeeper_extra_cmdline_options   returns a string with command line options that you want to be passed to the beekeeper.pl

    When defining anything except the keys of default_options() a call to $self->o('myoption') can be used.
    This call means "substitute this call for the value of 'myoption' at the time of configuring the pipeline".
    All option names mentioned in $self->o() calls within the five interface methods above can be given non-default values from the command line.

    Please make sure you have studied the pipeline configuraton examples in Bio::EnsEMBL::Hive::PipeConfig before creating your own PipeConfig modules.

Definition at line 54 of file HiveGeneric_conf.pm.

Member Function Documentation

◆ add_objects_from_config()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::add_objects_from_config ( )

    Description : The method that uses the Hive/EnsEMBL API to actually create all the analyses, jobs, dataflow and control rules and resource descriptions.

    Caller      : init_pipeline.pl or any other script that will drive this module.

Code:

click to view

sub add_objects_from_config {
    my $self        = shift @_;
    my $pipeline    = shift @_;
    $self->print_debug( "Adding hive_meta table entries ...\n" );
    my $new_meta_entries = $self->hive_meta_table();
    while( my ($meta_key, $meta_value) = each %$new_meta_entries ) {
        $pipeline->add_new_or_update( 'MetaParameters', $self->o('hive_debug_init'),
            'meta_key'      => $meta_key,
            'meta_value'    => $meta_value,
        );
    }
    $self->print_debug( "Done.\n\n" );
    $self->print_debug( "Adding pipeline-wide parameters ...\n" );
    my $new_pwp_entries = $self->pipeline_wide_parameters();
    while( my ($param_name, $param_value) = each %$new_pwp_entries ) {
        $pipeline->add_new_or_update( 'PipelineWideParameters', $self->o('hive_debug_init'),
            'param_name'    => $param_name,
            'param_value'   => stringify($param_value),
        );
    }
    $self->print_debug( "Done.\n\n" );
    $self->print_debug( "Adding Resources ...\n" );
    my $resource_classes_hash = $self->resource_classes;
    unless( exists $resource_classes_hash->{'default'} ) {
        warn "\tNB:'default' resource class is not in the database (did you forget to inherit from SUPER::resource_classes ?) - creating it for you\n";
        $resource_classes_hash->{'default'} = {};
    }
    my @resource_classes_order = sort { ($b eq 'default') or -($a eq 'default') or ($a cmp $b) } keys %$resource_classes_hash; # put 'default' to the front
    my %cached_resource_classes = map {$_->name => $_} $pipeline->collection_of('ResourceClass')->list();
    foreach my $rc_name (@resource_classes_order) {
        if($rc_name=~/^\d+$/) {
            die "-rc_id syntax is no longer supported, please use the new resource notation (-rc_name)";
        }
        my ($resource_class) = $pipeline->add_new_or_update( 'ResourceClass',   # NB: add_new_or_update returns a list
            'name'  => $rc_name,
        );
        $cached_resource_classes{$rc_name} = $resource_class;
        while( my($meadow_type, $resource_param_list) = each %{ $resource_classes_hash->{$rc_name} } ) {
            $resource_param_list = [ $resource_param_list ] unless(ref($resource_param_list));  # expecting either a scalar or a 2-element array
            my ($resource_description) = $pipeline->add_new_or_update( 'ResourceDescription', $self->o('hive_debug_init'),   # NB: add_new_or_update returns a list
                'resource_class'        => $resource_class,
                'meadow_type'           => $meadow_type,
                'submission_cmd_args'   => $resource_param_list->[0],
                'worker_cmd_args'       => $resource_param_list->[1],
            );
        }
    }
    $self->print_debug( "Done.\n\n" );
    my $amh = Bio::EnsEMBL::Hive::Valley->new()->available_meadow_hash();
    my %seen_logic_name = ();
    my %analyses_by_logic_name = map {$_->logic_name => $_} $pipeline->collection_of('Analysis')->list();
    $self->print_debug( "Adding Analyses ...\n" );
    foreach my $aha (@{$self->pipeline_analyses}) {
        my %aha_copy = %$aha;
        my ($logic_name, $module, $parameters_hash, $comment, $tags, $input_ids, $blocked, $batch_size, $hive_capacity, $failed_job_tolerance,
                $max_retry_count, $can_be_empty, $rc_id, $rc_name, $priority, $meadow_type, $analysis_capacity, $language, $wait_for, $flow_into)
         = delete @aha_copy{qw(-logic_name -module -parameters -comment -tags -input_ids -blocked -batch_size -hive_capacity -failed_job_tolerance
                 -max_retry_count -can_be_empty -rc_id -rc_name -priority -meadow_type -analysis_capacity -language -wait_for -flow_into)};   # slicing a hash reference
         my @unparsed_attribs = keys %aha_copy;
         if(@unparsed_attribs) {
             die "Could not parse the following analysis attributes: ".join(', ',@unparsed_attribs);
         }
        if( not $logic_name ) {
            die "'-logic_name' must be defined in every analysis";
        } elsif( $logic_name =~ /[+\-\%\.,]/ ) {
            die "Characters + - % . , are no longer allowed to be a part of an Analysis name. Please rename Analysis '$logic_name' and try again.\n";
        } elsif( looks_like_number($logic_name) ) {
            die "Numeric Analysis names are not allowed because they may clash with dbIDs. Please rename Analysis '$logic_name' and try again.\n";
        }
        if($seen_logic_name{$logic_name}++) {
            die "an entry with -logic_name '$logic_name' appears at least twice in the same configuration file, probably a typo";
        }
        if($rc_id) {
            die "(-rc_id => $rc_id) syntax is deprecated, please use (-rc_name => 'your_resource_class_name')";
        }
        my $analysis = $analyses_by_logic_name{$logic_name};  # the analysis with this logic_name may have already been stored in the db
        my $stats;
        if( $analysis ) {
            warn "Skipping creation of already existing analysis '$logic_name'.\n";
            next;
        } else {
            $rc_name ||= 'default';
            my $resource_class = $cached_resource_classes{$rc_name}
                or die "Could not find local resource with name '$rc_name', please check that resource_classes() method of your PipeConfig either contains or inherits it from the parent class";
            if ($meadow_type and not exists $amh->{$meadow_type}) {
                warn "The meadow '$meadow_type' is currently not registered (analysis '$logic_name')\n";
            }
            $parameters_hash ||= {};    # in case nothing was given
            die "'-parameters' has to be a hash" unless(ref($parameters_hash) eq 'HASH');
            ($analysis) = $pipeline->add_new_or_update( 'Analysis', $self->o('hive_debug_init'),   # NB: add_new_or_update returns a list
                'logic_name'            => $logic_name,
                'module'                => $module,
                'language'              => $language,
                'parameters'            => $parameters_hash,
                'comment'               => $comment,
                'tags'                  => ( (ref($tags) eq 'ARRAY') ? join(',', @$tags) : $tags ),
                'resource_class'        => $resource_class,
                'failed_job_tolerance'  => $failed_job_tolerance,
                'max_retry_count'       => $max_retry_count,
                'can_be_empty'          => $can_be_empty,
                'priority'              => $priority,
                'meadow_type'           => $meadow_type,
                'analysis_capacity'     => $analysis_capacity,
                'hive_capacity'         => $hive_capacity,
                'batch_size'            => $batch_size,
            );
            $analysis->get_compiled_module_name();  # check if it compiles and is named correctly
            ($stats) = $pipeline->add_new_or_update( 'AnalysisStats', $self->o('hive_debug_init'),   # NB: add_new_or_update returns a list
                'analysis'              => $analysis,
                'status'                => $blocked ? 'BLOCKED' : 'EMPTY',  # be careful, as this "soft" way of blocking may be accidentally unblocked by deep sync
                'total_job_count'       => 0,
                'semaphored_job_count'  => 0,
                'ready_job_count'       => 0,
                'done_job_count'        => 0,
                'failed_job_count'      => 0,
                'num_running_workers'   => 0,
                'sync_lock'             => 0,
            );
        }
            # Keep a link to the analysis object to speed up the creation of control and dataflow rules
        $analyses_by_logic_name{$logic_name} = $analysis;
            # now create the corresponding jobs (if there are any):
        if($input_ids) {
            push @{ $analysis->jobs_collection }, map { Bio::EnsEMBL::Hive::AnalysisJob->new(
                'prev_job'      => undef,           # these jobs are created by the initialization script, not by another job
                'analysis'      => $analysis,
                'input_id'      => $_,              # input_ids are now centrally stringified in the AnalysisJob itself
            ) } @$input_ids;
            unless( $pipeline->hive_use_triggers() ) {
                $stats->recalculate_from_job_counts( { 'READY' => scalar(@$input_ids) } );
            }
        }
    }
    $self->print_debug( "Done.\n\n" );
    $self->print_debug( "Adding Control and Dataflow Rules ...\n" );
    foreach my $aha (@{$self->pipeline_analyses}) {
        my ($logic_name, $wait_for, $flow_into)
             = @{$aha}{qw(-logic_name -wait_for -flow_into)};   # slicing a hash reference
        my $analysis = $analyses_by_logic_name{$logic_name};
        if($wait_for) {
            Bio::EnsEMBL::Hive::Utils::PCL::parse_wait_for($pipeline, $analysis, $wait_for, $self->o('hive_debug_init'));
        }
        if($flow_into) {
            Bio::EnsEMBL::Hive::Utils::PCL::parse_flow_into($pipeline, $analysis, $flow_into, $self->o('hive_debug_init'));
        }
    }
    $self->print_debug( "Done.\n\n" );
    # Block the analyses that should be blocked
    $self->print_debug( "Blocking the analyses that should be ...\n" );
    foreach my $stats ($pipeline->collection_of('AnalysisStats')->list()) {
        $stats->check_blocking_control_rules('no_die');
        $stats->determine_status();
    }
    $self->print_debug( "Done.\n\n" );
}

◆ beekeeper_extra_cmdline_options()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::beekeeper_extra_cmdline_options ( )

    Description : Interface method that should return a string with extra parameters that you want to be passed to beekeeper.pl

Code:

click to view

◆ db_cmd()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::db_cmd ( )

    Description :  Returns a db_cmd.pl-based command line that should execute by any supported driver (mysql/pgsql/sqlite)

Code:

click to view

◆ dbconn_2_url()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::dbconn_2_url ( )

    Description :  A convenience method used to stringify a connection-parameters hash into a 'pipeline_url' that beekeeper.pl will undestand

Code:

click to view

◆ default_options()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::default_options ( )

    Description : Interface method that should return a hash of option_name->default_option_value pairs.
                  Please see existing PipeConfig modules for examples.

Code:

click to view

sub default_options {
    my ($self) = @_;
    return {
        'hive_root_dir'         => $ENV{'EHIVE_ROOT_DIR'},                                      # this value is set up automatically if this code is run by init_pipeline.pl
        'hive_driver'           => 'mysql',
        'host'                  => $ENV{'EHIVE_HOST'} || 'localhost',                           # BEWARE that 'localhost' for mysql driver usually means a UNIX socket, not a TCPIP socket!
                                                                                                # If you need to connect to TCPIP socket, set  -host => '127.0.0.1' instead.
        'port'                  => $ENV{'EHIVE_PORT'},                                          # or remain undef, which means default for the driver
        'user'                  => $ENV{'EHIVE_USER'} // $self->o('user'),
        'password'              => $ENV{'EHIVE_PASS'} // $self->o('password'),                  # people will have to make an effort NOT to insert it into config files like .bashrc etc
        'dbowner'               => $ENV{'EHIVE_USER'} || whoami() || $self->o('dbowner'),       # although it is very unlikely that the current user has no name
        'hive_use_triggers'                 => 0,       # there have been a few cases of big pipelines misbehaving with triggers on, let's keep the default off.
        'hive_use_param_stack'              => 0,       # do not reconstruct the calling stack of parameters by default (yet)
        'hive_auto_rebalance_semaphores'    => 0,       # do not attempt to rebalance semaphores periodically by default
        'hive_default_max_retry_count'      => 3,       # default value for the max_retry_count parameter of each analysis
        'hive_force_init'                   => 0,       # setting it to 1 will drop the database prior to creation (use with care!)
        'hive_no_init'                      => 0,       # setting it to 1 will skip pipeline_create_commands (useful for topping up)
        'hive_debug_init'                   => 0,       # setting it to 1 will make init_pipeline.pl tell everything it's doing
        'pipeline_name'                     => $self->default_pipeline_name(),
        'pipeline_db'   => {
            -driver => $self->o('hive_driver'),
            -host   => $self->o('host'),
            -port   => $self->o('port'),
            -user   => $self->o('user'),
            -pass   => $self->o('password'),
            -dbname => $self->o('dbowner').'_'.$self->o('pipeline_name'),
        },
    };
}

◆ default_pipeline_name()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::default_pipeline_name ( )

Undocumented method

Code:

click to view

◆ hive_meta_table()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::hive_meta_table ( )

Undocumented method

Code:

click to view

◆ is_analysis_topup()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::is_analysis_topup ( )

Undocumented method

Code:

click to view

◆ overridable_pipeline_create_commands()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::overridable_pipeline_create_commands ( )

Undocumented method

Code:

click to view

◆ pipeline_analyses()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::pipeline_analyses ( )

    Description : Interface method that should return a list of hashes that define analysis bundled with corresponding jobs, dataflow and analysis_ctrl rules and resource_id.
                  Please see existing PipeConfig modules for examples.

Code:

click to view

◆ pipeline_create_commands()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::pipeline_create_commands ( )

    Description : Interface method that should return a list of command lines to be run in order to create and set up the pipeline database.
                  Please see existing PipeConfig modules for examples.

Code:

click to view

◆ pipeline_url()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::pipeline_url ( )

Undocumented method

Code:

click to view

◆ pipeline_wide_parameters()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::pipeline_wide_parameters ( )

    Description : Interface method that should return a hash of pipeline_wide_parameter_name->pipeline_wide_parameter_value pairs.
                  The value doesn't have to be a scalar, can be any Perl structure now (will be stringified and de-stringified automagically).
                  Please see existing PipeConfig modules for examples.

Code:

click to view

◆ pre_options()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::pre_options ( )

Undocumented method

Code:

click to view

◆ print_debug()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::print_debug ( )

Undocumented method

Code:

click to view

◆ process_options()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::process_options ( )

    Description : The method that does all the parameter parsing magic.
                  It is two-pass through the interface methods: first pass collects the options, second is intelligent substitution.

    Caller      : init_pipeline.pl or any other script that will drive this module.

    Note        : You can override parsing the command line bit by providing a hash as the argument to this method.
                  This hash should contain definitions of all the parameters you would otherwise be providing from the command line.
                  Useful if you are creating batches of hive pipelines using a script.

Code:

click to view

◆ process_pipeline_name()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::process_pipeline_name ( )

Undocumented method

Code:

click to view

◆ resource_classes()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::resource_classes ( )

    Description : Interface method that should return a hash of resource_description_id->resource_description_hash.
                  Please see existing PipeConfig modules for examples.

Code:

click to view

◆ run_pipeline_create_commands()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::run_pipeline_create_commands ( )

Undocumented method

Code:

click to view

◆ useful_commands_legend()

public Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf::useful_commands_legend ( )

Undocumented method

Code:

click to view

sub useful_commands_legend {
    my $self  = shift @_;
    my $pipeline_url = $self->pipeline_url();
    unless ($pipeline_url =~ /^[\'\"]/) {
        $pipeline_url   = '"' . $pipeline_url . '"';
    }
    my $pipeline_name   = $self->o('pipeline_name');
    my $extra_cmdline   = $self->beekeeper_extra_cmdline_options();
    my @output_lines = (
        '','',
        '# ' . '-' x 22 . '[Useful commands]' . '-' x 22,
        '',
        " # It is convenient to store the pipeline url in a variable:",
        "\texport EHIVE_URL=$pipeline_url\t\t\t# bash version",
        "(OR)",
        "\tsetenv EHIVE_URL $pipeline_url\t\t\t# [t]csh version",
        '',
        " # Add a new job to the pipeline (usually done once before running, but pipeline can be \"topped-up\" at any time) :",
        "\tseed_pipeline.pl -url $pipeline_url -logic_name <analysis_name> -input_id <param_hash>",
        '',
        " # At any moment during or after execution you can request a pipeline diagram in an image file (desired format is set via extension) :",
        "\tgenerate_graph.pl -url $pipeline_url -out $pipeline_name.png",
        '',
        " # Synchronize the Hive (to display fresh statistics about all analyses):",
        "\tbeekeeper.pl -url $pipeline_url -sync",
        '',
        " # Depending on the Meadow the pipeline is running on, you may be able to collect actual resource usage statistics :",
        "\tload_resource_usage.pl -url $pipeline_url",
        '',
        " # After having run load_resource_usage.pl, you can request a resource usage timeline in an image file (desired format is set via extension) :",
        "\tgenerate_timeline.pl -url $pipeline_url -out timeline_$pipeline_name.png",
        '',
        " # Peek into your pipeline database with a database client (useful to have open while the pipeline is running) :",
        "\tdb_cmd.pl -url $pipeline_url",
        '',
        " # Run the pipeline (can be interrupted and restarted) :",
        "\tbeekeeper.pl -url $pipeline_url $extra_cmdline -loop\t\t# run in looped automatic mode (a scheduling step performed every minute)",
        "(OR)",
        "\tbeekeeper.pl -url $pipeline_url $extra_cmdline -run \t\t# run one scheduling step of the pipeline and exit (useful for debugging/learning)",
        "(OR)",
        "\trunWorker.pl -url $pipeline_url $extra_cmdline      \t\t# run exactly one Worker locally (useful for debugging/learning)",
        '',
    );
    return join("\n", @output_lines);
}

The documentation for this class was generated from the following file:

modules/Bio/EnsEMBL/Hive/PipeConfig/HiveGeneric_conf.pm

Public Member Functions

Detailed Description

Synopsis

Description

Member Function Documentation

◆ add_objects_from_config()

◆ beekeeper_extra_cmdline_options()

◆ db_cmd()

◆ dbconn_2_url()

◆ default_options()

◆ default_pipeline_name()

◆ hive_meta_table()

◆ is_analysis_topup()

◆ overridable_pipeline_create_commands()

◆ pipeline_analyses()

◆ pipeline_create_commands()

◆ pipeline_url()

◆ pipeline_wide_parameters()

◆ pre_options()

◆ print_debug()

◆ process_options()

◆ process_pipeline_name()

◆ resource_classes()

◆ run_pipeline_create_commands()

◆ useful_commands_legend()