Inheritance diagram for Bio::EnsEMBL::Hive::Meadow::SLURM:

Public Member Functions
public	name ()

protected	_init_meadow ()

public	count_pending_workers_by_rc_name ()

public	count_running_workers ()

public	status_of_all_our_workers ()

public	check_worker_is_alive_and_mine ()

public	kill_worker ()

public	get_report_entries_for_process_ids ()

public	get_report_entries_for_time_interval ()

public	parse_report_source_line ()

public	submit_workers_return_meadow_pids ()

public	execute_command ()

public	get_current_worker_process_id ()

public	deregister_local_process ()

public	get_cause_of_death ()

Public Member Functions inherited from Bio::EnsEMBL::Hive::Meadow
public	get_meadow_major_version ()

public	get_meadow_version ()

public	check_version_compatibility ()

public	new ()

protected	_init_meadow ()

public	cached_name ()

public	type ()

public	get_current_hostname ()

public	signature ()

public	pipeline_name ()

public	runWorker_path ()

public	job_name_prefix ()

public	job_array_common_name ()

public	name ()

public	get_current_worker_process_id ()

public	status_of_all_our_workers ()

public	check_worker_is_alive_and_mine ()

public	kill_worker ()

public	parse_report_source_line ()

public	get_report_entries_for_process_ids ()

public	get_report_entries_for_time_interval ()

public	submit_workers_return_meadow_pids ()

public	run_on_host ()

public	cleanup_temp_directory ()

Public Member Functions inherited from Bio::EnsEMBL::Hive::Configurable
public	config ()

public	context ()

public	config_get ()

public	config_set ()

Detailed Description

Description

    This is the 'SLURM' implementation of an EnsEMBL eHive Meadow.

Definition at line 17 of file SLURM.pm.

Member Function Documentation

◆ _init_meadow()

protected Bio::EnsEMBL::Hive::Meadow::SLURM::_init_meadow ( )

Undocumented method

Code:

click to view

◆ check_worker_is_alive_and_mine()

public Bio::EnsEMBL::Hive::Meadow::SLURM::check_worker_is_alive_and_mine ( )

Undocumented method

Code:

click to view

◆ count_pending_workers_by_rc_name()

public Bio::EnsEMBL::Hive::Meadow::SLURM::count_pending_workers_by_rc_name ( )

Undocumented method

Code:

click to view

◆ count_running_workers()

public Bio::EnsEMBL::Hive::Meadow::SLURM::count_running_workers ( )

Undocumented method

Code:

click to view

◆ deregister_local_process()

public Bio::EnsEMBL::Hive::Meadow::SLURM::deregister_local_process ( )

Undocumented method

Code:

click to view

◆ execute_command()

public Bio::EnsEMBL::Hive::Meadow::SLURM::execute_command ( )

Undocumented method

Code:

click to view

◆ get_cause_of_death()

public Bio::EnsEMBL::Hive::Meadow::SLURM::get_cause_of_death ( )

Undocumented method

Code:

click to view

◆ get_current_worker_process_id()

public Bio::EnsEMBL::Hive::Meadow::SLURM::get_current_worker_process_id ( )

Undocumented method

Code:

click to view

◆ get_report_entries_for_process_ids()

public Bio::EnsEMBL::Hive::Meadow::SLURM::get_report_entries_for_process_ids ( )

Undocumented method

Code:

click to view

◆ get_report_entries_for_time_interval()

public Bio::EnsEMBL::Hive::Meadow::SLURM::get_report_entries_for_time_interval ( )

Undocumented method

Code:

click to view

◆ kill_worker()

public Bio::EnsEMBL::Hive::Meadow::SLURM::kill_worker ( )

Undocumented method

Code:

click to view

◆ name()

public Bio::EnsEMBL::Hive::Meadow::SLURM::name ( )

Undocumented method

Code:

click to view

◆ parse_report_source_line()

public Bio::EnsEMBL::Hive::Meadow::SLURM::parse_report_source_line ( )

Undocumented method

Code:

click to view

sub parse_report_source_line {
    my ($self, $bacct_source_line) = @_;
 
    my $jnp = $self->job_name_prefix();    # reederj_ehive_rosaprd_278-Hive-
 
    my @lines = execute_command($bacct_source_line);
 
    my %report_entry = ();
    my %job_id_to_state;
 
    for my $row (@lines) {
        my @col = split(/\|/, $row);
 
        my $job_name = $col[0];     # JobName - for explanation please look at sacct command
        my $job_id = $col[1];       # JobID
        my $exit_code = $col[2];    # ExitCode
        my $mem_used = $col[3];     # MaxRSS in units=M
                                    #my $reserved_time = $col[4];   # Reserved / pending time ...
        my $reserved_time = 0;
        my $max_disk_read = $col[4];       # MaxDiskRead
        my $total_cpu = $col[5];           # CPUTimeRAW
        my $elapsed = $col[6];             # ElapsedRAW
        my $state = $col[7];               # State
        my $exception_status = $col[8];    # DerivedExitCode
        my $endtime = $col[9];
 
        my $datetime = DateTime::Format::ISO8601->parse_datetime($endtime);
 
        # print "DEBUG: $job_id\t$state\n";
 
        # parse the correct state
        if ($job_name =~ m/$jnp/) {
            $job_id_to_state{$job_id} = $state;
        }
 
        next unless $job_name =~ m/batch/;
        $job_id =~ s/\.batch//;
 
        $mem_used      =~ s/M$//;    # results are reported in Megabytes
        $max_disk_read =~ s/M$//;    # results are reported in Megabytes
 
        if ($reserved_time =~ /:/) {
            my $reserved_timepiece = Time::Piece->strptime($reserved_time, '%H:%M:%S');
            $reserved_time =
                $reserved_timepiece->hour*3600 +
                $reserved_timepiece->minute*60 +
                $reserved_timepiece->second;
        }
 
        # get previously parsed status ( slurm returns 3 rows of statuses which are different ! )
        my $cause_of_death = get_cause_of_death($job_id_to_state{$job_id});
        $exception_status = $cause_of_death;
 
        $report_entry{$job_id} = {
            # entries for 'worker' table:
            'when_died'      => $datetime->date . ' ' . $datetime->hms,
            'cause_of_death' => $cause_of_death,
 
            # entries for 'worker_resource_usage' table:
            'exit_status'      => $exit_code,
            'exception_status' => $exception_status,
            'mem_megs'         => $mem_used,         # mem_in_units, returnd by sacct with --units=M
            'swap_megs'        => $max_disk_read,    # swap_in_units
            'pending_sec'      => $reserved_time || 0,
            'cpu_sec'          => $total_cpu,
            'lifespan_sec'     => $elapsed,
        };
    } ## end for my $row (@$lines)
    return \%report_entry;
} ## end sub parse_report_source_line

◆ status_of_all_our_workers()

public Bio::EnsEMBL::Hive::Meadow::SLURM::status_of_all_our_workers ( )

Undocumented method

Code:

click to view

◆ submit_workers_return_meadow_pids()

public Bio::EnsEMBL::Hive::Meadow::SLURM::submit_workers_return_meadow_pids ( )

Undocumented method

Code:

click to view

sub submit_workers_return_meadow_pids {
    my ($self, $worker_cmd, $required_worker_count, $iteration, $rc_name,
        $rc_specific_submission_cmd_args, $submit_log_subdir) = @_;
 
    my $job_array_common_name = $self->job_array_common_name($rc_name, $iteration);
 
    # Flag if we should submit a job array or not
    my $array_required = $required_worker_count > 1;
 
    my $job_array_spec = "1-${required_worker_count}";
    my $meadow_specific_submission_cmd_args = $self->config_get('SubmissionOptions');
 
    my ($submit_stdout_file, $submit_stderr_file);
 
    if ($submit_log_subdir) {
        $submit_stdout_file = $submit_log_subdir . "/log_${rc_name}_%A_%a.out";
        $submit_stderr_file = $submit_log_subdir . "/log_${rc_name}_%A_%a.err";
    } else {
        $submit_stdout_file = '/dev/null';
        $submit_stderr_file = '/dev/null';
    }
 
    #No equivalent in sbatch, but can be accomplished with stdbuf -oL -eL
    #$ENV{'LSB_STDOUT_DIRECT'} = 'y';  # unbuffer the output of the bsub command
 
    #Note: job arrays share the same name in slurm and are 0-based, but this may still work
    my @cmd;
    if ($array_required eq "1") {
        # We have to submit a job array - this will change the job ids in slurm to <job_id>_ARRAY
 
        @cmd = (
            'sbatch',
            '--parsable',
            '-o', $submit_stdout_file,
            '-e', $submit_stderr_file,
            # inform SLURM we submit an ARRAY
            # jobs submitted with 'sbatch -a ' get different ids back when executing
            # 'squeue --array -h -u <user> -o '%i|%T|%u|%A' later
            '-a', $job_array_spec,
            '-J', $job_array_common_name,
            split_for_bash($rc_specific_submission_cmd_args),
            split_for_bash($meadow_specific_submission_cmd_args),
            '--wrap',
            $worker_cmd,
        );
    } else {
        @cmd = (
            'sbatch',
            '--parsable',
            '-o', $submit_stdout_file,
            '-e', $submit_stderr_file,
            '-J', $job_array_common_name,
            split_for_bash($rc_specific_submission_cmd_args),
            split_for_bash($meadow_specific_submission_cmd_args),
            '--wrap',
            $worker_cmd,
        );
 
    }
 
    print "Executing [ " . $self->name . " ]: " . join(' ', @cmd) . "\n";
 
    my @output = execute_command(\@cmd);
 
    unless ($output[0] =~ m/(\d+)/) {
        die "Could not find job id in sbatch output";
    }
 
    my $slurm_job_id = $output[0];
 
    if (looks_like_number($slurm_job_id)) {
        my $return_value;
 
        # Modify the return values depending on the submission type: single job vs job array
        # as these have different indexes:
        # array jobs: [54729315_1, 54729315_2]
        # single job: [54728975]
        if ($array_required) {
            $return_value = [map {$slurm_job_id . '_' . $_ . ''} (1 .. $required_worker_count)];
        } else {
            $return_value = [$slurm_job_id];
        }
 
        return $return_value;
    } else {
        die "Unexpected output for sbatch command. Command: '@cmd'; STDOUT: '" . (join "\n", @output) . "'";
    }
} ## end sub submit_workers_return_meadow_pids

The documentation for this class was generated from the following file:

modules/Bio/EnsEMBL/Hive/Meadow/SLURM.pm

Public Member Functions

Detailed Description

Description

Member Function Documentation

◆ _init_meadow()

◆ check_worker_is_alive_and_mine()

◆ count_pending_workers_by_rc_name()

◆ count_running_workers()

◆ deregister_local_process()

◆ execute_command()

◆ get_cause_of_death()

◆ get_current_worker_process_id()

◆ get_report_entries_for_process_ids()

◆ get_report_entries_for_time_interval()

◆ kill_worker()

◆ name()

◆ parse_report_source_line()

◆ status_of_all_our_workers()

◆ submit_workers_return_meadow_pids()