qq_lib.core.config

  1# Released under MIT License.
  2# Copyright (c) 2025-2026 Ladislav Bartos and Robert Vacha Lab
  3
  4"""
  5Configuration system for qq.
  6
  7This module defines dataclasses representing all configurable aspects of qq,
  8including file suffixes, environment variables, timeouts, presentation settings,
  9batch-system options, and global defaults.
 10
 11The `Config` class loads user configuration from a TOML file (if available)
 12and provides a globally accessible `CFG` instance.
 13"""
 14
 15import os
 16import tomllib
 17from dataclasses import dataclass, field, fields, is_dataclass
 18from pathlib import Path
 19from typing import Any, Self
 20
 21
 22@dataclass(frozen=True)
 23class FileSuffixes:
 24    """File suffixes used by qq."""
 25
 26    # Suffix for qq info files.
 27    qq_info: str = ".qqinfo"
 28    # Suffix for qq output files.
 29    qq_out: str = ".qqout"
 30    # Suffix for captured stdout.
 31    stdout: str = ".out"
 32    # Suffix for captured stderr.
 33    stderr: str = ".err"
 34
 35    @property
 36    def all_suffixes(self) -> list[str]:
 37        """List of all file suffixes."""
 38        return [self.qq_info, self.qq_out, self.stdout, self.stderr]
 39
 40
 41@dataclass(frozen=True)
 42class EnvironmentVariables:
 43    """Environment variable names used by qq."""
 44
 45    # Indicates job is running inside the qq environment.
 46    guard: str = "QQ_ENV_SET"
 47    # Enables qq debug mode.
 48    debug_mode: str = "QQ_DEBUG"
 49    # Path to the qq info file for the job.
 50    info_file: str = "QQ_INFO"
 51    # Machine from which the job was submitted.
 52    input_machine: str = "QQ_INPUT_MACHINE"
 53    # Submission directory path.
 54    input_dir: str = "QQ_INPUT_DIR"
 55    # Whether submission was from shared storage.
 56    shared_submit: str = "QQ_SHARED_SUBMIT"
 57    # Name of the batch system used.
 58    batch_system: str = "QQ_BATCH_SYSTEM"
 59    # Current loop-cycle index.
 60    loop_current: str = "QQ_LOOP_CURRENT"
 61    # Starting loop-cycle index.
 62    loop_start: str = "QQ_LOOP_START"
 63    # Final loop-cycle index.
 64    loop_end: str = "QQ_LOOP_END"
 65    # Non-resubmit flag returned by a job script.
 66    no_resubmit: str = "QQ_NO_RESUBMIT"
 67    # Archive filename pattern.
 68    archive_format: str = "QQ_ARCHIVE_FORMAT"
 69    # Scratch directory on Metacentrum clusters.
 70    pbs_scratch_dir: str = "SCRATCHDIR"
 71    # Slurm account used for the job.
 72    slurm_job_account: str = "SLURM_JOB_ACCOUNT"
 73    # Storage type for LUMI scratch.
 74    lumi_scratch_type: str = "LUMI_SCRATCH_TYPE"
 75    # Total CPUs used.
 76    ncpus: str = "QQ_NCPUS"
 77    # Total GPUs used.
 78    ngpus: str = "QQ_NGPUS"
 79    # Total nodes used.
 80    nnodes: str = "QQ_NNODES"
 81    # Walltime in hours.
 82    walltime: str = "QQ_WALLTIME"
 83
 84
 85@dataclass(frozen=True)
 86class TimeoutSettings:
 87    """Timeout settings in seconds."""
 88
 89    # Timeout for SSH in seconds.
 90    ssh: int = 60
 91    # Timeout for rsync in seconds.
 92    rsync: int = 600
 93
 94
 95@dataclass(frozen=True)
 96class RunnerSettings:
 97    """Settings for Runner operations."""
 98
 99    # Maximum number of attempts when retrying an operation.
100    retry_tries: int = 3
101    # Wait time (in seconds) between retry attempts.
102    retry_wait: int = 300
103    # Delay (in seconds) between sending SIGTERM and SIGKILL to a job script.
104    sigterm_to_sigkill: int = 5
105    # Interval (in seconds) between successive checks of the running script's state.
106    subprocess_checks_wait_time: int = 2
107    # Default intepreter used to run the submitted scripts in the qq environment.
108    default_interpreter: str = "bash"
109
110
111@dataclass(frozen=True)
112class ResubmitterSettings:
113    """Settings for Resubmitter operations."""
114
115    # Maximum number of attempts when retrying an operation.
116    retry_tries: int = 3
117    # Wait time (in seconds) between retry attempts.
118    retry_wait: int = 300
119    # List of hosts from which job resubmission should be attempted.
120    # If empty, the batch system defaults are used.
121    default_resubmit_hosts: str = ""
122
123
124@dataclass(frozen=True)
125class ArchiverSettings:
126    """Settings for Archiver operations."""
127
128    # Maximum number of attempts when retrying an operation.
129    retry_tries: int = 3
130    # Wait time (in seconds) between retry attempts.
131    retry_wait: int = 300
132
133
134@dataclass(frozen=True)
135class GoerSettings:
136    """Settings for Goer operations."""
137
138    # Interval (in seconds) between successive checks of the job's state
139    # (when waiting for the job to start).
140    wait_time: int = 5
141
142
143@dataclass(frozen=True)
144class LoopJobSettings:
145    """Settings for qq loop jobs."""
146
147    # Pattern used for naming loop jobs.
148    pattern: str = "+%04d"
149    # Pattern used for names of archived files.
150    archive_format: str = "job%04d"
151    # Default name of the archive directory.
152    archive_dir: str = "storage"
153
154
155@dataclass(frozen=True)
156class JobStatusPanelSettings:
157    """Settings for creating a job status panel."""
158
159    # Maximal width of the job status panel.
160    max_width: int | None = None
161    # Minimal width of the job status panel.
162    min_width: int | None = 70
163    # Style of the border lines.
164    border_style: str = "white"
165    # Style of the title.
166    title_style: str = "white bold"
167
168
169@dataclass(frozen=True)
170class FullInfoPanelSettings:
171    """Settings for creating a full info panel."""
172
173    # Maximal width of the job info panel.
174    max_width: int | None = None
175    # Minimal width of the job info panel.
176    min_width: int | None = 80
177    # Style of the border lines.
178    border_style: str = "white"
179    # Style of the title.
180    title_style: str = "white bold"
181    # Style of the separators between individual sections of the panel.
182    rule_style: str = "white"
183
184
185@dataclass(frozen=True)
186class BriefInfoSettings:
187    """Settings for brief info display."""
188
189    # Color of the job ID in brief info.
190    job_id_color: str = "default"
191    # Color of the directory path in brief info.
192    dir_path_color: str = "cyan"
193    # Color of the loop info in brief info.
194    loop_info_color: str = "grey70"
195
196
197@dataclass(frozen=True)
198class PresenterSettings:
199    """Settings for Presenter."""
200
201    # Settings for the job status panel
202    job_status_panel: JobStatusPanelSettings = field(
203        default_factory=JobStatusPanelSettings
204    )
205
206    # Settings for the job info panel
207    full_info_panel: FullInfoPanelSettings = field(
208        default_factory=FullInfoPanelSettings
209    )
210
211    # Settings for the brief info display.
212    brief_info: BriefInfoSettings = field(default_factory=BriefInfoSettings)
213
214    # Style used for the keys in job status/info panel.
215    key_style: str = "default bold"
216    # Style used for values in job status/info panel.
217    value_style: str = "white"
218    # Style used for notes in job status/info panel.
219    notes_style: str = "grey50"
220
221
222@dataclass(frozen=True)
223class JobsPresenterSettings:
224    """Settings for JobsPresenter."""
225
226    # Maximal width of the jobs panel.
227    max_width: int | None = None
228    # Minimal width of the jobs panel.
229    min_width: int | None = 80
230    # Maximum displayed length of a job name before truncation.
231    max_job_name_length: int = 20
232    # Maximum displayed length of working nodes before truncation.
233    max_nodes_length: int = 40
234    # Style used for border lines.
235    border_style: str = "white"
236    # Style used for the title.
237    title_style: str = "white bold"
238    # Style used for the subtitle (server name).
239    subtitle_style: str = "white bold"
240    # Style used for table headers.
241    headers_style: str = "default"
242    # Style used for table values.
243    main_style: str = "white"
244    # Style used for job statistics.
245    secondary_style: str = "grey70"
246    # Style used for extra notes.
247    extra_info_style: str = "grey50"
248    # Style used for strong warning messages.
249    strong_warning_style: str = "bright_red"
250    # Style used for mild warning messages.
251    mild_warning_style: str = "bright_yellow"
252    # List of columns to show in the output.
253    # If not set, the settings for the current batch system will be used.
254    columns_to_show: list[str] | None = None
255
256    # Code used to signify "total jobs".
257    sum_jobs_code: str = "Σ"
258
259
260@dataclass(frozen=True)
261class QueuesPresenterSettings:
262    """Settings for QueuesPresenter."""
263
264    # Maximal width of the queues panel.
265    max_width: int | None = None
266    # Minimal width of the queues panel.
267    min_width: int | None = 80
268    # Style used for border lines.
269    border_style: str = "white"
270    # Style used for the title.
271    title_style: str = "white bold"
272    # Style used for the subtitle (server name).
273    subtitle_style: str = "white bold"
274    # Style used for table headers.
275    headers_style: str = "default"
276
277    # Mark used to denote main queues.
278    main_mark = "●"
279    # Mark used to denote reroutings.
280    rerouted_mark = " ··>"
281
282    # Style used for the mark if the queue is available.
283    available_mark_style: str = "bright_green"
284    # Style used for the mark if the queue is not available.
285    unavailable_mark_style: str = "bright_red"
286    # Style used for the mark if the queue is dangling.
287    dangling_mark_style: str = "bright_yellow"
288
289    # Style used for information about main queues.
290    main_text_style: str = "white"
291    # Style used for information about reroutings.
292    rerouted_text_style: str = "grey50"
293
294    # Code used to signify "other jobs".
295    other_jobs_code: str = "O"
296    # Code used to signify "total jobs".
297    sum_jobs_code: str = "Σ"
298
299
300@dataclass(frozen=True)
301class NodesPresenterSettings:
302    """Settings for NodesPresenter."""
303
304    # Maximal width of the nodes panel.
305    max_width: int | None = None
306    # Minimal width of the nodes panel.
307    min_width: int | None = 80
308    # Maximal width of the shared properties section.
309    max_props_panel_width: int = 40
310    # Style used for border lines.
311    border_style: str = "white"
312    # Style used for the title.
313    title_style: str = "white bold"
314    # Style used for the subtitle (server name).
315    subtitle_style: str = "white bold"
316    # Style used for table headers.
317    headers_style: str = "default"
318    # Style of the separators between individual sections of the panel.
319    rule_style: str = "white"
320    # Name to use for the leftover nodes that were not assigned to any group.
321    others_group_name: str = "other"
322    # Name to use for the group if it contains all nodes.
323    all_nodes_group_name: str = "all nodes"
324
325    # Mark used to denote nodes.
326    state_mark = "●"
327
328    # Style used for main information about the nodes.
329    main_text_style: str = "white"
330    # Style used for statistics and shared properties.
331    secondary_text_style: str = "grey70"
332    # Style used for the mark and resources if the node is free.
333    free_node_style: str = "bright_green bold"
334    # Style used for the mark and resources if the node is partially free.
335    part_free_node_style: str = "green"
336    # Style used for the mark and resources if the node is busy.
337    busy_node_style: str = "blue"
338    # Style used for all information about unavailable nodes.
339    unavailable_node_style = "bright_red"
340
341
342@dataclass(frozen=True)
343class DateFormats:
344    """Date and time format strings."""
345
346    # Standard date format used by qq.
347    standard: str = "%Y-%m-%d %H:%M:%S"
348    # Date format used by PBS Pro.
349    pbs: str = "%a %b %d %H:%M:%S %Y"
350    # Date format used by Slurm.
351    slurm: str = "%Y-%m-%dT%H:%M:%S"
352
353
354@dataclass(frozen=True)
355class ExitCodes:
356    """Exit codes used for various errors."""
357
358    # Returned when a qq script is run outside the qq environment.
359    not_qq_env: int = 90
360    # Default error code for failures of qq commands or most errors in the qq environment.
361    default: int = 91
362    # Returned when a qq job fails and its error state cannot be written to the qq info file.
363    qq_run_fatal: int = 92
364    # Returned when a qq job fails due to a communication error between qq services.
365    qq_run_communication: int = 93
366    # Used by job scripts to signal that a loop job should not be resubmitted.
367    qq_run_no_resubmit: int = 95
368    # Returned on an unexpected or unhandled error.
369    unexpected_error: int = 99
370
371
372@dataclass(frozen=True)
373class StateColors:
374    """Color scheme for RealState display."""
375
376    # Style used for queued jobs.
377    queued: str = "bright_magenta"
378    # Style used for held jobs.
379    held: str = "bright_magenta"
380    # Style used for suspended jobs.
381    suspended: str = "bright_black"
382    # Style used for waiting jobs.
383    waiting: str = "bright_magenta"
384    # Style used for running jobs.
385    running: str = "bright_blue"
386    # Style used for booting jobs.
387    booting: str = "bright_cyan"
388    # Style used for killed jobs.
389    killed: str = "bright_red"
390    # Style used for failed jobs.
391    failed: str = "bright_red"
392    # Style used for finished jobs.
393    finished: str = "bright_green"
394    # Style used for exiting jobs.
395    exiting: str = "bright_yellow"
396    # Style used for jobs in an inconsistent state.
397    in_an_inconsistent_state: str = "grey70"
398    # Style used for jobs in an unknown state.
399    unknown: str = "grey70"
400    # Style used whenever a summary of jobs is provided.
401    sum: str = "white"
402    # Style used for "other" job states.
403    other: str = "grey70"
404
405
406@dataclass(frozen=True)
407class SizeOptions:
408    """Options associated with the Size dataclass."""
409
410    # Maximal relative error acceptable when rounding Size values for display.
411    max_rounding_error: float = 0.1
412
413
414@dataclass(frozen=True)
415class PBSOptions:
416    """Options associated with PBS."""
417
418    # Name of the subdirectory inside SCRATCHDIR used as the job's working directory.
419    scratch_dir_inner: str = "main"
420
421
422@dataclass(frozen=True)
423class SlurmOptions:
424    """Options associated with Slurm."""
425
426    # Maximal number of threads used to collect information about jobs using scontrol.
427    jobs_scontrol_nthreads: int = 8
428
429
430@dataclass(frozen=True)
431class SlurmIT4IOptions:
432    """Options associated with Slurm on IT4I clusters."""
433
434    # Number of attempts when preparing a working directory on scratch.
435    scratch_dir_attempts: int = 3
436
437
438@dataclass(frozen=True)
439class SlurmLumiOptions:
440    """Options associated with Slurm on LUMI."""
441
442    # Number of attempts when preparing a working directory on scratch.
443    scratch_dir_attempts: int = 3
444
445
446@dataclass(frozen=True)
447class TransferFilesOptions:
448    """Options associated with transferring and archiving files."""
449
450    # Default archive mode used for jobs.
451    default_archive_mode: str = "success"
452
453    # Default transfer mode used for jobs.
454    default_transfer_mode: str = "success"
455
456
457@dataclass(frozen=True)
458class BatchServersOptions:
459    """Options associated with selecting and specifying batch servers."""
460
461    # Dictionary mapping known server shortcuts to full server names.
462    known_servers: dict[str, str] = field(
463        default_factory=lambda: {
464            "robox": "robox-pro.ceitec.muni.cz",
465            "sokar": "sokar-pbs.ncbr.muni.cz",
466            "metacentrum": "pbs-m1.metacentrum.cz",
467            "meta": "pbs-m1.metacentrum.cz",
468        }
469    )
470
471    # Dictionary mapping known server names to frontends.
472    known_output_hosts: dict[str, str] = field(
473        default_factory=lambda: {
474            "robox-pro.ceitec.muni.cz": "st1.ceitec.muni.cz",
475            "sokar-pbs.ncbr.muni.cz": "sokar.ncbr.muni.cz",
476            "pbs-m1.metacentrum.cz": "perian.metacentrum.cz",
477        }
478    )
479
480
481@dataclass(frozen=True)
482class ParallelizationOptions:
483    """Options associated with multithreaded execution."""
484
485    # Maximal number of threads used to collect job information.
486    job_info_max_threads: int = 16
487
488    # Maximal number of threads used to submit jobs in parallel.
489    submission_max_threads: int = 8
490
491    # Maximal number of threads used to clear runtime files in parallel.
492    clear_max_threads: int = 8
493
494
495@dataclass(frozen=True)
496class Config:
497    """Main configuration for qq."""
498
499    suffixes: FileSuffixes = field(default_factory=FileSuffixes)
500    env_vars: EnvironmentVariables = field(default_factory=EnvironmentVariables)
501    timeouts: TimeoutSettings = field(default_factory=TimeoutSettings)
502    runner: RunnerSettings = field(default_factory=RunnerSettings)
503    resubmitter: ResubmitterSettings = field(default_factory=ResubmitterSettings)
504    archiver: ArchiverSettings = field(default_factory=ArchiverSettings)
505    goer: GoerSettings = field(default_factory=GoerSettings)
506    presenter: PresenterSettings = field(default_factory=PresenterSettings)
507    loop_jobs: LoopJobSettings = field(default_factory=LoopJobSettings)
508    jobs_presenter: JobsPresenterSettings = field(default_factory=JobsPresenterSettings)
509    queues_presenter: QueuesPresenterSettings = field(
510        default_factory=QueuesPresenterSettings
511    )
512    nodes_presenter: NodesPresenterSettings = field(
513        default_factory=NodesPresenterSettings
514    )
515    date_formats: DateFormats = field(default_factory=DateFormats)
516    exit_codes: ExitCodes = field(default_factory=ExitCodes)
517    state_colors: StateColors = field(default_factory=StateColors)
518    size: SizeOptions = field(default_factory=SizeOptions)
519    pbs_options: PBSOptions = field(default_factory=PBSOptions)
520    slurm_options: SlurmOptions = field(default_factory=SlurmOptions)
521    slurm_it4i_options: SlurmIT4IOptions = field(default_factory=SlurmIT4IOptions)
522    slurm_lumi_options: SlurmLumiOptions = field(default_factory=SlurmLumiOptions)
523    transfer_files_options: TransferFilesOptions = field(
524        default_factory=TransferFilesOptions
525    )
526    batch_servers_options: BatchServersOptions = field(
527        default_factory=BatchServersOptions
528    )
529    parallelization_options: ParallelizationOptions = field(
530        default_factory=ParallelizationOptions
531    )
532
533    # Name of the qq binary.
534    binary_name: str = "qq"
535
536    @classmethod
537    def load(cls, config_path: Path | None = None) -> Self:
538        """
539        Load configuration from TOML file or use defaults.
540
541        Args:
542            config_path: Explicit path to config file. If None, searches standard locations.
543
544        Returns:
545            Config instance with loaded or default values.
546        """
547        if config_path is None:
548            config_path = Config._get_config_path()
549
550        try:
551            if config_path and config_path.exists():
552                with config_path.open("rb") as f:
553                    config_data = tomllib.load(f)
554                return _dict_to_dataclass(cls, config_data)
555        except Exception as e:
556            print(
557                f"[ FATAL CONFIGURATION ERROR ] Could not read qq config '{config_path}': {e}."
558            )
559            print(
560                "[ FATAL CONFIGURATION ERROR ] Falling back to default configuration.\n\n"
561            )
562
563        # no config found - use defaults
564        return cls()
565
566    @staticmethod
567    def _get_config_path() -> Path | None:
568        """
569        Search for config file in standard locations (XDG compliant).
570        Returns the first existing config file, or None.
571        """
572        config_locations: list[Path | None] = [
573            # 1. Explicit environment variable (highest priority)
574            Path(env_path) if (env_path := os.getenv("QQ_CONFIG")) else None,
575            # 2. Current working directory (for development/override)
576            Path.cwd() / "qq_config.toml",
577            # 3. XDG config home (standard user config location)
578            Path(os.getenv("XDG_CONFIG_HOME", Path.home() / ".config"))
579            / "qq"
580            / "config.toml",
581        ]
582
583        for path in config_locations:
584            if path and path.is_file():
585                return path
586
587        return None
588
589
590def _dict_to_dataclass(cls, data: dict[str, Any]):
591    """
592    Recursively convert a dictionary to a dataclass instance.
593    Handles nested dataclasses properly.
594    """
595    if not is_dataclass(cls):
596        return data
597
598    field_values = {}
599    for field_info in fields(cls):
600        field_name = field_info.name
601        field_type = field_info.type
602
603        if field_name in data:
604            value = data[field_name]
605            if is_dataclass(field_type) and isinstance(value, dict):
606                field_values[field_name] = _dict_to_dataclass(field_type, value)
607            else:
608                field_values[field_name] = value
609
610    return cls(**field_values)
611
612
613# Global configuration for qq.
614CFG = Config.load()