trnbl.loggers.local.locallogger API documentation

docs for trnbl v0.1.1
View Source on GitHub

# trnbl.loggers.local.locallogger

View Source
View on GitHub

      1import datetime
      2import hashlib
      3import json
      4from typing import Any
      5from pathlib import Path
      6import io
      7import inspect
      8
      9import yaml  # type: ignore[import-untyped]
     10from trnbl.loggers.base import TrainingLoggerBase, rand_syllabic_string
     11
     12
     13class FilePaths:
     14	# configs and metadata
     15	TRAIN_CONFIG: Path = Path("config.json")
     16	LOGGER_META: Path = Path("meta.json")
     17	# configs and metadata in yaml format for easier human readability
     18	TRAIN_CONFIG_YML: Path = Path("config.yml")
     19	LOGGER_META_YML: Path = Path("meta.yml")
     20
     21	# logs, metrics, and artifacts
     22	ARTIFACTS: Path = Path("artifacts.jsonl")
     23	METRICS: Path = Path("metrics.jsonl")
     24	LOG: Path = Path("log.jsonl")
     25	# keeps error message if an error occurs
     26	ERROR_FILE: Path = Path("ERROR.txt")
     27
     28	# manifest is shared between all runs in a project
     29	# relative to project path instead of run path
     30	RUNS_MANIFEST: Path = Path("runs.jsonl")
     31	# directory in project path for runs
     32	# relative to project path instead of run path
     33	RUNS_DIR: Path = Path("runs")
     34
     35	# frontend files
     36	HTML_INDEX: Path = Path("index.html")
     37	START_SERVER: Path = Path("start_server.py")
     38
     39
     40class LocalLogger(TrainingLoggerBase):
     41	def __init__(
     42		self,
     43		project: str,
     44		metric_names: list[str],
     45		train_config: dict,
     46		group: str = "",
     47		base_path: str | Path = Path("trnbl-logs"),
     48		memusage_as_metrics: bool = True,
     49		console_msg_prefix: str = "# ",
     50	):
     51		# set up lists
     52		self.log_list: list[dict] = list()
     53		self.metrics_list: list[dict] = list()
     54		self.artifacts_list: list[dict] = list()
     55
     56		# copy kwargs
     57		self.train_config: dict = train_config
     58		self.project: str = project
     59		self.group: str = group
     60		self.group_str: str = self.group + ("-" if group and group[-1] != "-" else "")
     61		self.base_path: Path = Path(base_path)
     62		self.console_msg_prefix: str = console_msg_prefix
     63
     64		# set up id
     65		self._syllabic_id: str = rand_syllabic_string()
     66		self.run_init_timestamp: datetime.datetime = datetime.datetime.now()
     67		self.run_id: str = self._get_run_id()
     68
     69		# set up paths
     70		self.project_path: Path = self.base_path / project
     71		self._run_path: Path = self.project_path / FilePaths.RUNS_DIR / self.run_id
     72		# make sure the run path doesn't already exist
     73		assert not self._run_path.exists()
     74		self._run_path.mkdir(parents=True, exist_ok=True)
     75
     76		# set up files and objects for logs, artifacts, and metrics
     77		# ----------------------------------------
     78
     79		self.log_file: io.TextIOWrapper = open(self.run_path / FilePaths.LOG, "a")
     80
     81		self.metrics_file: io.TextIOWrapper = open(
     82			self.run_path / FilePaths.METRICS, "a"
     83		)
     84
     85		self.artifacts_file: io.TextIOWrapper = open(
     86			self.run_path / FilePaths.ARTIFACTS, "a"
     87		)
     88
     89		# metric names (getting mem usage might cause problems if we have an error)
     90		self.metric_names: list[str] = metric_names
     91		if memusage_as_metrics:
     92			self.metric_names += list(self.get_mem_usage().keys())
     93
     94		# put everything in a config
     95		self.logger_meta: dict = dict(
     96			run_id=self.run_id,
     97			run_path=self.run_path.as_posix(),
     98			syllabic_id=self.syllabic_id,
     99			group=self.group,
    100			project=self.project,
    101			run_init_timestamp=str(self.run_init_timestamp.isoformat()),
    102			metric_names=metric_names,
    103			train_config=train_config,  # TODO: this duplicates the contents of FilePaths.TRAIN_CONFIG, is that ok?
    104		)
    105
    106		# write to the project jsonl
    107		with open(self.project_path / FilePaths.RUNS_MANIFEST, "a") as f:
    108			json.dump(self.logger_meta, f)
    109			f.write("\n")
    110
    111		# write the index.html and start_server.py files
    112		# ----------------------------------------
    113		from trnbl.loggers.local.html_frontend import get_html_frontend
    114
    115		with open(self.project_path / FilePaths.HTML_INDEX, "w") as f:
    116			f.write(get_html_frontend())
    117
    118		import trnbl.loggers.local.start_server as start_server_module
    119
    120		with open(self.project_path / FilePaths.START_SERVER, "w") as f:
    121			f.write(inspect.getsource(start_server_module))
    122
    123		# write init files
    124		# ----------------------------------------
    125
    126		# logger metadata
    127		with open(self.run_path / FilePaths.LOGGER_META, "w") as f:
    128			json.dump(self.logger_meta, f, indent="\t")
    129
    130		with open(self.run_path / FilePaths.LOGGER_META_YML, "w") as f:
    131			yaml.dump(self.logger_meta, f)
    132
    133		# training/model/dataset config
    134		with open(self.run_path / FilePaths.TRAIN_CONFIG, "w") as f:
    135			json.dump(train_config, f, indent="\t")
    136
    137		with open(self.run_path / FilePaths.TRAIN_CONFIG_YML, "w") as f:
    138			yaml.dump(train_config, f)
    139
    140		self.message(f"starting logger with id {self.run_id}")
    141
    142	@property
    143	def _run_hash(self) -> str:
    144		return hashlib.md5(str(self.train_config).encode()).hexdigest()
    145
    146	@property
    147	def syllabic_id(self) -> str:
    148		return self._syllabic_id
    149
    150	def _get_run_id(self) -> str:
    151		return f"{self.group_str}h{self._run_hash[:5]}-{self.run_init_timestamp.strftime('%y%m%d_%H%M')}-{self.syllabic_id}"
    152
    153	def get_timestamp(self) -> str:
    154		return datetime.datetime.now().isoformat()
    155
    156	def _log(self, message: str, **kwargs) -> None:
    157		"""(internal) log a progress message"""
    158		# TODO: also log messages via regular logger to stdout
    159		msg_dict: dict = dict(
    160			message=message,
    161			timestamp=self.get_timestamp(),
    162		)
    163		if kwargs:
    164			msg_dict.update(kwargs)
    165
    166		self.log_list.append(msg_dict)
    167		self.log_file.write(json.dumps(msg_dict) + "\n")
    168		self.log_file.flush()
    169
    170	def debug(self, message: str, **kwargs) -> None:
    171		"""log a debug message"""
    172		self._log(message, __dbg__=True, **kwargs)
    173
    174	def message(self, message: str, **kwargs) -> None:
    175		"""log a progress message"""
    176		# TODO: also log messages via regular logger to stdout
    177		self._log(message, **kwargs)
    178		print(self.console_msg_prefix + message)
    179
    180	def warning(self, message: str, **kwargs) -> None:
    181		"""log a warning message"""
    182		self.message(
    183			f"WARNING: {message}",
    184			__warning__=True,
    185			**kwargs,
    186		)
    187
    188	def error(self, message: str, **kwargs) -> None:
    189		"""log an error message"""
    190		self.message(
    191			f"ERROR: {message}",
    192			__error__=True,
    193			**kwargs,
    194		)
    195		with open(self.run_path / FilePaths.ERROR_FILE, "a") as f:
    196			f.write("=" * 80 + "\n")
    197			f.write("exception at " + self.get_timestamp() + "\n")
    198			f.write(message)
    199			f.write("\n")
    200			f.flush()
    201
    202	def metrics(self, data: dict[str, Any]) -> None:
    203		"""log a dictionary of metrics"""
    204		data["timestamp"] = self.get_timestamp()
    205
    206		self.metrics_list.append(data)
    207		self.metrics_file.write(json.dumps(data) + "\n")
    208
    209	def artifact(
    210		self,
    211		path: Path,
    212		type: str,
    213		aliases: list[str] | None = None,
    214		metadata: dict | None = None,
    215	) -> None:
    216		"""log an artifact from a file"""
    217		artifact_dict: dict = dict(
    218			timestamp=self.get_timestamp(),
    219			path=path.as_posix(),
    220			type=type,
    221			aliases=aliases,
    222			metadata=metadata if metadata else {},
    223		)
    224
    225		self.artifacts_list.append(artifact_dict)
    226		self.artifacts_file.write(json.dumps(artifact_dict) + "\n")
    227
    228	@property
    229	def url(self) -> str:
    230		"""Get the URL for the current logging run"""
    231		return self.run_path.as_posix()
    232
    233	@property
    234	def run_path(self) -> Path:
    235		"""Get the path to the current logging run"""
    236		return self._run_path
    237
    238	def flush(self) -> None:
    239		self.log_file.flush()
    240		self.metrics_file.flush()
    241		self.artifacts_file.flush()
    242
    243	def finish(self) -> None:
    244		self.message("closing logger")
    245
    246		self.log_file.flush()
    247		self.log_file.close()
    248
    249		self.metrics_file.flush()
    250		self.metrics_file.close()
    251
    252		self.artifacts_file.flush()
    253		self.artifacts_file.close()

class FilePaths:
View Source
View on GitHub

    14class FilePaths:
    15	# configs and metadata
    16	TRAIN_CONFIG: Path = Path("config.json")
    17	LOGGER_META: Path = Path("meta.json")
    18	# configs and metadata in yaml format for easier human readability
    19	TRAIN_CONFIG_YML: Path = Path("config.yml")
    20	LOGGER_META_YML: Path = Path("meta.yml")
    21
    22	# logs, metrics, and artifacts
    23	ARTIFACTS: Path = Path("artifacts.jsonl")
    24	METRICS: Path = Path("metrics.jsonl")
    25	LOG: Path = Path("log.jsonl")
    26	# keeps error message if an error occurs
    27	ERROR_FILE: Path = Path("ERROR.txt")
    28
    29	# manifest is shared between all runs in a project
    30	# relative to project path instead of run path
    31	RUNS_MANIFEST: Path = Path("runs.jsonl")
    32	# directory in project path for runs
    33	# relative to project path instead of run path
    34	RUNS_DIR: Path = Path("runs")
    35
    36	# frontend files
    37	HTML_INDEX: Path = Path("index.html")
    38	START_SERVER: Path = Path("start_server.py")

TRAIN_CONFIG: pathlib.Path = WindowsPath('config.json')
LOGGER_META: pathlib.Path = WindowsPath('meta.json')
TRAIN_CONFIG_YML: pathlib.Path = WindowsPath('config.yml')
LOGGER_META_YML: pathlib.Path = WindowsPath('meta.yml')
ARTIFACTS: pathlib.Path = WindowsPath('artifacts.jsonl')
METRICS: pathlib.Path = WindowsPath('metrics.jsonl')
LOG: pathlib.Path = WindowsPath('log.jsonl')
ERROR_FILE: pathlib.Path = WindowsPath('ERROR.txt')
RUNS_MANIFEST: pathlib.Path = WindowsPath('runs.jsonl')
RUNS_DIR: pathlib.Path = WindowsPath('runs')
HTML_INDEX: pathlib.Path = WindowsPath('index.html')
START_SERVER: pathlib.Path = WindowsPath('start_server.py')
class LocalLogger(trnbl.loggers.base.TrainingLoggerBase):
View Source
View on GitHub

     41class LocalLogger(TrainingLoggerBase):
     42	def __init__(
     43		self,
     44		project: str,
     45		metric_names: list[str],
     46		train_config: dict,
     47		group: str = "",
     48		base_path: str | Path = Path("trnbl-logs"),
     49		memusage_as_metrics: bool = True,
     50		console_msg_prefix: str = "# ",
     51	):
     52		# set up lists
     53		self.log_list: list[dict] = list()
     54		self.metrics_list: list[dict] = list()
     55		self.artifacts_list: list[dict] = list()
     56
     57		# copy kwargs
     58		self.train_config: dict = train_config
     59		self.project: str = project
     60		self.group: str = group
     61		self.group_str: str = self.group + ("-" if group and group[-1] != "-" else "")
     62		self.base_path: Path = Path(base_path)
     63		self.console_msg_prefix: str = console_msg_prefix
     64
     65		# set up id
     66		self._syllabic_id: str = rand_syllabic_string()
     67		self.run_init_timestamp: datetime.datetime = datetime.datetime.now()
     68		self.run_id: str = self._get_run_id()
     69
     70		# set up paths
     71		self.project_path: Path = self.base_path / project
     72		self._run_path: Path = self.project_path / FilePaths.RUNS_DIR / self.run_id
     73		# make sure the run path doesn't already exist
     74		assert not self._run_path.exists()
     75		self._run_path.mkdir(parents=True, exist_ok=True)
     76
     77		# set up files and objects for logs, artifacts, and metrics
     78		# ----------------------------------------
     79
     80		self.log_file: io.TextIOWrapper = open(self.run_path / FilePaths.LOG, "a")
     81
     82		self.metrics_file: io.TextIOWrapper = open(
     83			self.run_path / FilePaths.METRICS, "a"
     84		)
     85
     86		self.artifacts_file: io.TextIOWrapper = open(
     87			self.run_path / FilePaths.ARTIFACTS, "a"
     88		)
     89
     90		# metric names (getting mem usage might cause problems if we have an error)
     91		self.metric_names: list[str] = metric_names
     92		if memusage_as_metrics:
     93			self.metric_names += list(self.get_mem_usage().keys())
     94
     95		# put everything in a config
     96		self.logger_meta: dict = dict(
     97			run_id=self.run_id,
     98			run_path=self.run_path.as_posix(),
     99			syllabic_id=self.syllabic_id,
    100			group=self.group,
    101			project=self.project,
    102			run_init_timestamp=str(self.run_init_timestamp.isoformat()),
    103			metric_names=metric_names,
    104			train_config=train_config,  # TODO: this duplicates the contents of FilePaths.TRAIN_CONFIG, is that ok?
    105		)
    106
    107		# write to the project jsonl
    108		with open(self.project_path / FilePaths.RUNS_MANIFEST, "a") as f:
    109			json.dump(self.logger_meta, f)
    110			f.write("\n")
    111
    112		# write the index.html and start_server.py files
    113		# ----------------------------------------
    114		from trnbl.loggers.local.html_frontend import get_html_frontend
    115
    116		with open(self.project_path / FilePaths.HTML_INDEX, "w") as f:
    117			f.write(get_html_frontend())
    118
    119		import trnbl.loggers.local.start_server as start_server_module
    120
    121		with open(self.project_path / FilePaths.START_SERVER, "w") as f:
    122			f.write(inspect.getsource(start_server_module))
    123
    124		# write init files
    125		# ----------------------------------------
    126
    127		# logger metadata
    128		with open(self.run_path / FilePaths.LOGGER_META, "w") as f:
    129			json.dump(self.logger_meta, f, indent="\t")
    130
    131		with open(self.run_path / FilePaths.LOGGER_META_YML, "w") as f:
    132			yaml.dump(self.logger_meta, f)
    133
    134		# training/model/dataset config
    135		with open(self.run_path / FilePaths.TRAIN_CONFIG, "w") as f:
    136			json.dump(train_config, f, indent="\t")
    137
    138		with open(self.run_path / FilePaths.TRAIN_CONFIG_YML, "w") as f:
    139			yaml.dump(train_config, f)
    140
    141		self.message(f"starting logger with id {self.run_id}")
    142
    143	@property
    144	def _run_hash(self) -> str:
    145		return hashlib.md5(str(self.train_config).encode()).hexdigest()
    146
    147	@property
    148	def syllabic_id(self) -> str:
    149		return self._syllabic_id
    150
    151	def _get_run_id(self) -> str:
    152		return f"{self.group_str}h{self._run_hash[:5]}-{self.run_init_timestamp.strftime('%y%m%d_%H%M')}-{self.syllabic_id}"
    153
    154	def get_timestamp(self) -> str:
    155		return datetime.datetime.now().isoformat()
    156
    157	def _log(self, message: str, **kwargs) -> None:
    158		"""(internal) log a progress message"""
    159		# TODO: also log messages via regular logger to stdout
    160		msg_dict: dict = dict(
    161			message=message,
    162			timestamp=self.get_timestamp(),
    163		)
    164		if kwargs:
    165			msg_dict.update(kwargs)
    166
    167		self.log_list.append(msg_dict)
    168		self.log_file.write(json.dumps(msg_dict) + "\n")
    169		self.log_file.flush()
    170
    171	def debug(self, message: str, **kwargs) -> None:
    172		"""log a debug message"""
    173		self._log(message, __dbg__=True, **kwargs)
    174
    175	def message(self, message: str, **kwargs) -> None:
    176		"""log a progress message"""
    177		# TODO: also log messages via regular logger to stdout
    178		self._log(message, **kwargs)
    179		print(self.console_msg_prefix + message)
    180
    181	def warning(self, message: str, **kwargs) -> None:
    182		"""log a warning message"""
    183		self.message(
    184			f"WARNING: {message}",
    185			__warning__=True,
    186			**kwargs,
    187		)
    188
    189	def error(self, message: str, **kwargs) -> None:
    190		"""log an error message"""
    191		self.message(
    192			f"ERROR: {message}",
    193			__error__=True,
    194			**kwargs,
    195		)
    196		with open(self.run_path / FilePaths.ERROR_FILE, "a") as f:
    197			f.write("=" * 80 + "\n")
    198			f.write("exception at " + self.get_timestamp() + "\n")
    199			f.write(message)
    200			f.write("\n")
    201			f.flush()
    202
    203	def metrics(self, data: dict[str, Any]) -> None:
    204		"""log a dictionary of metrics"""
    205		data["timestamp"] = self.get_timestamp()
    206
    207		self.metrics_list.append(data)
    208		self.metrics_file.write(json.dumps(data) + "\n")
    209
    210	def artifact(
    211		self,
    212		path: Path,
    213		type: str,
    214		aliases: list[str] | None = None,
    215		metadata: dict | None = None,
    216	) -> None:
    217		"""log an artifact from a file"""
    218		artifact_dict: dict = dict(
    219			timestamp=self.get_timestamp(),
    220			path=path.as_posix(),
    221			type=type,
    222			aliases=aliases,
    223			metadata=metadata if metadata else {},
    224		)
    225
    226		self.artifacts_list.append(artifact_dict)
    227		self.artifacts_file.write(json.dumps(artifact_dict) + "\n")
    228
    229	@property
    230	def url(self) -> str:
    231		"""Get the URL for the current logging run"""
    232		return self.run_path.as_posix()
    233
    234	@property
    235	def run_path(self) -> Path:
    236		"""Get the path to the current logging run"""
    237		return self._run_path
    238
    239	def flush(self) -> None:
    240		self.log_file.flush()
    241		self.metrics_file.flush()
    242		self.artifacts_file.flush()
    243
    244	def finish(self) -> None:
    245		self.message("closing logger")
    246
    247		self.log_file.flush()
    248		self.log_file.close()
    249
    250		self.metrics_file.flush()
    251		self.metrics_file.close()
    252
    253		self.artifacts_file.flush()
    254		self.artifacts_file.close()

Base class for training loggers

LocalLogger( project: str, metric_names: list[str], train_config: dict, group: str = '', base_path: str | pathlib.Path = WindowsPath('trnbl-logs'), memusage_as_metrics: bool = True, console_msg_prefix: str = '# ')
View Source
View on GitHub

     42	def __init__(
     43		self,
     44		project: str,
     45		metric_names: list[str],
     46		train_config: dict,
     47		group: str = "",
     48		base_path: str | Path = Path("trnbl-logs"),
     49		memusage_as_metrics: bool = True,
     50		console_msg_prefix: str = "# ",
     51	):
     52		# set up lists
     53		self.log_list: list[dict] = list()
     54		self.metrics_list: list[dict] = list()
     55		self.artifacts_list: list[dict] = list()
     56
     57		# copy kwargs
     58		self.train_config: dict = train_config
     59		self.project: str = project
     60		self.group: str = group
     61		self.group_str: str = self.group + ("-" if group and group[-1] != "-" else "")
     62		self.base_path: Path = Path(base_path)
     63		self.console_msg_prefix: str = console_msg_prefix
     64
     65		# set up id
     66		self._syllabic_id: str = rand_syllabic_string()
     67		self.run_init_timestamp: datetime.datetime = datetime.datetime.now()
     68		self.run_id: str = self._get_run_id()
     69
     70		# set up paths
     71		self.project_path: Path = self.base_path / project
     72		self._run_path: Path = self.project_path / FilePaths.RUNS_DIR / self.run_id
     73		# make sure the run path doesn't already exist
     74		assert not self._run_path.exists()
     75		self._run_path.mkdir(parents=True, exist_ok=True)
     76
     77		# set up files and objects for logs, artifacts, and metrics
     78		# ----------------------------------------
     79
     80		self.log_file: io.TextIOWrapper = open(self.run_path / FilePaths.LOG, "a")
     81
     82		self.metrics_file: io.TextIOWrapper = open(
     83			self.run_path / FilePaths.METRICS, "a"
     84		)
     85
     86		self.artifacts_file: io.TextIOWrapper = open(
     87			self.run_path / FilePaths.ARTIFACTS, "a"
     88		)
     89
     90		# metric names (getting mem usage might cause problems if we have an error)
     91		self.metric_names: list[str] = metric_names
     92		if memusage_as_metrics:
     93			self.metric_names += list(self.get_mem_usage().keys())
     94
     95		# put everything in a config
     96		self.logger_meta: dict = dict(
     97			run_id=self.run_id,
     98			run_path=self.run_path.as_posix(),
     99			syllabic_id=self.syllabic_id,
    100			group=self.group,
    101			project=self.project,
    102			run_init_timestamp=str(self.run_init_timestamp.isoformat()),
    103			metric_names=metric_names,
    104			train_config=train_config,  # TODO: this duplicates the contents of FilePaths.TRAIN_CONFIG, is that ok?
    105		)
    106
    107		# write to the project jsonl
    108		with open(self.project_path / FilePaths.RUNS_MANIFEST, "a") as f:
    109			json.dump(self.logger_meta, f)
    110			f.write("\n")
    111
    112		# write the index.html and start_server.py files
    113		# ----------------------------------------
    114		from trnbl.loggers.local.html_frontend import get_html_frontend
    115
    116		with open(self.project_path / FilePaths.HTML_INDEX, "w") as f:
    117			f.write(get_html_frontend())
    118
    119		import trnbl.loggers.local.start_server as start_server_module
    120
    121		with open(self.project_path / FilePaths.START_SERVER, "w") as f:
    122			f.write(inspect.getsource(start_server_module))
    123
    124		# write init files
    125		# ----------------------------------------
    126
    127		# logger metadata
    128		with open(self.run_path / FilePaths.LOGGER_META, "w") as f:
    129			json.dump(self.logger_meta, f, indent="\t")
    130
    131		with open(self.run_path / FilePaths.LOGGER_META_YML, "w") as f:
    132			yaml.dump(self.logger_meta, f)
    133
    134		# training/model/dataset config
    135		with open(self.run_path / FilePaths.TRAIN_CONFIG, "w") as f:
    136			json.dump(train_config, f, indent="\t")
    137
    138		with open(self.run_path / FilePaths.TRAIN_CONFIG_YML, "w") as f:
    139			yaml.dump(train_config, f)
    140
    141		self.message(f"starting logger with id {self.run_id}")

log_list: list[dict]
metrics_list: list[dict]
artifacts_list: list[dict]
train_config: dict
project: str
group: str
group_str: str
base_path: pathlib.Path
console_msg_prefix: str
run_init_timestamp: datetime.datetime
run_id: str
project_path: pathlib.Path
log_file: _io.TextIOWrapper
metrics_file: _io.TextIOWrapper
artifacts_file: _io.TextIOWrapper
metric_names: list[str]
logger_meta: dict
syllabic_id: str
View Source
View on GitHub

    147	@property
    148	def syllabic_id(self) -> str:
    149		return self._syllabic_id

def get_timestamp(self) -> str:
View Source
View on GitHub

    154	def get_timestamp(self) -> str:
    155		return datetime.datetime.now().isoformat()

def debug(self, message: str, **kwargs) -> None:
View Source
View on GitHub

    171	def debug(self, message: str, **kwargs) -> None:
    172		"""log a debug message"""
    173		self._log(message, __dbg__=True, **kwargs)

log a debug message

def message(self, message: str, **kwargs) -> None:
View Source
View on GitHub

    175	def message(self, message: str, **kwargs) -> None:
    176		"""log a progress message"""
    177		# TODO: also log messages via regular logger to stdout
    178		self._log(message, **kwargs)
    179		print(self.console_msg_prefix + message)

log a progress message

def warning(self, message: str, **kwargs) -> None:
View Source
View on GitHub

    181	def warning(self, message: str, **kwargs) -> None:
    182		"""log a warning message"""
    183		self.message(
    184			f"WARNING: {message}",
    185			__warning__=True,
    186			**kwargs,
    187		)

log a warning message

def error(self, message: str, **kwargs) -> None:
View Source
View on GitHub

    189	def error(self, message: str, **kwargs) -> None:
    190		"""log an error message"""
    191		self.message(
    192			f"ERROR: {message}",
    193			__error__=True,
    194			**kwargs,
    195		)
    196		with open(self.run_path / FilePaths.ERROR_FILE, "a") as f:
    197			f.write("=" * 80 + "\n")
    198			f.write("exception at " + self.get_timestamp() + "\n")
    199			f.write(message)
    200			f.write("\n")
    201			f.flush()

log an error message

def metrics(self, data: dict[str, typing.Any]) -> None:
View Source
View on GitHub

    203	def metrics(self, data: dict[str, Any]) -> None:
    204		"""log a dictionary of metrics"""
    205		data["timestamp"] = self.get_timestamp()
    206
    207		self.metrics_list.append(data)
    208		self.metrics_file.write(json.dumps(data) + "\n")

log a dictionary of metrics

def artifact( self, path: pathlib.Path, type: str, aliases: list[str] | None = None, metadata: dict | None = None) -> None:
View Source
View on GitHub

    210	def artifact(
    211		self,
    212		path: Path,
    213		type: str,
    214		aliases: list[str] | None = None,
    215		metadata: dict | None = None,
    216	) -> None:
    217		"""log an artifact from a file"""
    218		artifact_dict: dict = dict(
    219			timestamp=self.get_timestamp(),
    220			path=path.as_posix(),
    221			type=type,
    222			aliases=aliases,
    223			metadata=metadata if metadata else {},
    224		)
    225
    226		self.artifacts_list.append(artifact_dict)
    227		self.artifacts_file.write(json.dumps(artifact_dict) + "\n")

log an artifact from a file

url: str
View Source
View on GitHub

    229	@property
    230	def url(self) -> str:
    231		"""Get the URL for the current logging run"""
    232		return self.run_path.as_posix()

Get the URL for the current logging run

run_path: pathlib.Path
View Source
View on GitHub

    234	@property
    235	def run_path(self) -> Path:
    236		"""Get the path to the current logging run"""
    237		return self._run_path

Get the path to the current logging run

def flush(self) -> None:
View Source
View on GitHub

    239	def flush(self) -> None:
    240		self.log_file.flush()
    241		self.metrics_file.flush()
    242		self.artifacts_file.flush()

Flush the logger

def finish(self) -> None:
View Source
View on GitHub

    244	def finish(self) -> None:
    245		self.message("closing logger")
    246
    247		self.log_file.flush()
    248		self.log_file.close()
    249
    250		self.metrics_file.flush()
    251		self.metrics_file.close()
    252
    253		self.artifacts_file.flush()
    254		self.artifacts_file.close()

Finish logging

##### Inherited Members

trnbl.loggers.base.TrainingLoggerBase
  get_mem_usage
  spinner_task