# -*- coding: utf-8 -*-
"""
Sleep Heart Health Study (SHHS) database
"""

import itertools
import re
import warnings
from datetime import datetime
from numbers import Real
from pathlib import Path
from typing import Any, Dict, List, Optional, Union, Tuple

import numpy as np
import pandas as pd
import scipy.signal as SS
import xmltodict as xtd
from tqdm.auto import tqdm

from ...cfg import DEFAULTS
from ...utils.misc import add_docstring
from ...utils.utils_interval import intervals_union
from ..base import NSRRDataBase, DataBaseInfo, PSGDataBaseMixin


__all__ = [
    "SHHS",
]


_SHHS_INFO = DataBaseInfo(
    title="""
    Sleep Heart Health Study
    """,
    about=r"""
    *ABOUT the dataset*:
    1. shhs1 (Visit 1):
        1.1. the baseline clinic visit and polysomnogram performed between November 1, 1995 and January 31, 1998
        1.2. in all, 6,441 men and women aged 40 years and older were enrolled
        1.3. 5,804 rows, down from the original 6,441 due to data sharing rules on certain cohorts and subjects
    2. shhs-interim-followup (Interim Follow-up):
        2.1. an interim clinic visit or phone call 2-3 years after baseline (shhs1)
        2.2. 5,804 rows, despite some subjects not having complete data, all original subjects are present in the dataset
    3. shhs2 (Visit 2):
        3.1. the follow-up clinic visit and polysomnogram performed between January 2001 and June 2003
        3.2. during this exam cycle 3, a second polysomnogram was obtained in 3,295 of the participants
        3.3. 4,080 rows, not all cohorts and subjects took part
    4. shhs-cvd (CVD Outcomes):
        4.1. the tracking of adjudicated heart health outcomes (e.g. stroke, heart attack) between baseline (shhs1) and 2008-2011 (varies by parent cohort)
        4.2. 5,802 rows, outcomes data were not provided on all subjects
    5. shhs-cvd-events (CVD Outcome Events):
        5.1. event-level details for the tracking of heart health outcomes (shhs-cvd)
        5.2. 4,839 rows, representing individual events

    6. ECG was sampled at 125 Hz in shhs1 and 250/256 Hz in shhs2
    7. `annotations-events-nsrr` and `annotations-events-profusion`: annotation files both contain xml files, the former processed in the EDF Editor and Translator tool, the latter exported from Compumedics Profusion
    8. about 10% of the records have HRV (including sleep stages and sleep events) annotations

    *DATA Analysis Tips*:
    1. Respiratory Disturbance Index (RDI):
        1.1. A number of RDI variables exist in the data set. These variables are highly skewed.
        1.2. log-transformation is recommended, among which the following transformation performed best, at least in some subsets:
            NEWVA = log(OLDVAR + 0.1)
    2. Obstructive Apnea Index (OAI):
        2.1. There is one OAI index in the data set. It reflects obstructive events associated with a 4% desaturation or arousal. Nearly 30% of the cohort has a zero value for this variable
        2.2. Dichotomization is suggested (e.g. >=3 or >=4 events per hour indicates positive)
    3. Central Apnea Index (CAI):
        3.1. Several variables describe central breathing events, with different thresholds for desaturation and requirement/non-requirement of arousals. ~58% of the cohort have zero values
        3.2. Dichotomization is suggested (e.g. >=3 or >=4 events per hour indicates positive)
    4. Sleep Stages:
        4.1. Stage 1 and stage 3-4 are not normally distributed, but stage 2 and REM sleep are.
        4.2. To use these data as continuous dependent variables, stages 1 and 3-4 must be transformed. The following formula is suggested:
            –log(-log(val/100+0.001))
    5. Sleep time below 90% O2:
        5.1. Percent of total sleep time with oxygen levels below 75%, 80%, 85% and 90% were recorded
        5.2. Dichotomization is suggested (e.g. >5% and >10% of sleep time with oxygen levels below a specific O2 level indicates positive)

    *ABOUT signals*: ([ref 10](#ref10))
    1. C3/A2 and C4/A1 EEGs, sampled at 125 Hz
    2. right and left electrooculograms (EOGs), sampled at 50 Hz
    3. a bipolar submental electromyogram (EMG), sampled at 125 Hz
    4. thoracic and abdominal excursions (THOR and ABDO), recorded by inductive plethysmography bands and sampled at 10 Hz
    5. "AIRFLOW" detected by a nasal-oral thermocouple, sampled at 10 Hz
    6. finger-tip pulse oximetry sampled at 1 Hz
    7. ECG from a bipolar lead, sampled at 125 Hz for most SHHS-1 studies and 250 (and 256?) Hz for SHHS-2 studies
    8. Heart rate (PR) derived from the ECG and sampled at 1 Hz
    9. body position (using a mercury gauge sensor)
    10. ambient light (on/off, by a light sensor secured to the recording garment)

    *ABOUT annotations* (NOT including "nsrrid","visitnumber","pptid" etc.):
    1. hrv annotations: (in csv files, ref. [2])
        Start__sec_ --- 5 minute window start time
        NN_RR	    --- Ratio of consecutive normal sinus beats (NN) over all cardiac inter-beat (RR) intervals
        AVNN	    --- Mean of all normal sinus to normal sinus interbeat intervals (NN)
        IHR	        --- Instantaneous heart rate
        SDNN	    --- Standard deviation of all normal sinus to normal sinus interbeat (NN) intervals
        SDANN	    --- Standard deviation of the averages of normal sinus to normal sinus interbeat (NN) intervals in all 5-minute segments
        SDNNIDX	    --- Mean of the standard deviations of normal sinus to normal sinus interbeat (NN) intervals in all 5-minute segments
        rMSSD	    --- Square root of the mean of the squares of difference between adjacent normal sinus to normal sinus interbeat (NN) intervals
        pNN10	    --- Percentage of differences between adjacent normal sinus to normal sinus interbeat (NN) intervals that are >10 ms
        pNN20	    --- Percentage of differences between adjacent normal sinus to normal sinus interbeat (NN) intervals that are >20 ms
        pNN30	    --- Percentage of differences between adjacent normal sinus to normal sinus interbeat (NN) intervals that are >30 ms
        pNN40	    --- Percentage of differences between adjacent normal sinus to normal sinus interbeat (NN) intervals that are >40 ms
        pNN50	    --- Percentage of differences between adjacent normal sinus to normal sinus interbeat (NN) intervals that are >50 ms
        tot_pwr	    --- Total normal sinus to normal sinus interbeat (NN) interval spectral power up to 0.4 Hz
        ULF	        --- Ultra-low frequency power, the normal sinus to normal sinus interbeat (NN) interval spectral power between 0 and 0.003 Hz
        VLF	        --- Very low frequency power, the normal sinus to normal sinus interbeat (NN) interval spectral power between 0.003 and 0.04 Hz
        LF	        --- Low frequency power, the normal sinus to normal sinus interbeat (NN) interval spectral power between 0.04 and 0.15 Hz
        HF	        --- High frequency power, the normal sinus to normal sinus interbeat (NN) interval spectral power between 0.15 and 0.4 Hz
        LF_HF	    --- The ratio of low to high frequency power
        LF_n	    --- Low frequency power (normalized)
        HF_n	    --- High frequency power (normalized)
    2. wave delineation annotations: (in csv files, NOTE: see "CAUTION" by the end of this part, ref. [2])
        RPoint	    --- Sample Number indicating R Point (peak of QRS)
        Start	    --- Sample Number indicating start of beat
        End	        --- Sample Number indicating end of beat
        STLevel1    --- Level of ECG 1 in Raw data ( 65536 peak to peak rawdata = 10mV peak to peak)
        STSlope1    --- Slope of ECG 1 stored as int and to convert to a double divide raw value by 1000.0
        STLevel2    --- Level of ECG 2 in Raw data ( 65536 peak to peak rawdata = 10mV peak to peak)
        STSlope2    --- Slope of ECG 2 stored as int and to convert to a double divide raw value by 1000.0
        Manual      --- (True / False) True if record was manually inserted
        Type        --- Type of beat (0 = Artifact / 1 = Normal Sinus Beat / 2 = VE / 3 = SVE)
        Class       --- no longer used
        PPoint      --- Sample Number indicating peak of the P wave (-1 if no P wave detected)
        PStart      --- Sample Number indicating start of the P wave
        PEnd        --- Sample Number indicating end of the P wave
        TPoint      --- Sample Number indicating peak of the T wave (-1 if no T wave detected)
        TStart      --- Sample Number indicating start of the T wave
        TEnd        --- Sample Number indicating end of the T wave
        TemplateID  --- The ID of the template to which this beat has been assigned (-1 if not assigned to a template)
        nsrrid      --- nsrrid of this record
        samplingrate--- frequency of the ECG signal of this record
        seconds     --- Number of seconds from beginning of recording to R-point (Rpoint / sampling rate)
        epoch       --- Epoch (30 second) number
        rpointadj   --- R Point adjusted sample number (RPoint * (samplingrate/256))
    CAUTION: all the above sampling numbers except for rpointadj assume 256 Hz, while the rpointadj column has been added to provide an adjusted sample number based on the actual sampling rate.
    3. event annotations: (in xml files)
        TODO
    4. event_profusion annotations: (in xml files)
        TODO

    *DEFINITION of concepts in sleep study*:
    1. Arousal: (ref. [3],[4])
        1.1. interruptions of sleep lasting 3 to 15 seconds
        1.2. can occur spontaneously or as a result of sleep-disordered breathing or other sleep disorders
        1.3. sends you back to a lighter stage of sleep
        1.4. if the arousal last more than 15 seconds, it becomes an awakening
        1.5. the higher the arousal index (occurrences per hour), the more tired you are likely to feel, though people vary in their tolerance of sleep disruptions
    2. Central Sleep Apnea (CSA): (ref. [3],[5],[6])
        2.1. breathing repeatedly stops and starts during sleep
        2.2. occurs because your brain (central nervous system) doesn't send proper signals to the muscles that control your breathing, which is point that differs from obstructive sleep apnea
        2.3. may occur as a result of other conditions, such as heart failure, stroke, high altitude, etc.
    3. Obstructive Sleep Apnea (OSA): (ref. [3],[7])
        3.1. occurs when throat muscles intermittently relax and block upper airway during sleep
        3.2. a noticeable sign of obstructive sleep apnea is snoring
    4. Complex (Mixed) Sleep Apnea: (ref. [3])
        4.1. combination of both CSA and OSA
        4.2. exact mechanism of the loss of central respiratory drive during sleep in OSA is unknown but is most likely related to incorrect settings of the CPAP (Continuous Positive Airway Pressure) treatment and other medical conditions the person has
    5. Hypopnea:
        overly shallow breathing or an abnormally low respiratory rate. Hypopnea is defined by some to be less severe than apnea (the complete cessation of breathing)
    6. Apnea Hypopnea Index (AHI): to write
        6.1. used to indicate the severity of OSA
        6.2. number of apneas or hypopneas recorded during the study per hour of sleep
        6.3. based on the AHI, the severity of OSA is classified as follows
            - none/minimal: AHI < 5 per hour
            - mild: AHI ≥ 5, but < 15 per hour
            - moderate: AHI ≥ 15, but < 30 per hour
            - severe: AHI ≥ 30 per hour
    7. Oxygen Desaturation:
        7.1. used to indicate the severity of OSA
        7.2. reductions in blood oxygen levels (desaturation)
        7.3. at sea level, a normal blood oxygen level (saturation) is usually 96 - 97%
        7.4. (no generally accepted classifications for severity of oxygen desaturation)
            - mild: >= 90%
            - moderate: 80% - 89%
            - severe: < 80%
    """,
    usage=[
        "Sleep stage",
        "Sleep apnea",
    ],
    issues="""
    1. `Start__sec_` might not be the start time, but rather the end time, of the 5 minute windows in some records
    2. the current version "0.15.0" removed EEG spectral summary variables
    """,
    references=[
        "https://sleepdata.org/datasets/shhs/pages/",
        "https://sleepdata.org/datasets/shhs/pages/13-hrv-analysis.md",
        "https://en.wikipedia.org/wiki/Sleep_apnea",
        "https://www.sleepapnea.org/treat/getting-sleep-apnea-diagnosis/sleep-study-details/",
        "https://www.mayoclinic.org/diseases-conditions/central-sleep-apnea/symptoms-causes/syc-20352109",
        "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2287191/",
        "https://www.mayoclinic.org/diseases-conditions/obstructive-sleep-apnea/symptoms-causes/syc-20352090",
        "https://en.wikipedia.org/wiki/Hypopnea",
        "http://healthysleep.med.harvard.edu/sleep-apnea/diagnosing-osa/understanding-results",
        "https://sleepdata.org/datasets/shhs/pages/full-description.md",
    ],
    doi=[
        "10.1093/jamia/ocy064",
    ],  # PMID: 9493915 not added
)


@add_docstring(_SHHS_INFO.format_database_docstring())
class SHHS(NSRRDataBase, PSGDataBaseMixin):
    """ """

    __name__ = "SHHS"

    def __init__(
        self,
        db_dir: Optional[Union[str, Path]] = None,
        working_dir: Optional[Union[str, Path]] = None,
        verbose: int = 1,
        **kwargs: Any,
    ) -> None:
        """
        Parameters
        ----------
        db_dir: str or Path, optional,
            storage path of the database
        working_dir: str or Path, optional,
            working directory, to store intermediate files and log file
        verbose: int, default 1
            log verbosity
        kwargs: auxilliary key word arguments

        """
        super().__init__(
            db_name="SHHS",
            db_dir=db_dir,
            working_dir=working_dir,
            verbose=verbose,
            **kwargs,
        )

        self.__create_constants(**kwargs)

        # `current_version` will be when calling `_ls_rec`
        self.current_version = kwargs.get("current_version", "0.19.0")
        self.version_pattern = "\\d+\\.\\d+\\.\\d+"

        self.rec_name_pattern = "^shhs[12]\\-\\d{6}$"

        self.psg_data_path = None
        self.ann_path = None
        self.hrv_ann_path = None
        self.eeg_ann_path = None
        self.wave_deli_path = None
        self.event_ann_path = None
        self.event_profusion_ann_path = None
        self.form_paths()

        self._df_records = pd.DataFrame()
        self._all_records = []
        self.rec_with_hrv_summary_ann = []
        self.rec_with_hrv_detailed_ann = []
        self.rec_with_event_ann = []
        self.rec_with_event_profusion_ann = []
        self.rec_with_rpeaks_ann = []
        self._tables = {}
        self._ls_rec()

        self.fs = None
        self.file_opened = None

    def form_paths(self) -> None:
        """ """
        self.psg_data_path = self.db_dir / "polysomnography" / "edfs"
        self.ann_path = self.db_dir / "datasets"
        self.hrv_ann_path = self.ann_path / "hrv-analysis"
        self.eeg_ann_path = self.ann_path / "eeg-spectral-analysis"
        self.wave_deli_path = self.db_dir / "polysomnography" / "annotations-rpoints"
        self.event_ann_path = (
            self.db_dir / "polysomnography" / "annotations-events-nsrr"
        )
        self.event_profusion_ann_path = (
            self.db_dir / "polysomnography" / "annotations-events-profusion"
        )

    def _ls_rec(self) -> None:
        """ """
        self.logger.info("Finding `edf` records....")
        self._df_records = pd.DataFrame()
        self._df_records["path"] = sorted(self.db_dir.rglob("*.edf"))

        if self._subsample is not None:
            size = min(
                len(self._df_records),
                max(1, int(round(self._subsample * len(self._df_records)))),
            )
            self._df_records = self._df_records.sample(
                n=size, random_state=DEFAULTS.SEED, replace=False
            )

        # if self._df_records is non-empty, call `form_paths` again if necessary
        # typically path for a record is like:
        # self.db_dir / "polysomnography/edfs/shhs1/shhs1-200001.edf"
        if (
            len(self._df_records) > 0
            and self._df_records.iloc[0]["path"].parents[3] != self.db_dir
        ):
            self.db_dir = self._df_records.iloc[0]["path"].parents[3]
            self.form_paths()

        # get other columns
        self._df_records["record"] = self._df_records["path"].apply(lambda x: x.stem)
        self._df_records["tranche"] = self._df_records["record"].apply(
            lambda x: x.split("-")[0]
        )
        self._df_records["visitnumber"] = self._df_records["record"].apply(
            lambda x: int(x.split("-")[0][4:])
        )
        self._df_records["nsrrid"] = self._df_records["record"].apply(
            lambda x: int(x.split("-")[1])
        )

        # auxiliary and annotation files
        if not self._df_records.empty:
            for key in self.extension:
                self._df_records[key] = self._df_records.apply(
                    lambda row: self.folder_or_file[key]
                    / row["tranche"]
                    / (row["record"] + self.extension[key]),
                    axis=1,
                )

        self._df_records.set_index("record", inplace=True)
        self._all_records = self._df_records.index.tolist()

        # update `current_version`
        if self.ann_path.is_dir():
            for file in self.ann_path.iterdir():
                if (
                    file.is_file()
                    and len(re.findall(self.version_pattern, file.name)) > 0
                ):
                    self.current_version = re.findall(self.version_pattern, file.name)[
                        0
                    ]
                    break

            self.logger.info("Loading tables....")
            # gather tables in self.ann_path and in self.hrv_ann_path
            for file in itertools.chain(
                self.ann_path.glob("*.csv"), self.hrv_ann_path.glob("*.csv")
            ):
                if not file.suffix == ".csv":
                    continue
                table_name = file.stem.replace(f"-{self.current_version}", "")
                try:
                    self._tables[table_name] = pd.read_csv(file, low_memory=False)
                except UnicodeDecodeError:
                    self._tables[table_name] = pd.read_csv(
                        file, low_memory=False, encoding="latin-1"
                    )

        self.logger.info("Finding records with HRV annotations....")
        # find records with hrv annotations
        self.rec_with_hrv_summary_ann = []
        for table_name in ["shhs1-hrv-summary", "shhs2-hrv-summary"]:
            if table_name in self._tables:
                self.rec_with_hrv_summary_ann.extend(
                    [
                        f"shhs{int(row['visitnumber'])}-{int(row['nsrrid'])}"
                        for _, row in self._tables[table_name].iterrows()
                    ]
                )
        self.rec_with_hrv_summary_ann = sorted(list(set(self.rec_with_hrv_summary_ann)))
        self.rec_with_hrv_detailed_ann = []
        for table_name in ["shhs1-hrv-5min", "shhs2-hrv-5min"]:
            if table_name in self._tables:
                self.rec_with_hrv_detailed_ann.extend(
                    [
                        f"shhs{int(row['visitnumber'])}-{int(row['nsrrid'])}"
                        for _, row in self._tables[table_name].iterrows()
                    ]
                )
        self.rec_with_hrv_detailed_ann = sorted(
            list(set(self.rec_with_hrv_detailed_ann))
        )

        self.logger.info("Finding records with rpeaks annotations....")
        # find available rpeak annotation files
        self.rec_with_rpeaks_ann = sorted(
            [
                f.stem.replace("-rpoint", "")
                for f in self.wave_deli_path.rglob("shhs*-rpoint.csv")
            ]
        )

        self.logger.info("Finding records with event annotations....")
        # find available event annotation files
        self.rec_with_event_ann = sorted(
            [
                f.stem.replace("-nsrr", "")
                for f in self.event_ann_path.rglob("shhs*-nsrr.xml")
            ]
        )
        self.rec_with_event_profusion_ann = sorted(
            [
                f.stem.replace("-profusion", "")
                for f in self.event_profusion_ann_path.rglob("shhs*-profusion.xml")
            ]
        )

        self._df_records["available_signals"] = None
        if not self.lazy:
            self.get_available_signals(None)

        # END OF `_ls_rec`

    def list_table_names(self) -> List[str]:
        """list available table names"""
        return list(self._tables.keys())

    def get_table(self, table_name: str) -> pd.DataFrame:
        """
        get table by name

        Parameters
        ----------
        table_name: str,
            table name,
            for available table names, call method `list_table_names`

        Returns
        -------
        table: pd.DataFrame

        """
        return self._tables[table_name]

    def update_sleep_stage_names(self) -> None:
        """ """
        if self.sleep_stage_protocol == "aasm":
            nb_stages = 5
        elif self.sleep_stage_protocol == "simplified":
            nb_stages = 4
        elif self.sleep_stage_protocol == "shhs":
            nb_stages = 6
        else:
            raise ValueError(f"No stage protocol named `{self.sleep_stage_protocol}`")

        self.sleep_stage_names = self.all_sleep_stage_names[:nb_stages]

    def get_subject_id(self, rec: Union[str, int]) -> int:
        """
        get a unique subject id for a record

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`

        Returns
        -------
        pid, int, `subject_id` derived from `rec`

        """
        if isinstance(rec, int):
            rec = self[rec]
        head_shhs1, head_shhs2v3, head_shhs2v4 = "30000", "30001", "30002"
        tranche, nsrrid, visitnumber = [
            self.split_rec_name(rec)[k] for k in ["tranche", "nsrrid", "visitnumber"]
        ]
        if visitnumber == "2":
            raise ValueError(
                "SHHS2 has two different sampling frequencies, "
                "currently could not be distinguished using only `rec`"
            )
        pid = int(head_shhs1 + str(visitnumber) + str(nsrrid))
        return pid

    def get_available_signals(
        self, rec: Union[str, int, None]
    ) -> Union[List[str], None]:
        """
        get available signals for a record;
        if input `rec` is None, will find available signals for all records,
        and assign to `self._df_records['available_signals']`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`

        Returns
        -------
        available_signals: list of str,
            names of available signals for `rec`

        """
        if rec is None:
            # iterrows with tqdm
            for _, row in tqdm(
                self._df_records.iterrows(),
                total=len(self._df_records),
                desc="Finding available signals",
                unit="record",
                dynamic_ncols=True,
                mininterval=1.0,
                disable=(self.verbose < 1),
            ):
                rec = row.name
                if self._df_records.loc[rec, "available_signals"] is not None:
                    continue
                available_signals = self.get_available_signals(rec)
                self._df_records.at[rec, "available_signals"] = available_signals
            return

        if isinstance(rec, int):
            rec = self[rec]

        if rec in self._df_records.index:
            available_signals = self._df_records.loc[rec, "available_signals"]
            if available_signals is not None and len(available_signals) > 0:
                return available_signals

            frp = self.get_absolute_path(rec)
            try:
                # perhaps broken file
                # or the downloading is not finished
                self.safe_edf_file_operation("open", frp)
            except OSError:
                return None
            available_signals = [s.lower() for s in self.file_opened.getSignalLabels()]
            self.safe_edf_file_operation("close")
            self._df_records.at[rec, "available_signals"] = available_signals
            self.all_signals = self.all_signals.union(set(available_signals))
        else:
            available_signals = []
        return available_signals

    def split_rec_name(self, rec: Union[str, int]) -> Dict[str, Union[str, int]]:
        """
        split `rec` into `tranche`, `visitnumber`, `nsrrid`

        Parameters
        ----------
        rec: str,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`

        Returns
        -------
        dict, keys: "tranche", "visitnumber", "nsrrid"

        """
        if isinstance(rec, int):
            rec = self[rec]
        assert isinstance(rec, str) and re.match(
            self.rec_name_pattern, rec
        ), f"Invalid record name: `{rec}`"
        tranche, nsrrid = rec.split("-")
        visitnumber = tranche[-1]
        return {
            "tranche": tranche,
            "visitnumber": int(visitnumber),
            "nsrrid": int(nsrrid),
        }

    def get_visitnumber(self, rec: Union[str, int]) -> int:
        """
        get `visitnumber` from `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`

        Returns
        -------
        int, visit number extracted from `rec`

        """
        return self.split_rec_name(rec)["visitnumber"]

    def get_tranche(self, rec: Union[str, int]) -> str:
        """
        get `tranche` ("shhs1" or "shhs2") from `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`

        Returns
        -------
        str, tranche extracted from `rec`

        """
        return self.split_rec_name(rec)["tranche"]

    def get_nsrrid(self, rec: Union[str, int]) -> int:
        """
        get `nsrrid` from `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`

        Returns
        -------
        int, nsrrid extracted from `rec`

        """
        return self.split_rec_name(rec)["nsrrid"]

    def get_fs(
        self,
        rec: Union[str, int],
        sig: str = "ECG",
        rec_path: Optional[Union[str, Path]] = None,
    ) -> Real:
        """
        get the sampling frequency of a signal of a record

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        sig: str, default "ECG",
            signal name or annotation name (e.g. "rpeak"),
            some annotation files (*-rpeak.csv) have sampling frequency column
        rec_path: str or Path, optional,
            path of the file which contains the PSG data,
            if not given, default path will be used

        Returns
        -------
        fs, real number,
            the sampling frequency of the signal `sig` of the record `rec`.
            If corresponding signal (.edf) file is not available,
            or the signal file does not contain the signal `sig`,
            return -1

        """
        if isinstance(rec, int):
            rec = self[rec]
        sig = self.match_channel(sig, raise_error=False)
        assert sig in self.all_signals.union({"rpeak"}), f"Invalid signal name: `{sig}`"
        if sig.lower() == "rpeak":
            df_rpeaks_with_type_info = self.load_wave_delineation_ann(rec)
            if df_rpeaks_with_type_info.empty:
                self.logger.info(
                    f"Rpeak annotation file corresponding to `{rec}` is not available."
                )
                return -1
            return df_rpeaks_with_type_info.iloc[0]["samplingrate"]

        frp = self.get_absolute_path(rec, rec_path)
        if not frp.exists():
            self.logger.info(
                f"Signal (.edf) file corresponding to `{rec}` is not available."
            )
            return -1
        self.safe_edf_file_operation("open", frp)
        sig = self.match_channel(sig)
        available_signals = [s.lower() for s in self.file_opened.getSignalLabels()]
        if sig not in available_signals:
            self.logger.info(
                f"Signal `{sig}` is not available in signal file corresponding to `{rec}`."
            )
            return -1
        chn_num = available_signals.index(sig)
        fs = self.file_opened.getSampleFrequency(chn_num)
        self.safe_edf_file_operation("close")
        return fs

    def get_chn_num(
        self,
        rec: Union[str, int],
        sig: str = "ECG",
        rec_path: Optional[Union[str, Path]] = None,
    ) -> int:
        """
        get the index of the channel of the signal `sig` of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        sig: str, default "ECG",
            signal name
        rec_path: str or Path, optional,
            path of the file which contains the PSG data,
            if not given, default path will be used

        Returns
        -------
        chn_num, int,
            the index of channel of the signal `sig` of the record `rec`.
            If corresponding signal (.edf) file is not available,
            or the signal file does not contain the signal `sig`,
            return -1

        """
        sig = self.match_channel(sig)
        available_signals = self.get_available_signals(rec)
        if sig not in available_signals:
            if isinstance(rec, int):
                rec = self[rec]
            self.logger.info(
                f"Signal (.edf) file corresponding to `{rec}` is not available, or"
                f"signal `{sig}` is not available in signal file corresponding to `{rec}`."
            )
            return -1
        chn_num = available_signals.index(self.match_channel(sig))
        return chn_num

    def match_channel(self, channel: str, raise_error: bool = True) -> str:
        """
        match the channel name to the standard channel name in SHHS

        Parameters
        ----------
        channel: str,
            channel name
        raise_error: bool, default True,
            whether to raise error if no match is found.
            If False, return the input `channel` directly

        Returns
        -------
        sig: str,
            the standard channel name in SHHS.
            If no match is found, and `raise_error` is False,
            return the input `channel` directly

        """
        if channel.lower() in self.all_signals:
            return channel.lower()
        if raise_error:
            raise ValueError(f"No channel named `{channel}`")
        return channel

    def get_absolute_path(
        self,
        rec: Union[str, int],
        rec_path: Optional[Union[str, Path]] = None,
        rec_type: str = "psg",
    ) -> Path:
        """
        get the absolute path of the record `rec` with type `rec_type`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        rec_path: str or Path, optional,
            path of the file which contains the desired data,
            if not given, default path will be used
        rec_type: str, default "psg",
            record type, data or annotations

        Returns
        -------
        rp: Path,
            absolute path of the record `rec` with type `rec_type`

        """
        if rec_path is not None:
            rp = Path(rec_path)
            return rp

        assert rec_type in self.folder_or_file, (
            "`rec_type` should be one of "
            f"`{list(self.folder_or_file.keys())}`, but got `{rec_type}`"
        )

        if isinstance(rec, int):
            rec = self[rec]

        tranche, nsrrid = [self.split_rec_name(rec)[k] for k in ["tranche", "nsrrid"]]
        # rp = self._df_records.loc[rec, rec_type]
        rp = (
            self.folder_or_file[rec_type] / tranche / f"{rec}{self.extension[rec_type]}"
        )
        return rp

    def database_stats(self) -> None:
        """ """
        raise NotImplementedError

    def show_rec_stats(
        self, rec: Union[str, int], rec_path: Optional[Union[str, Path]] = None
    ) -> None:
        """
        print the statistics of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        rec_path: str or Path, optional,
            path of the file which contains the PSG data,
            if not given, default path will be used

        """
        frp = self.get_absolute_path(rec, rec_path, rec_type="psg")
        self.safe_edf_file_operation("open", frp)
        for chn, lb in enumerate(self.file_opened.getSignalLabels()):
            print("SignalLabel:", lb)
            print("Prefilter:", self.file_opened.getPrefilter(chn))
            print("Transducer:", self.file_opened.getTransducer(chn))
            print("PhysicalDimension:", self.file_opened.getPhysicalDimension(chn))
            print("SampleFrequency:", self.file_opened.getSampleFrequency(chn))
            print("*" * 40)
        self.safe_edf_file_operation("close")

    def load_psg_data(
        self,
        rec: Union[str, int],
        channel: str = "all",
        rec_path: Optional[Union[str, Path]] = None,
        sampfrom: Optional[Real] = None,
        sampto: Optional[Real] = None,
        fs: Optional[int] = None,
        physical: bool = True,
    ) -> Union[Dict[str, Tuple[np.ndarray, Real]], Tuple[np.ndarray, Real]]:
        """
        load PSG data of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        channel: str, default "all",
            name of the channel of PSG,
            if is "all", then all channels will be returned
        rec_path: str or Path, optional,
            path of the file which contains the PSG data,
            if not given, default path will be used
        sampfrom: real number, optional,
            start time (units in seconds) of the data to be loaded,
            valid only when `channel` is some specific channel
        sampto: real number, optional,
            end time (units in seconds) of the data to be loaded,
            valid only when `channel` is some specific channel
        fs: real number, optional,
            if not None, the loaded data will be resampled to this frequency
            valid only when `channel` is some specific channel
        physical: bool, default True,
            if True, then the data will be converted to physical units
            otherwise, the data will be in digital units

        Returns
        -------
        dict,
            keys: PSG channel names;
            values: PSG data and sampling frequency

        (np.ndarray, real number):
            PSG data of the channel `channel` and its sampling frequency

        """
        chn = self.match_channel(channel) if channel.lower() != "all" else "all"
        frp = self.get_absolute_path(rec, rec_path, rec_type="psg")
        self.safe_edf_file_operation("open", frp)

        if chn == "all":
            ret_data = {
                k: (
                    self.file_opened.readSignal(idx, digital=not physical),
                    self.file_opened.getSampleFrequency(idx),
                )
                for idx, k in enumerate(self.file_opened.getSignalLabels())
            }
        else:
            all_signals = [s.lower() for s in self.file_opened.getSignalLabels()]
            assert (
                chn in all_signals
            ), f"`channel` should be one of `{self.file_opened.getSignalLabels()}`, but got `{chn}`"
            idx = all_signals.index(chn)
            data_fs = self.file_opened.getSampleFrequency(idx)
            data = self.file_opened.readSignal(idx, digital=not physical)
            # the `readSignal` method of `EdfReader` does NOT treat
            # the parameters `start` and `n` correctly
            # so we have to do it manually
            if sampfrom is not None:
                idx_from = int(round(sampfrom * data_fs))
            else:
                idx_from = 0
            if sampto is not None:
                idx_to = int(round(sampto * data_fs))
            else:
                idx_to = len(data)
            data = data[idx_from:idx_to]
            if fs is not None and fs != data_fs:
                data = SS.resample_poly(data, fs, data_fs).astype(data.dtype)
                data_fs = fs
            ret_data = (data, data_fs)

        self.safe_edf_file_operation("close")

        return ret_data

    def load_ecg_data(
        self,
        rec: Union[str, int],
        rec_path: Optional[Union[str, Path]] = None,
        sampfrom: Optional[int] = None,
        sampto: Optional[int] = None,
        data_format: str = "channel_first",
        units: Union[str, type(None)] = "mV",
        fs: Optional[int] = None,
    ) -> Tuple[np.ndarray, Real]:
        """
        load ECG data of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        rec_path: str or Path, optional,
            path of the file which contains the ECG data,
            if not given, default path will be used
        sampfrom: int, optional,
            start index of the data to be loaded
        sampto: int, optional,
            end index of the data to be loaded
        data_format: str, default "channel_first",
            format of the ecg data,
            "channel_last" (alias "lead_last"), or
            "channel_first" (alias "lead_first"), or
            "flat" (alias "plain") which is valid only when `leads` is a single lead
        units: str or None, default "mV",
            units of the output signal, can also be "μV", with aliases of "uV", "muV";
            None for digital data, without digital-to-physical conversion
        fs: real number, optional,
            if not None, the loaded data will be resampled to this frequency

        Returns
        -------
        data: ndarray,
            ECG data
        data_fs: real number,
            sampling frequency

        """
        allowed_data_format = [
            "channel_first",
            "lead_first",
            "channel_last",
            "lead_last",
            "flat",
            "plain",
        ]
        assert (
            data_format.lower() in allowed_data_format
        ), f"`data_format` should be one of `{allowed_data_format}`, but got `{data_format}`"
        allowed_units = ["mv", "uv", "μv", "muv"]
        assert (
            units is None or units.lower() in allowed_units
        ), f"`units` should be one of `{allowed_units}` or None, but got `{units}`"

        data, data_fs = self.load_psg_data(
            rec=rec,
            channel="ecg",
            rec_path=rec_path,
            sampfrom=sampfrom,
            sampto=sampto,
            fs=fs,
            physical=units is not None,
        )
        data = data.astype(DEFAULTS.DTYPE.NP)

        if units is not None and units.lower() in ["μv", "uv", "muv"]:
            data *= 1e3
        if data_format.lower() in ["channel_first", "lead_first"]:
            data = data[np.newaxis, :]
        elif data_format.lower() in ["channel_last", "lead_last"]:
            data = data[:, np.newaxis]

        return data, data_fs

    @add_docstring(
        "NOTE: one should call `load_psg_data` to load other channels.", mode="append"
    )
    @add_docstring(load_ecg_data.__doc__)
    def load_data(
        self,
        rec: Union[str, int],
        rec_path: Optional[Union[str, Path]] = None,
        sampfrom: Optional[int] = None,
        sampto: Optional[int] = None,
        data_format: str = "channel_first",
        units: Union[str, type(None)] = "mV",
        fs: Optional[int] = None,
    ) -> Tuple[np.ndarray, Real]:
        """alias of `load_ecg_data`"""
        return self.load_ecg_data(
            rec=rec,
            rec_path=rec_path,
            sampfrom=sampfrom,
            sampto=sampto,
            data_format=data_format,
            units=units,
            fs=fs,
        )

    def load_ann(
        self,
        rec: Union[str, int],
        ann_type: str,
        ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> Union[np.ndarray, pd.DataFrame, dict]:
        """
        load annotations of specific type of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        ann_type: str,
            type of the annotation, can be
            "event", "event_profusion", "hrv_summary", "hrv_detailed",
            "sleep", "sleep_stage", "sleep_event", "apnea" (alias "sleep_apnea"),
            "wave_delineation", "rpeak", "rr", "nn"
        ann_path: str or Path, optional,
            path of the file which contains the annotations,
            if not given, default path will be used
        kwargs: dict,
            other arguments for specific annotation type

        Returns
        -------
        annotations: ndarray or DataFrame or dict,

        """
        if ann_type.lower() == "event":
            return self.load_event_ann(rec=rec, event_ann_path=ann_path, **kwargs)
        elif ann_type.lower() == "event_profusion":
            return self.load_event_profusion_ann(
                rec=rec, event_profusion_ann_path=ann_path, **kwargs
            )
        elif ann_type.lower() == "hrv_summary":
            return self.load_hrv_summary_ann(rec=rec, hrv_ann_path=ann_path, **kwargs)
        elif ann_type.lower() == "hrv_detailed":
            return self.load_hrv_detailed_ann(rec=rec, hrv_ann_path=ann_path, **kwargs)
        elif ann_type.lower() == "sleep":
            return self.load_sleep_ann(rec=rec, sleep_ann_path=ann_path, **kwargs)
        elif ann_type.lower() == "sleep_stage":
            return self.load_sleep_stage_ann(
                rec=rec, sleep_stage_ann_path=ann_path, **kwargs
            )
        elif ann_type.lower() == "sleep_event":
            return self.load_sleep_event_ann(
                rec=rec, sleep_event_ann_path=ann_path, **kwargs
            )
        elif ann_type.lower() in ["sleep_apnea", "apnea"]:
            return self.load_apnea_ann(rec=rec, apnea_ann_path=ann_path, **kwargs)
        elif ann_type.lower() == "wave_delineation":
            return self.load_wave_delineation_ann(
                rec=rec, wave_deli_path=ann_path, **kwargs
            )
        elif ann_type.lower() == "rpeak":
            return self.load_rpeak_ann(rec=rec, rpeak_ann_path=ann_path, **kwargs)
        elif ann_type.lower() in ["rr", "rr_interval"]:
            return self.load_rr_ann(rec=rec, rpeak_ann_path=ann_path, **kwargs)
        elif ann_type.lower() in ["nn", "nn_interval"]:
            return self.load_nn_ann(rec=rec, rpeak_ann_path=ann_path, **kwargs)

    def load_event_ann(
        self,
        rec: Union[str, int],
        event_ann_path: Optional[Union[str, Path]] = None,
        simplify: bool = False,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load event annotations of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        event_ann_path: str or Path, optional,
            path of the file which contains the events-nsrr annotations,
            if not given, default path will be used

        Returns
        -------
        df_events: DataFrame,
            event annotations

        """
        file_path = self.get_absolute_path(rec, event_ann_path, rec_type="event")
        if not file_path.exists():
            # rec not in `self.rec_with_event_ann`
            return pd.DataFrame()
        doc = xtd.parse(file_path.read_text())
        df_events = pd.DataFrame(
            doc["PSGAnnotation"]["ScoredEvents"]["ScoredEvent"][1:]
        )
        if simplify:
            df_events["EventType"] = df_events["EventType"].apply(
                lambda s: s.split("|")[1]
            )
            df_events["EventConcept"] = df_events["EventConcept"].apply(
                lambda s: s.split("|")[1]
            )
        for c in ["Start", "Duration", "SpO2Nadir", "SpO2Baseline"]:
            df_events[c] = df_events[c].apply(self.str_to_real_number)

        return df_events

    def load_event_profusion_ann(
        self,
        rec: Union[str, int],
        event_profusion_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> dict:
        """
        load `event_profusion` annotations of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        event_profusion_ann_path: str or Path, optional,
            path of the file which contains the events-profusion annotations,
            if not given, default path will be used

        Returns
        -------
        dict, with items "sleep_stage_list", "df_events"

        TODO
        ----
        merge "sleep_stage_list" and "df_events" into one DataFrame

        """
        file_path = self.get_absolute_path(
            rec, event_profusion_ann_path, rec_type="event_profusion"
        )
        if not file_path.exists():
            # rec not in `self.rec_with_event_profusion_ann`
            return {"sleep_stage_list": [], "df_events": pd.DataFrame()}
        doc = xtd.parse(file_path.read_text())
        sleep_stage_list = [
            int(ss) for ss in doc["CMPStudyConfig"]["SleepStages"]["SleepStage"]
        ]
        df_events = pd.DataFrame(doc["CMPStudyConfig"]["ScoredEvents"]["ScoredEvent"])
        for c in ["Start", "Duration", "LowestSpO2", "Desaturation"]:
            df_events[c] = df_events[c].apply(self.str_to_real_number)
        ret = {"sleep_stage_list": sleep_stage_list, "df_events": df_events}

        return ret

    def load_hrv_summary_ann(
        self,
        rec: Optional[Union[str, int]] = None,
        hrv_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load summary HRV annotations of the record `rec`

        Parameters
        ----------
        rec: str or int, optional,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        hrv_ann_path: str or Path, optional,
            path of the summary HRV annotation file,
            if not given, default path will be used

        Returns
        -------
        df_hrv_ann, DataFrame,
            if `rec` is not None, df_hrv_ann is the summary HRV annotations of `rec`;
            if `rec` is None, df_hrv_ann is the summary HRV annotations of all records
            that had HRV annotations (about 10% of all the records in SHHS)

        """
        if rec is None:
            df_hrv_ann = pd.concat(
                [
                    self._tables[table_name]
                    for table_name in ["shhs1-hrv-summary", "shhs2-hrv-summary"]
                    if table_name in self._tables
                ],
                ignore_index=True,
            )
            return df_hrv_ann

        if isinstance(rec, int):
            rec = self[rec]

        if rec not in self.rec_with_hrv_summary_ann:
            return pd.DataFrame()

        tranche, nsrrid = [self.split_rec_name(rec)[k] for k in ["tranche", "nsrrid"]]
        table_name = f"{tranche}-hrv-summary"
        df_hrv_ann = self._tables[table_name][
            self._tables[table_name].nsrrid == int(nsrrid)
        ].reset_index(drop=True)
        return df_hrv_ann

    def load_hrv_detailed_ann(
        self,
        rec: Union[str, int],
        hrv_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load detailed HRV annotations of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        hrv_ann_path: str or Path, optional,
            path of the detailed HRV annotation file,
            if not given, default path will be used

        Returns
        -------
        df_hrv_ann, DataFrame,
            detailed HRV annotations of `rec`

        """
        if isinstance(rec, int):
            rec = self[rec]
        if rec not in self.rec_with_hrv_detailed_ann:
            return pd.DataFrame()

        tranche, nsrrid = [self.split_rec_name(rec)[k] for k in ["tranche", "nsrrid"]]
        table_name = f"{tranche}-hrv-5min"
        df_hrv_ann = self._tables[table_name][
            self._tables[table_name].nsrrid == int(nsrrid)
        ].reset_index(drop=True)

        return df_hrv_ann

    def load_sleep_ann(
        self,
        rec: Union[str, int],
        source: str = "event",
        sleep_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> Union[pd.DataFrame, dict]:
        """
        load sleep annotations of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        source: str, default "event",
            source of the annotations, case insensitive,
            can be "hrv", "event", "event_profusion"
        sleep_ann_path: str or Path, optional,
            path of the file which contains the sleep annotations,
            if not given, default path will be used

        Returns
        -------
        df_sleep_ann, DataFrame or dict,
            all annotations on sleep of `rec`

        """
        if isinstance(rec, int):
            rec = self[rec]
        if source.lower() == "hrv":
            df_hrv_ann = self.load_hrv_detailed_ann(
                rec=rec, hrv_ann_path=sleep_ann_path
            )
            if not df_hrv_ann.empty:
                df_sleep_ann = df_hrv_ann[self.sleep_ann_keys_from_hrv].reset_index(
                    drop=True
                )
            else:
                df_sleep_ann = pd.DataFrame(columns=self.sleep_ann_keys_from_hrv)
            self.logger.debug(
                f"record `{rec}` has `{len(df_sleep_ann)}` sleep annotations from corresponding "
                f"hrv-5min (detailed) annotation file, with `{len(self.sleep_ann_keys_from_hrv)}` column(s)"
            )
        elif source.lower() == "event":
            df_event_ann = self.load_event_ann(
                rec, event_ann_path=sleep_ann_path, simplify=False
            )
            _cols = ["EventType", "EventConcept", "Start", "Duration", "SignalLocation"]
            if not df_event_ann.empty:
                df_sleep_ann = df_event_ann[_cols]
            else:
                df_sleep_ann = pd.DataFrame(columns=_cols)
            self.logger.debug(
                f"record `{rec}` has `{len(df_sleep_ann)}` sleep annotations from corresponding "
                f"event-nsrr annotation file, with `{len(_cols)}` column(s)"
            )
        elif source.lower() == "event_profusion":
            dict_event_ann = self.load_event_profusion_ann(rec)
            # temporarily finished
            # latter to make imporvements
            df_sleep_ann = dict_event_ann
            self.logger.debug(
                f"record `{rec}` has `{len(df_sleep_ann['df_events'])}` sleep event annotations "
                "from corresponding event-profusion annotation file, "
                f"with `{len(df_sleep_ann['df_events'].columns)}` column(s)"
            )
        else:
            raise ValueError(
                f"Source `{source}` not supported, "
                "only `hrv`, `event`, `event_profusion` are supported"
            )
        return df_sleep_ann

    def load_sleep_stage_ann(
        self,
        rec: Union[str, int],
        source: str = "event",
        sleep_stage_ann_path: Optional[Union[str, Path]] = None,
        sleep_stage_protocol: str = "aasm",
        with_stage_names: bool = True,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load sleep stage annotations of the record `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        source: str, default "event",
            source of the annotations, case insensitive,
            can be "hrv", "event", "event_profusion"
        sleep_stage_ann_path: str or Path, optional,
            path of the file which contains the sleep stage annotations,
            if not given, default path will be used
        sleep_stage_protocol: str, default "aasm",
            the protocol to classify sleep stages. currently can be "aasm", "simplified", "shhs"
            the only difference lies in the number of different stages of the NREM periods
        with_stage_names: bool, default True,
            as the argument name implies

        Returns
        -------
        df_sleep_stage_ann, DataFrame,
            all annotations on sleep stage of `rec`

        """
        if isinstance(rec, int):
            rec = self[rec]
        self.sleep_stage_protocol = sleep_stage_protocol
        self.update_sleep_stage_names()

        df_sleep_ann = self.load_sleep_ann(
            rec=rec, source=source, sleep_ann_path=sleep_stage_ann_path
        )

        df_sleep_stage_ann = pd.DataFrame(columns=self.sleep_stage_keys)
        if source.lower() == "hrv":
            df_tmp = df_sleep_ann[self.sleep_stage_ann_keys_from_hrv].reset_index(
                drop=True
            )
            for _, row in df_tmp.iterrows():
                start_sec = row["Start__sec_"]
                l_start_sec = np.arange(
                    start_sec,
                    start_sec + self.hrv_ann_epoch_len_sec,
                    self.sleep_epoch_len_sec,
                )
                l_sleep_stage = np.array(
                    [
                        row[self.sleep_stage_ann_keys_from_hrv[i]]
                        for i in range(
                            1,
                            1 + self.hrv_ann_epoch_len_sec // self.sleep_epoch_len_sec,
                        )
                    ]
                )
                df_to_concat = pd.DataFrame(
                    {"start_sec": l_start_sec, "sleep_stage": l_sleep_stage}
                )
                df_sleep_stage_ann = pd.concat(
                    [df_sleep_stage_ann, df_to_concat], axis=0, ignore_index=True
                )
        elif source.lower() == "event":
            df_tmp = df_sleep_ann[df_sleep_ann["EventType"] == "Stages|Stages"][
                ["EventConcept", "Start", "Duration"]
            ].reset_index(drop=True)
            df_tmp["EventConcept"] = df_tmp["EventConcept"].apply(
                lambda s: int(s.split("|")[1])
            )
            for _, row in df_tmp.iterrows():
                start_sec = int(row["Start"])
                duration = int(row["Duration"])
                l_start_sec = np.arange(
                    start_sec, start_sec + duration, self.sleep_epoch_len_sec
                )
                l_sleep_stage = np.full(
                    shape=len(l_start_sec), fill_value=int(row["EventConcept"])
                )
                df_to_concat = pd.DataFrame(
                    {"start_sec": l_start_sec, "sleep_stage": l_sleep_stage}
                )
                df_sleep_stage_ann = pd.concat(
                    [df_sleep_stage_ann, df_to_concat], axis=0, ignore_index=True
                )
        elif source.lower() == "event_profusion":
            df_sleep_stage_ann = pd.DataFrame(
                {
                    "start_sec": 30 * np.arange(len(df_sleep_ann["sleep_stage_list"])),
                    "sleep_stage": df_sleep_ann["sleep_stage_list"],
                }
            )
        else:
            raise ValueError(
                f"Source `{source}` not supported, "
                "only `hrv`, `event`, `event_profusion` are supported"
            )

        df_sleep_stage_ann = df_sleep_stage_ann[self.sleep_stage_keys]

        if self.sleep_stage_protocol == "aasm":
            df_sleep_stage_ann["sleep_stage"] = df_sleep_stage_ann["sleep_stage"].apply(
                lambda a: self._to_aasm_states[a]
            )
        elif self.sleep_stage_protocol == "simplified":
            df_sleep_stage_ann["sleep_stage"] = df_sleep_stage_ann["sleep_stage"].apply(
                lambda a: self._to_simplified_states[a]
            )
        elif self.sleep_stage_protocol == "shhs":
            df_sleep_stage_ann["sleep_stage"] = df_sleep_stage_ann["sleep_stage"].apply(
                lambda a: self._to_shhs_states[a]
            )

        if with_stage_names:
            df_sleep_stage_ann["sleep_stage_name"] = df_sleep_stage_ann[
                "sleep_stage"
            ].apply(lambda a: self.sleep_stage_names[a])

        if source.lower() != "event_profusion":
            self.logger.debug(
                f"record `{rec}` has `{len(df_tmp)}` raw (epoch_len = 5min) sleep stage annotations, "
                f"with `{len(self.sleep_stage_ann_keys_from_hrv)}` column(s)"
            )
            self.logger.debug(
                f"after being transformed (epoch_len = 30sec), record `{rec}` has {len(df_sleep_stage_ann)} "
                f"sleep stage annotations, with `{len(self.sleep_stage_keys)}` column(s)"
            )

        return df_sleep_stage_ann

    def load_sleep_event_ann(
        self,
        rec: Union[str, int],
        source: str = "event",
        event_types: Optional[List[str]] = None,
        sleep_event_ann_path: Optional[Union[str, Path]] = None,
    ) -> pd.DataFrame:
        """
        load sleep event annotations of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        source: str, default "event",
            source of the annotations, case insensitive,
            can be "hrv", "event", "event_profusion"
        event_types: list of str (cases ignored), optional,
            "Respiratory" (including "Apnea", "SpO2"), "Arousal",
            "Apnea" (including "CSA", "OSA", "MSA", "Hypopnea"), "SpO2",
            "CSA", "OSA", "MSA", "Hypopnea",
            used only when `source` is "event" or "event_profusion"
        sleep_event_ann_path: str or Path, optional,
            path of the file which contains the sleep event annotations,
            if not given, default path will be used

        Returns
        -------
        df_sleep_event_ann, DataFrame,
            all annotations on sleep events of `rec`

        """
        if isinstance(rec, int):
            rec = self[rec]
        df_sleep_ann = self.load_sleep_ann(
            rec=rec, source=source, sleep_ann_path=sleep_event_ann_path
        )
        if isinstance(df_sleep_ann, pd.DataFrame) and df_sleep_ann.empty:
            return df_sleep_ann
        elif isinstance(df_sleep_ann, dict) and df_sleep_ann["df_events"].empty:
            return df_sleep_ann["df_events"]

        df_sleep_event_ann = pd.DataFrame(columns=self.sleep_event_keys)

        if source.lower() == "hrv":
            df_sleep_ann = df_sleep_ann[self.sleep_event_ann_keys_from_hrv].reset_index(
                drop=True
            )
            df_sleep_event_ann = pd.DataFrame(columns=self.sleep_event_keys[1:3])
            for _, row in df_sleep_ann.iterrows():
                if row["hasrespevent"] == 0:
                    continue
                l_events = row[self.sleep_event_ann_keys_from_hrv[1:-1]].values.reshape(
                    (len(self.sleep_event_ann_keys_from_hrv) // 2 - 1, 2)
                )
                l_events = l_events[~np.isnan(l_events[:, 0])]
                df_to_concat = pd.DataFrame(
                    l_events, columns=self.sleep_event_keys[1:3]
                )
                df_sleep_event_ann = pd.concat(
                    [df_sleep_event_ann, df_to_concat], axis=0, ignore_index=True
                )
            df_sleep_event_ann["event_name"] = None
            df_sleep_event_ann["event_duration"] = df_sleep_event_ann.apply(
                lambda row: row["event_end"] - row["event_start"], axis=1
            )
            df_sleep_event_ann = df_sleep_event_ann[self.sleep_event_keys]

            self.logger.debug(
                f"record `{rec}` has `{len(df_sleep_ann)}` raw (epoch_len = 5min) sleep event "
                f"annotations from hrv, with `{len(self.sleep_event_ann_keys_from_hrv)}` column(s)"
            )
            self.logger.debug(
                f"after being transformed, record `{rec}` has `{len(df_sleep_event_ann)}` sleep event(s)"
            )
        elif source.lower() == "event":
            if event_types is None:
                event_types = ["respiratory", "arousal"]
            else:
                event_types = [e.lower() for e in event_types]
            assert (
                set()
                < set(event_types)
                <= set(
                    [
                        "respiratory",
                        "arousal",
                        "apnea",
                        "spo2",
                        "csa",
                        "osa",
                        "msa",
                        "hypopnea",
                    ]
                )
            ), (
                "`event_types` should be a subset of "
                "'respiratory', 'arousal', 'apnea', 'spo2', 'csa', 'osa', 'msa', 'hypopnea'",
                f"but got `{event_types}`",
            )
            _cols = set()
            if "respiratory" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[:6])
            if "arousal" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[6:11])
            if "apnea" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[:4])
            if "spo2" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[4:6])
            if "csa" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[0:1])
            if "osa" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[1:2])
            if "msa" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[2:3])
            if "hypopnea" in event_types:
                _cols = _cols | set(self.long_event_names_from_event[3:4])
            _cols = list(_cols)

            self.logger.debug(f"for record `{rec}`, _cols = `{_cols}`")

            df_sleep_event_ann = df_sleep_ann[
                df_sleep_ann["EventConcept"].isin(_cols)
            ].reset_index(drop=True)
            df_sleep_event_ann = df_sleep_event_ann.rename(
                {
                    "EventConcept": "event_name",
                    "Start": "event_start",
                    "Duration": "event_duration",
                },
                axis=1,
            )
            df_sleep_event_ann["event_name"] = df_sleep_event_ann["event_name"].apply(
                lambda s: s.split("|")[1]
            )
            df_sleep_event_ann["event_end"] = df_sleep_event_ann.apply(
                lambda row: row["event_start"] + row["event_duration"], axis=1
            )
            df_sleep_event_ann = df_sleep_event_ann[self.sleep_event_keys]
        elif source.lower() == "event_profusion":
            df_sleep_ann = df_sleep_ann["df_events"]
            _cols = set()
            if event_types is None:
                event_types = ["respiratory", "arousal"]
            else:
                event_types = [e.lower() for e in event_types]
            assert (
                set()
                < set(event_types)
                <= set(
                    [
                        "respiratory",
                        "arousal",
                        "apnea",
                        "spo2",
                        "csa",
                        "osa",
                        "msa",
                        "hypopnea",
                    ]
                )
            ), (
                "`event_types` should be a subset of "
                "'respiratory', 'arousal', 'apnea', 'spo2', 'csa', 'osa', 'msa', 'hypopnea', "
                f"but got `{event_types}`"
            )
            if "respiratory" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[:6])
            if "arousal" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[6:8])
            if "apnea" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[:4])
            if "spo2" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[4:6])
            if "csa" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[0:1])
            if "osa" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[1:2])
            if "msa" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[2:3])
            if "hypopnea" in event_types:
                _cols = _cols | set(self.event_names_from_event_profusion[3:4])
            _cols = list(_cols)

            self.logger.debug(f"for record `{rec}`, _cols = `{_cols}`")

            df_sleep_event_ann = df_sleep_ann[
                df_sleep_ann["Name"].isin(_cols)
            ].reset_index(drop=True)
            df_sleep_event_ann = df_sleep_event_ann.rename(
                {
                    "Name": "event_name",
                    "Start": "event_start",
                    "Duration": "event_duration",
                },
                axis=1,
            )
            df_sleep_event_ann["event_end"] = df_sleep_event_ann.apply(
                lambda row: row["event_start"] + row["event_duration"], axis=1
            )
            df_sleep_event_ann = df_sleep_event_ann[self.sleep_event_keys]
        else:
            raise ValueError(
                f"Source `{source}` not supported, "
                "only `hrv`, `event`, `event_profusion` are supported"
            )

        return df_sleep_event_ann

    def load_apnea_ann(
        self,
        rec: Union[str, int],
        source: str = "event",
        apnea_types: Optional[List[str]] = None,
        apnea_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load annotations on apnea events of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        source: str, default "event",
            source of the annotations, case insensitive,
            can be one of "event", "event_profusion"
        apnea_types: list of str (cases ignored), optional,
            "CSA", "OSA", "MSA", "Hypopnea",
            if is None, then all types of apnea will be loaded
        apnea_ann_path: str or Path, optional,
            path of the file which contains the apnea event annotations,
            if not given, default path will be used

        Returns
        -------
        df_apnea_ann, DataFrame,
            all annotations on apnea events of `rec`

        """
        event_types = ["apnea"] if apnea_types is None else apnea_types
        if source.lower() not in ["event", "event_profusion"]:
            raise ValueError(
                f"Source `{source}` contains no apnea annotations, "
                "should be one of 'event', 'event_profusion'"
            )
        df_apnea_ann = self.load_sleep_event_ann(
            rec=rec,
            source=source,
            event_types=event_types,
            sleep_event_ann_path=apnea_ann_path,
        )
        return df_apnea_ann

    def load_wave_delineation_ann(
        self,
        rec: Union[str, int],
        wave_deli_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load annotations on wave delineations of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        wave_deli_path: str or Path, optional,
            path of the file which contains wave delineation annotations,
            if not given, default path will be used

        Returns
        -------
        df_wave_delineation, DataFrame,
            all annotations on wave delineations of `rec`

        NOTE
        ----
        see the part describing wave delineation annotations of the docstring of the class,
        or call `self.database_info(detailed=True)`

        """
        if isinstance(rec, int):
            rec = self[rec]

        file_path = self.get_absolute_path(
            rec, wave_deli_path, rec_type="wave_delineation"
        )

        if not file_path.is_file():
            self.logger.debug(
                f"The annotation file of wave delineation of record `{rec}` has not been downloaded yet. "
                f"Or the path `{str(file_path)}` is not correct. "
                f"Or `{rec}` does not have `rpeak.csv` annotation file. Please check!"
            )
            return pd.DataFrame()

        df_wave_delineation = pd.read_csv(file_path, engine="python")
        df_wave_delineation = df_wave_delineation[self.wave_deli_keys].reset_index(
            drop=True
        )
        return df_wave_delineation

    def load_rpeak_ann(
        self,
        rec: Union[str, int],
        rpeak_ann_path: Optional[Union[str, Path]] = None,
        exclude_artifacts: bool = True,
        exclude_abnormal_beats: bool = True,
        units: Optional[str] = None,
        **kwargs: Any,
    ) -> np.ndarray:
        """
        load annotations on R peaks of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        rpeak_ann_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used
        exclude_artifacts: bool, default True,
            exlcude those beats (R peaks) that are labelled artifact or not
        exclude_abnormal_beats: bool, default True,
            exlcude those beats (R peaks) that are labelled abnormal ("VE" and "SVE") or not
        units: str, optional,
            units of the returned R peak locations,
            can be one of "s", "ms", case insensitive,
            None for no conversion, using indices of samples

        Returns
        -------
        ndarray, shape (n_rpeaks, ),
            locations of R peaks of `rec`

        """
        info_items = ["Type", "rpointadj", "samplingrate"]
        df_rpeaks_with_type_info = self.load_wave_delineation_ann(rec, rpeak_ann_path)
        if df_rpeaks_with_type_info.empty:
            return np.array([], dtype=int)
        df_rpeaks_with_type_info = df_rpeaks_with_type_info[info_items]
        exclude_beat_types = []
        # 0 = Artifact, 1 = Normal Sinus Beat, 2 = VE, 3 = SVE
        if exclude_artifacts:
            exclude_beat_types.append(0)
        if exclude_abnormal_beats:
            exclude_beat_types += [2, 3]

        rpeaks = df_rpeaks_with_type_info[
            ~df_rpeaks_with_type_info["Type"].isin(exclude_beat_types)
        ]["rpointadj"].values

        if units is None:
            rpeaks = (np.round(rpeaks)).astype(int)
        elif units.lower() == "s":
            fs = df_rpeaks_with_type_info.iloc[0]["samplingrate"]
            rpeaks = rpeaks / fs
        elif units.lower() == "ms":
            fs = df_rpeaks_with_type_info.iloc[0]["samplingrate"]
            rpeaks = rpeaks / fs * 1000
            rpeaks = (np.round(rpeaks)).astype(int)
        else:
            raise ValueError(
                "`units` should be one of 's', 'ms', case insensitive, "
                "or None for no conversion, using indices of samples, "
                f"but got `{units}`"
            )

        return rpeaks

    def load_rr_ann(
        self,
        rec: Union[str, int],
        rpeak_ann_path: Optional[Union[str, Path]] = None,
        units: Union[str, None] = "s",
        **kwargs: Any,
    ) -> np.ndarray:
        """
        load annotations on RR intervals of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        rpeak_ann_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used
        units: str, default "s",
            units of the returned R peak locations,
            can be one of "s", "ms", case insensitive,
            None for no conversion, using indices of samples

        Returns
        -------
        rr: ndarray,
            array of rr intervals, shape (n_rpeaks - 1, 2),
            each row is a rr interval, the first column is the location of the R peak

        """
        rpeaks_ts = self.load_rpeak_ann(
            rec=rec,
            rpeak_ann_path=rpeak_ann_path,
            exclude_artifacts=True,
            exclude_abnormal_beats=True,
            units=units,
        )
        rr = np.diff(rpeaks_ts)
        rr = np.column_stack((rpeaks_ts[:-1], rr))
        return rr

    def load_nn_ann(
        self,
        rec: Union[str, int],
        rpeak_ann_path: Optional[Union[str, Path]] = None,
        units: Union[str, None] = "s",
        **kwargs: Any,
    ) -> np.ndarray:
        """
        load annotations on NN intervals of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        rpeak_ann_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used
        units: str, default "s",
            units of the returned R peak locations,
            can be one of "s", "ms", case insensitive,
            None for no conversion, using indices of samples

        Returns
        -------
        nn: ndarray,
            array of nn intervals, shape (n, 2),
            each row is a nn interval, the first column is the location of the R peak

        """
        info_items = ["Type", "rpointadj", "samplingrate"]
        df_rpeaks_with_type_info = self.load_wave_delineation_ann(rec, rpeak_ann_path)
        if df_rpeaks_with_type_info.empty:
            return np.array([]).reshape(0, 2)

        df_rpeaks_with_type_info = df_rpeaks_with_type_info[info_items]
        fs = df_rpeaks_with_type_info.iloc[0]["samplingrate"]
        rpeaks = df_rpeaks_with_type_info["rpointadj"]

        if units is None:
            rpeaks = (np.round(rpeaks)).astype(int)
        elif units.lower() == "s":
            rpeaks = rpeaks / fs
        elif units.lower() == "ms":
            rpeaks = rpeaks / fs * 1000
            rpeaks = (np.round(rpeaks)).astype(int)
        else:
            raise ValueError(
                "`units` should be one of 's', 'ms', case insensitive, "
                "or None for no conversion, using indices of samples, "
                f"but got `{units}`"
            )

        rr = np.diff(rpeaks)
        rr = np.column_stack((rpeaks[:-1], rr))

        normal_sinus_rpeak_indices = np.where(
            df_rpeaks_with_type_info["Type"].values == 1
        )[
            0
        ]  # 1 = Normal Sinus Beat
        keep_indices = np.where(np.diff(normal_sinus_rpeak_indices) == 1)[0].tolist()
        nn = rr[normal_sinus_rpeak_indices[keep_indices]]
        return nn.reshape(-1, 2)

    def locate_artifacts(
        self,
        rec: Union[str, int],
        wave_deli_path: Optional[Union[str, Path]] = None,
        units: Optional[str] = None,
    ) -> np.ndarray:
        """
        locate "artifacts" in `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        wave_deli_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used
        units: str, optional,
            units of the returned artifact locations,
            can be one of "s", "ms", case insensitive,
            None for no conversion, using indices of samples

        Returns
        -------
        artifacts: ndarray,
            indices (or time) of artifacts locations,
            shape (n_artifacts,)

        """
        df_rpeaks_with_type_info = self.load_wave_delineation_ann(rec, wave_deli_path)
        if df_rpeaks_with_type_info.empty:
            dtype = int if units is None or units.lower() != "s" else float
            return np.array([], dtype=dtype)
        # df_rpeaks_with_type_info = df_rpeaks_with_type_info[["Type", "rpointadj"]]

        artifacts = (
            np.round(
                df_rpeaks_with_type_info[df_rpeaks_with_type_info["Type"] == 0][
                    "rpointadj"
                ].values
            )
        ).astype(int)

        if units is not None:
            fs = df_rpeaks_with_type_info.iloc[0]["samplingrate"]
            if units.lower() == "s":
                artifacts = artifacts / fs
            elif units.lower() == "ms":
                artifacts = artifacts / fs * 1000
                artifacts = (np.round(artifacts)).astype(int)
            else:
                raise ValueError(
                    "`units` should be one of 's', 'ms', case insensitive, "
                    "or None for no conversion, using indices of samples, "
                    f"but got `{units}`"
                )

        return artifacts

    def locate_abnormal_beats(
        self,
        rec: Union[str, int],
        wave_deli_path: Optional[Union[str, Path]] = None,
        abnormal_type: Optional[str] = None,
        units: Optional[str] = None,
    ) -> Union[Dict[str, np.ndarray], np.ndarray]:
        """
        locate "abnormal beats" in `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        wave_deli_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used
        abnormal_type: str, optional,
            type of abnormal beat type to locate, can be "VE", "SVE",
            if not given, both "VE" and "SVE" will be located
        units: str, optional,
            units of the returned R peak locations,
            can be one of "s", "ms", case insensitive,
            None for no conversion, using indices of samples

        Returns
        -------
        abnormal_rpeaks: dict,
            dictionary of abnormal beat locations,
            keys are "VE" and/or "SVE",
            values are indices (or time) of abnormal beats, shape (n,)

        """
        if abnormal_type is not None and abnormal_type not in ["VE", "SVE"]:
            raise ValueError(
                f"No abnormal type of `{abnormal_type}` in "
                "wave delineation annotation (*-rpeak.csv) files"
            )

        df_rpeaks_with_type_info = self.load_wave_delineation_ann(rec, wave_deli_path)

        if not df_rpeaks_with_type_info.empty:
            # df_rpeaks_with_type_info = df_rpeaks_with_type_info[["Type", "rpointadj"]]
            # 2 = VE, 3 = SVE
            ve = (
                np.round(
                    df_rpeaks_with_type_info[df_rpeaks_with_type_info["Type"] == 2][
                        "rpointadj"
                    ].values
                )
            ).astype(int)
            sve = (
                np.round(
                    df_rpeaks_with_type_info[df_rpeaks_with_type_info["Type"] == 3][
                        "rpointadj"
                    ].values
                )
            ).astype(int)
            abnormal_rpeaks = {"VE": ve, "SVE": sve}
        else:
            dtype = int if units is None or units.lower() != "s" else float
            abnormal_rpeaks = {
                "VE": np.array([], dtype=dtype),
                "SVE": np.array([], dtype=dtype),
            }

        if units is not None and not df_rpeaks_with_type_info.empty:
            fs = df_rpeaks_with_type_info.iloc[0]["samplingrate"]
            if units.lower() == "s":
                abnormal_rpeaks = {
                    abnormal_type: abnormal_rpeaks[abnormal_type] / fs
                    for abnormal_type in abnormal_rpeaks
                }
            elif units.lower() == "ms":
                abnormal_rpeaks = {
                    abnormal_type: abnormal_rpeaks[abnormal_type] / fs * 1000
                    for abnormal_type in abnormal_rpeaks
                }
                abnormal_rpeaks = {
                    abnormal_type: (np.round(abnormal_rpeaks[abnormal_type])).astype(
                        int
                    )
                    for abnormal_type in abnormal_rpeaks
                }
            else:
                raise ValueError(
                    "`units` should be one of 's', 'ms', case insensitive, "
                    "or None for no conversion, using indices of samples, "
                    f"but got `{units}`"
                )

        if abnormal_type is None:
            return abnormal_rpeaks
        elif abnormal_type in ["VE", "SVE"]:
            return abnormal_rpeaks[abnormal_type]

    def load_eeg_band_ann(
        self,
        rec: Union[str, int],
        eeg_band_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load annotations on EEG bands of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        eeg_band_ann_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used

        Returns
        -------
        to write,

        """
        if self.current_version >= "0.15.0":
            self.logger.info(
                f"EEG spectral summary variables are removed in version {self.current_version}"
            )
        else:
            raise NotImplementedError

    def load_eeg_spectral_ann(
        self,
        rec: Union[str, int],
        eeg_spectral_ann_path: Optional[Union[str, Path]] = None,
        **kwargs: Any,
    ) -> pd.DataFrame:
        """
        load annotations on EEG spectral summary of `rec`

        Parameters
        ----------
        rec: str or int,
            record name, typically in the form "shhs1-200001",
            or index of the record in `self.all_records`
        eeg_spectral_ann_path: str or Path, optional,
            annotation file path,
            if not given, default path will be used

        Returns
        -------
        to write,

        """
        if self.current_version >= "0.15.0":
            self.logger.info(
                f"EEG spectral summary variables are removed in version {self.current_version}"
            )
        else:
            raise NotImplementedError

    # TODO: add more functions for annotation reading
    # TODO: add plotting functions

    def plot_ann(
        self,
        rec: Union[str, int],
        stage_source: Optional[str] = None,
        stage_kw: dict = {},
        event_source: Optional[str] = None,
        event_kw: dict = {},
        plot_format: str = "span",
    ) -> None:
        """
        Parameters
        ----------
        rec, str,
            record name, typically in the form "shhs1-200001"
        stage_source: str, optional,
            source of the sleep stage annotations,
            can be one of "hrv", "event", "event_profusion",
            if is None, then annotations of sleep stages of `rec` won"t be plotted
        stage_kw: dict, default {},
            arguments to the function `self.load_sleep_stage_ann`
        event_source: str, optional,
            source of the sleep event annotations,
            can be one of "hrv", "event", "event_profusion",
            if is None, then annotations of sleep events of `rec` won"t be plotted
        event_kw: dict, default {},
            arguments to the function `self.load_sleep_event_ann`
        plot_format: str, default "span",
            format of the plot,
            can be one of "span", "hypnogram", case insensitive

        """
        if all([stage_source is None, event_source is None]):
            raise ValueError("`stage_source` and `event_source` cannot be both `None`")

        if stage_source is not None:
            df_sleep_stage = self.load_sleep_stage_ann(
                rec, source=stage_source, **stage_kw
            )
            if df_sleep_stage.empty:
                if isinstance(rec, int):
                    rec = self[rec]
                raise ValueError(
                    f"No sleep stage annotations found for record `{rec}` "
                    f"with source `{stage_source}`"
                )
        else:
            df_sleep_stage = None
        if event_source is not None:
            df_sleep_event = self.load_sleep_event_ann(
                rec, source=event_source, **event_kw
            )
            if df_sleep_event.empty:
                if isinstance(rec, int):
                    rec = self[rec]
                raise ValueError(
                    f"No sleep event annotations found for record `{rec}` "
                    f"with source `{event_source}`"
                )
        else:
            df_sleep_event = None

        self._plot_ann(
            df_sleep_stage=df_sleep_stage,
            df_sleep_event=df_sleep_event,
            plot_format=plot_format,
        )

    def _plot_ann(
        self,
        df_sleep_stage: Optional[pd.DataFrame] = None,
        df_sleep_event: Optional[pd.DataFrame] = None,
        plot_format: str = "span",
    ) -> None:
        """
        Parameters
        ----------
        df_sleep_stage: DataFrame, optional,
            sleep stage annotations
        df_sleep_event: DataFrame, optional,
            sleep event annotations
        plot_format: str, default "span",
            format of the plot,
            can be one of "span", "hypnogram", case insensitive

        TODO
        ----
        1. ~~implement the `hypnogram` format~~
        2. implement plotting of sleep events
        """
        import matplotlib.patches as mpatches
        import matplotlib.pyplot as plt

        check = [df_sleep_stage is None, df_sleep_event is None]
        nb_axes = len(check) - np.sum(check)

        if nb_axes == 0:
            raise ValueError("No input data!")

        if plot_format.lower() not in ["span", "hypnogram"]:
            raise ValueError(
                f"Unknown plot format `{plot_format}`! "
                f"`plot_format` can only be one of `span`, `hypnogram`"
            )

        if df_sleep_stage is not None:
            sleep_stages = {}
            for k in self.sleep_stage_names:
                sleep_stages[k] = intervals_union(
                    interval_list=[
                        [sec, sec + self.sleep_epoch_len_sec]
                        for sec in df_sleep_stage[
                            df_sleep_stage["sleep_stage"]
                            == self.sleep_stage_name_value_mapping[k]
                        ]["start_sec"].values
                    ],
                    join_book_endeds=True,
                )

        if df_sleep_event is not None:
            current_legal_events = [
                "Central Apnea",
                "Obstructive Apnea",
                "Mixed Apnea",
                "Hypopnea",
            ]
            if len(current_legal_events) != len(
                set(current_legal_events) | set(df_sleep_event["event_name"])
            ):
                raise NotImplementedError(
                    "Plotting of some type of events in `df_sleep_event` has not been implemented yet!"
                )

        if plot_format.lower() == "hypnogram":
            stage_mask = df_sleep_stage["sleep_stage"].values
            stage_mask = len(self.sleep_stage_names) - 1 - stage_mask
            fig, ax = self.plot_hypnogram(stage_mask, granularity=30)
            return

        patches = {k: mpatches.Patch(color=c, label=k) for k, c in self.palette.items()}

        _, axes = plt.subplots(nb_axes, 1, figsize=(20, 4 * nb_axes), sharex=True)
        plt.subplots_adjust(hspace=0)
        plot_alpha = 0.5

        ax_stages, ax_events = None, None
        if nb_axes == 1 and df_sleep_stage is not None:
            ax_stages = axes
            ax_stages.set_title("Sleep Stages", fontsize=24)
            ax_stages.set_xlabel("Time", fontsize=16)
            # ax_stages.set_ylabel("Stages", fontsize=16)
        elif nb_axes == 1 and df_sleep_event is not None:
            ax_events = axes
            ax_events.set_title("Sleep Events", fontsize=24)
            ax_events.set_xlabel("Time", fontsize=16)
            # ax_events.set_ylabel("Events", fontsize=16)
        else:
            ax_stages, ax_events = axes
            ax_stages.set_title("Sleep Stages and Events", fontsize=24)
            ax_events.set_xlabel("Time", fontsize=16)

        if ax_stages is not None:
            for k, v in sleep_stages.items():
                for itv in v:
                    ax_stages.axvspan(
                        datetime.fromtimestamp(itv[0]),
                        datetime.fromtimestamp(itv[1]),
                        color=self.palette[k],
                        alpha=plot_alpha,
                    )
            ax_stages.legend(
                handles=[
                    patches[k]
                    for k in self.all_sleep_stage_names
                    if k in sleep_stages.keys()
                ],
                loc="best",
            )  # keep ordering
            plt.setp(ax_stages.get_yticklabels(), visible=False)
            ax_stages.tick_params(axis="y", which="both", length=0)

        if ax_events is not None:
            for _, row in df_sleep_event.iterrows():
                ax_events.axvspan(
                    datetime.fromtimestamp(row["event_start"]),
                    datetime.fromtimestamp(row["event_end"]),
                    color=self.palette[row["event_name"]],
                    alpha=plot_alpha,
                )
            ax_events.legend(
                handles=[
                    patches[k]
                    for k in current_legal_events
                    if k in set(df_sleep_event["event_name"])
                ],
                loc="best",
            )  # keep ordering
            plt.setp(ax_events.get_yticklabels(), visible=False)
            ax_events.tick_params(axis="y", which="both", length=0)

    def str_to_real_number(self, s: Union[str, Real]) -> Real:
        """
        some columns in the annotations might incorrectly
        been converted from real number to string, using `xmltodict`.

        Parameters
        ----------
        s: str or real number (NaN)

        Returns
        -------
        real number

        """
        if isinstance(s, str):
            if "." in s:
                return float(s)
            else:
                return int(s)
        else:  # NaN case
            return s

    def __create_constants(self, **kwargs) -> None:
        """ """
        self.lazy = kwargs.get("lazy", False)
        self.extension = {
            "psg": ".edf",
            "wave_delineation": "-rpoint.csv",
            "event": "-nsrr.xml",
            "event_profusion": "-profusion.xml",
        }

        # fmt: off

        self.all_signals = [
            "EEG(sec)", "ECG", "EMG", "EOG(L)", "EOG(R)", "EEG",
            "AIRFLOW", "THOR RES", "ABDO RES", "NEW AIR", "OX stat", "SaO2", "H.R.",
            "POSITION", "SOUND", "LIGHT",
            "AUX", "CPAP", "EPMS", "OX STAT", "PR",
        ]
        self.all_signals = set([s.lower() for s in self.all_signals])

        # annotations regarding sleep analysis
        self.hrv_ann_summary_keys = [
            "nsrrid", "visitnumber", "NN_RR", "AVNN", "IHR",
            "SDNN", "SDANN", "SDNNIDX", "rMSSD",
            "pNN10", "pNN20", "pNN30", "pNN40", "pNN50",
            "tot_pwr", "ULF", "VLF", "LF", "HF", "LF_HF", "LF_n", "HF_n",
        ]
        self.hrv_ann_detailed_keys = [
            "nsrrid", "visitnumber", "Start__sec_", "ihr", "hasrespevent",
            "NN_RR", "AVNN", "SDNN", "rMSSD",
            "PNN10", "PNN20", "PNN30", "PNN40", "PNN50",
            "TOT_PWR", "VLF", "LF", "LF_n", "HF", "HF_n", "LF_HF",
            "sleepstage01", "sleepstage02", "sleepstage03", "sleepstage04", "sleepstage05",
            "sleepstage06", "sleepstage07", "sleepstage08", "sleepstage09", "sleepstage10",
            "event01start", "event01end",
            "event02start", "event02end",
            "event03start", "event03end",
            "event04start", "event04end",
            "event05start", "event05end",
            "event06start", "event06end",
            "event07start", "event07end",
            "event08start", "event08end",
            "event09start", "event09end",
            "event10start", "event10end",
            "event11start", "event11end",
            "event12start", "event12end",
            "event13start", "event13end",
            "event14start", "event14end",
            "event15start", "event15end",
            "event16start", "event16end",
            "event17start", "event17end",
            "event18start", "event18end",
        ]
        self.hrv_ann_epoch_len_sec = 300  # 5min
        self.sleep_ann_keys_from_hrv = [
            "Start__sec_", "hasrespevent",
            "sleepstage01", "sleepstage02", "sleepstage03", "sleepstage04", "sleepstage05",
            "sleepstage06", "sleepstage07", "sleepstage08", "sleepstage09", "sleepstage10",
            "event01start", "event01end",
            "event02start", "event02end",
            "event03start", "event03end",
            "event04start", "event04end",
            "event05start", "event05end",
            "event06start", "event06end",
            "event07start", "event07end",
            "event08start", "event08end",
            "event09start", "event09end",
            "event10start", "event10end",
            "event11start", "event11end",
            "event12start", "event12end",
            "event13start", "event13end",
            "event14start", "event14end",
            "event15start", "event15end",
            "event16start", "event16end",
            "event17start", "event17end",
            "event18start", "event18end",
        ]
        self.sleep_stage_ann_keys_from_hrv = [
            "Start__sec_",
            "sleepstage01", "sleepstage02", "sleepstage03", "sleepstage04", "sleepstage05",
            "sleepstage06", "sleepstage07", "sleepstage08", "sleepstage09", "sleepstage10",
        ]
        self.sleep_event_ann_keys_from_hrv = [
            "Start__sec_", "hasrespevent",
            "event01start", "event01end",
            "event02start", "event02end",
            "event03start", "event03end",
            "event04start", "event04end",
            "event05start", "event05end",
            "event06start", "event06end",
            "event07start", "event07end",
            "event08start", "event08end",
            "event09start", "event09end",
            "event10start", "event10end",
            "event11start", "event11end",
            "event12start", "event12end",
            "event13start", "event13end",
            "event14start", "event14end",
            "event15start", "event15end",
            "event16start", "event16end",
            "event17start", "event17end",
            "event18start", "event18end",
        ]

        # annotations from events-nsrr and events-profusion folders
        self.event_keys = [
            "EventType", "EventConcept", "Start", "Duration",
            "SignalLocation", "SpO2Nadir", "SpO2Baseline",
        ]
        # NOTE: the union of names from shhs1-200001 to shhs1-200399
        # NOT a full search
        self.short_event_types_from_event = [
            "Respiratory", "Stages", "Arousals",
        ]
        self.long_event_types_from_event = [
            "Respiratory|Respiratory",
            "Stages|Stages",
            "Arousals|Arousals",
        ]
        # NOTE: the union of names from shhs1-200001 to shhs1-200399
        # NOT a full search
        # NOT including sleep stages
        self.short_event_names_from_event = [
            "Central Apnea",
            "Obstructive Apnea",
            "Mixed Apnea",
            "Hypopnea",
            "SpO2 artifact",
            "SpO2 desaturation",
            "Arousal ()",
            "Arousal (Standard)",
            "Arousal (STANDARD)",
            "Arousal (CHESHIRE)",
            "Arousal (ASDA)",
            "Unsure",
        ]
        self.long_event_names_from_event = [
            "Central apnea|Central Apnea",
            "Obstructive apnea|Obstructive Apnea",
            "Mixed apnea|Mixed Apnea",
            "Hypopnea|Hypopnea",
            "SpO2 artifact|SpO2 artifact",
            "SpO2 desaturation|SpO2 desaturation",
            "Arousal|Arousal ()",
            "Arousal|Arousal (Standard)",
            "Arousal|Arousal (STANDARD)",
            "Arousal resulting from Chin EMG|Arousal (CHESHIRE)",
            "ASDA arousal|Arousal (ASDA)",
            "Unsure|Unsure",
        ]
        self.event_profusion_keys = [
            "Name", "Start", "Duration",
            "Input", "LowestSpO2", "Desaturation",
        ]
        # NOTE: currently the union of names from shhs1-200001 to shhs1-200099,
        # NOT a full search
        self.event_names_from_event_profusion = [
            "Central Apnea",
            "Obstructive Apnea",
            "Mixed Apnea",
            "Hypopnea",
            "SpO2 artifact",
            "SpO2 desaturation",
            "Arousal ()",
            "Arousal (ASDA)",
            "Unsure",
        ]

        self.apnea_types = [
            "Central Apnea",
            "Obstructive Apnea",
            "Mixed Apnea",
            "Hypopnea",
        ]

        # annotations regarding wave delineation
        self.wave_deli_keys = [
            "RPoint", "Start", "End",
            "STLevel1", "STSlope1", "STLevel2", "STSlope2",
            "Manual", "Type", "rpointadj",
            "PPoint", "PStart", "PEnd",
            "TPoint", "TStart", "TEnd",
            "TemplateID", "nsrrid", "samplingrate", "seconds", "epoch",
        ]
        self.wave_deli_samp_num_keys = [
            "RPoint", "Start", "End",
            "PPoint", "PStart", "PEnd",
            "TPoint", "TStart", "TEnd",
        ]

        # TODO: other annotation files: EEG

        # self-defined items
        self.sleep_stage_keys = ["start_sec", "sleep_stage"]
        self.sleep_event_keys = [
            "event_name", "event_start", "event_end", "event_duration",
        ]
        self.sleep_epoch_len_sec = 30
        self.ann_sleep_stages = [0, 1, 2, 3, 4, 5, 9]
        """
        0	--- Wake
        1	--- sleep stage 1
        2	--- sleep stage 2
        3	--- sleep stage 3/4
        4	--- sleep stage 3/4
        5	--- REM stage
        9	--- Movement/Wake or Unscored?
        """
        self.sleep_stage_protocol = kwargs.get("sleep_stage_protocol", "aasm")
        self.all_sleep_stage_names = ["W", "R", "N1", "N2", "N3", "N4"]
        self.sleep_stage_name_value_mapping = {
            "W": 0,
            "R": 1,
            "N1": 2,
            "N2": 3,
            "N3": 4,
            "N4": 5,
        }
        self.sleep_stage_names = []
        self.update_sleep_stage_names()
        self._to_simplified_states = {9: 0, 0: 0, 5: 1, 1: 2, 2: 2, 3: 3, 4: 3}
        """ 9 to nan?
        0   --- awake
        1   --- REM
        2   --- N1 (NREM1/2), shallow sleep
        3   --- N2 (NREM3/4), deep sleep
        """
        self._to_aasm_states = {9: 0, 0: 0, 5: 1, 1: 2, 2: 3, 3: 4, 4: 4}
        """ 9 to nan?
        0   --- awake
        1   --- REM
        2   --- N1 (NREM1)
        3   --- N2 (NREM2)
        4   --- N3 (NREM3/4)
        """
        self._to_shhs_states = {9: 0, 0: 0, 5: 1, 1: 2, 2: 3, 3: 4, 4: 5}

        # for plotting
        self.palette = {
            "W": "orange",
            "R": "yellow",
            "N1": "green",
            "N2": "cyan",
            "N3": "blue",
            "N4": "purple",
            "Central Apnea": "red",
            "Obstructive Apnea": "yellow",
            "Mixed Apnea": "cyan",
            "Hypopnea": "purple",
        }  # TODO: add more

        # fmt: on

    @property
    def folder_or_file(self) -> Dict[str, Path]:
        return {
            "psg": self.psg_data_path,
            "wave_delineation": self.wave_deli_path,
            "event": self.event_ann_path,
            "event_profusion": self.event_profusion_ann_path,
        }

    @property
    def url(self) -> str:
        warnings.warn(
            "one has to apply for a token from `sleepdata.org` "
            "and uses `nsrr` to download the data",
            RuntimeWarning,
        )
        return ""

    @property
    def database_info(self) -> DataBaseInfo:
        return _SHHS_INFO
