sertit.snap API documentation
Description: SNAP tools

# Module `sertit.snap`

SNAP tools

Expand source code

    # -*- coding: utf-8 -*-
    # Copyright 2021, SERTIT-ICube - France, https://sertit.unistra.fr/
    # This file is part of sertit-utils project
    #     https://github.com/sertit/sertit-utils
    #
    # Licensed under the Apache License, Version 2.0 (the "License");
    # you may not use this file except in compliance with the License.
    # You may obtain a copy of the License at
    #
    #     http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    """
    SNAP tools
    """
    import logging
    import os

    import psutil

    from sertit import misc, strings
    from sertit.logs import SU_NAME

    MAX_CORES = os.cpu_count() - 2
    MAX_MEM = int(os.environ.get("JAVA_OPTS_XMX", 0.95 * psutil.virtual_memory().total))
    TILE_SIZE = 2048
    LOGGER = logging.getLogger(SU_NAME)

    def bytes2snap(nof_bytes: int) -> str:
        """
        Convert nof bytes into snap-compatible Java options.

        ```python
        >>> bytes2snap(32000)
        '31K'
        ```

        Args:
            nof_bytes (int): Byte nb

        Returns:
            str: Human-readable in bits

        """
        symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
        prefix = {}
        for idx, sym in enumerate(symbols):
            prefix[sym] = 1 << (idx + 1) * 10
        for sym in reversed(symbols):
            if nof_bytes >= prefix[sym]:
                value = int(float(nof_bytes) / prefix[sym])
                return "%s%s" % (value, sym)
        return "%sB" % nof_bytes

    def get_gpt_cli(
        graph_path: str, other_args: list, display_snap_opt: bool = False
    ) -> list:
        """
        Get GPT command line with system OK optimizations.
        To see options, type this command line with --diag (but it won't run the graph)

        ```python
        >>> get_gpt_cli("graph_path", other_args=[], display_snap_opt=True)
        SNAP Release version 8.0
        SNAP home: C:\Program Files\snap\bin\/..
        SNAP debug: null
        SNAP log level: WARNING
        Java home: c:\program files\snap\jre\jre
        Java version: 1.8.0_242
        Processors: 16
        Max memory: 53.3 GB
        Cache size: 30.0 GB
        Tile parallelism: 14
        Tile size: 2048 x 2048 pixels
        To configure your gpt memory usage:
        Edit snap/bin/gpt.vmoptions
        To configure your gpt cache size and parallelism:
        Edit .snap/etc/snap.properties or gpt -c ${cachesize-in-GB}G -q ${parallelism}

        ['gpt', '"graph_path"', '-q', 14, '-J-Xms2G -J-Xmx60G', '-J-Dsnap.log.level=WARNING',
        '-J-Dsnap.jai.defaultTileSize=2048', '-J-Dsnap.dataio.reader.tileWidth=2048',
        '-J-Dsnap.dataio.reader.tileHeigh=2048', '-J-Dsnap.jai.prefetchTiles=true', '-c 30G']

        ```

        Args:
            graph_path (str): Graph path
            other_args (list): Other args as a list such as `['-Pfile="in_file.zip", '-Pout="out_file.dim"']`
            display_snap_opt (bool): Display SNAP options via --diag

        Returns:
            list: GPT command line as a list
        """
        gpt_cli = [
            "gpt",
            strings.to_cmd_string(graph_path),
            "-q",
            MAX_CORES,  # Maximum parallelism
            f"-J-Xms2G -J-Xmx{bytes2snap(MAX_MEM)}",  # Initially/max allocated memory
            "-J-Dsnap.log.level=WARNING",
            f"-J-Dsnap.jai.defaultTileSize={TILE_SIZE}",
            f"-J-Dsnap.dataio.reader.tileWidth={TILE_SIZE}",
            f"-J-Dsnap.dataio.reader.tileHeight={TILE_SIZE}",
            "-J-Dsnap.jai.prefetchTiles=true",
            f"-c {bytes2snap(int(0.5 * MAX_MEM))}",  # Tile cache set to 50% of max memory (up to 75%)
            # '-x',
            *other_args,
        ]  # Clears the internal tile cache after writing a complete row to the target file

        # LOGs
        LOGGER.debug(gpt_cli)
        if display_snap_opt:
            misc.run_cli(gpt_cli + ["--diag"])

        return gpt_cli

## Functions

`

def bytes2snap(

nof_bytes)

`

  Convert nof bytes into snap-compatible Java options.

      >>> bytes2snap(32000)
      '31K'

  ## Args

  `nof_bytes` :`int`
    Byte nb

  ## Returns

  `str`
    Human-readable in bits
  Expand source code

      def bytes2snap(nof_bytes: int) -> str:
          """
          Convert nof bytes into snap-compatible Java options.

          ```python
          >>> bytes2snap(32000)
          '31K'
          ```

          Args:
              nof_bytes (int): Byte nb

          Returns:
              str: Human-readable in bits

          """
          symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
          prefix = {}
          for idx, sym in enumerate(symbols):
              prefix[sym] = 1 << (idx + 1) * 10
          for sym in reversed(symbols):
              if nof_bytes >= prefix[sym]:
                  value = int(float(nof_bytes) / prefix[sym])
                  return "%s%s" % (value, sym)
          return "%sB" % nof_bytes

`

def get_gpt_cli(

graph_path,
other_args,
display_snap_opt=False)

`

  Get GPT command line with system OK optimizations. To see options, type this command line with –diag (but it won't run the graph)

      >>> get_gpt_cli("graph_path", other_args=[], display_snap_opt=True)
      SNAP Release version 8.0
      SNAP home: C:\Program Files\snapin\/..
      SNAP debug: null
      SNAP log level: WARNING
      Java home: c:\program files\snap\jre\jre
      Java version: 1.8.0_242
      Processors: 16
      Max memory: 53.3 GB
      Cache size: 30.0 GB
      Tile parallelism: 14
      Tile size: 2048 x 2048 pixels
      To configure your gpt memory usage:
      Edit snap/bin/gpt.vmoptions
      To configure your gpt cache size and parallelism:
      Edit .snap/etc/snap.properties or gpt -c ${cachesize-in-GB}G -q ${parallelism}

      ['gpt', '"graph_path"', '-q', 14, '-J-Xms2G -J-Xmx60G', '-J-Dsnap.log.level=WARNING',
      '-J-Dsnap.jai.defaultTileSize=2048', '-J-Dsnap.dataio.reader.tileWidth=2048',
      '-J-Dsnap.dataio.reader.tileHeigh=2048', '-J-Dsnap.jai.prefetchTiles=true', '-c 30G']

  ## Args

  `graph_path` :`str`
    Graph path
  `other_args` :`list`
    Other args as a list such as `['-Pfile="in_file.zip", '-Pout="out_file.dim"']`
  `display_snap_opt` :`bool`
    Display SNAP options via –diag

  ## Returns

  `list`
    GPT command line as a list
  Expand source code

      def get_gpt_cli(
          graph_path: str, other_args: list, display_snap_opt: bool = False
      ) -> list:
          """
          Get GPT command line with system OK optimizations.
          To see options, type this command line with --diag (but it won't run the graph)

          ```python
          >>> get_gpt_cli("graph_path", other_args=[], display_snap_opt=True)
          SNAP Release version 8.0
          SNAP home: C:\Program Files\snap\bin\/..
          SNAP debug: null
          SNAP log level: WARNING
          Java home: c:\program files\snap\jre\jre
          Java version: 1.8.0_242
          Processors: 16
          Max memory: 53.3 GB
          Cache size: 30.0 GB
          Tile parallelism: 14
          Tile size: 2048 x 2048 pixels
          To configure your gpt memory usage:
          Edit snap/bin/gpt.vmoptions
          To configure your gpt cache size and parallelism:
          Edit .snap/etc/snap.properties or gpt -c ${cachesize-in-GB}G -q ${parallelism}

          ['gpt', '"graph_path"', '-q', 14, '-J-Xms2G -J-Xmx60G', '-J-Dsnap.log.level=WARNING',
          '-J-Dsnap.jai.defaultTileSize=2048', '-J-Dsnap.dataio.reader.tileWidth=2048',
          '-J-Dsnap.dataio.reader.tileHeigh=2048', '-J-Dsnap.jai.prefetchTiles=true', '-c 30G']

          ```

          Args:
              graph_path (str): Graph path
              other_args (list): Other args as a list such as `['-Pfile="in_file.zip", '-Pout="out_file.dim"']`
              display_snap_opt (bool): Display SNAP options via --diag

          Returns:
              list: GPT command line as a list
          """
          gpt_cli = [
              "gpt",
              strings.to_cmd_string(graph_path),
              "-q",
              MAX_CORES,  # Maximum parallelism
              f"-J-Xms2G -J-Xmx{bytes2snap(MAX_MEM)}",  # Initially/max allocated memory
              "-J-Dsnap.log.level=WARNING",
              f"-J-Dsnap.jai.defaultTileSize={TILE_SIZE}",
              f"-J-Dsnap.dataio.reader.tileWidth={TILE_SIZE}",
              f"-J-Dsnap.dataio.reader.tileHeight={TILE_SIZE}",
              "-J-Dsnap.jai.prefetchTiles=true",
              f"-c {bytes2snap(int(0.5 * MAX_MEM))}",  # Tile cache set to 50% of max memory (up to 75%)
              # '-x',
              *other_args,
          ]  # Clears the internal tile cache after writing a complete row to the target file

          # LOGs
          LOGGER.debug(gpt_cli)
          if display_snap_opt:
              misc.run_cli(gpt_cli + ["--diag"])

          return gpt_cli