parscival_spec_version: '2.1.0'
description: 'Europresse HTML parscival specification'
source: 'europresse'
schema: 'markup'
format: 'html'
version: '1.0.0'
author: 'martinec'

# Europresse documentation is avalable at
# https://www.europresse.com

# general options
options:
  # only keep parsed keys listed under 'keys:'
  only_requested_keys: true
  # indicates whether the temporary directories should
  # be cleaned up after processing
  remove_transient: true

# defines the data keys to be parsed and mapped
keys:
  # list of keys to be used during parsing
  # it is suggested the use of two-, three-, four-character uppercase variable names
  parsing:
    AU:  # Author
      type: string
      qualifier: optional
    AN:  # Aside notes
      type: string
      qualifier: optional
    AB:  # Content
      type: string
      qualifier: required
    DS:  # Copyright info
      type: string
      qualifier: optional
    HD:  # Header
      type: string
      qualifier: optional
    IB:  # Info Box (encadré)
      type: string
      qualifier: optional
    PID: # Publication ID
      type: string
      qualifier: required
    SC:  # Publication source code
      type: string
      qualifier: optional
    SD:  # Other source date
      type: string
      qualifier: optional
    SN:  # Other source name
      type: string
      qualifier: optional
    SO:  # Journal Title
      type: string
      qualifier: required
    ST:  # Subtitle
      type: string
      qualifier: optional
    TI:  # Title
      type: string
      qualifier: required

  # list of keys to be used during mapping
  mapping:
    # Unique identifier for the article
    ArticleID:
      type: string
    # Name(s) of the author(s) of the article
    Author:
      type: string
    # Main content/body of the article
    Content:
      type: string
    # Header information of the article
    Header:
      type: string
    # Unique identifier used within the system
    ID:
      type: string
    # Additional information or summary box
    InfoBox:
      type: string
    # Publication date in ISI format
    ISIpubdate:
      type: integer
    # Name of the journal or publication
    Journal:
      type: string
    # Legal or copyright information
    LegalInfo:
      type: string
    # Additional notes or comments
    Notes:
      type: string
    # Release date of the other source
    OtherSourceDate:
      type: string
    # Name of the other source
    OtherSourceName:
      type: string
    # Classification of the publication: 'news', 'report', 'web',...
    PubClass:
      type: string
    # Publication date
    PubDate:
      type: string
    # Type of the publication, fixed to 'J'
    PubType:
      type: string
    # Source of the publication, fixed to 'Europresse'
    PubSource:
      type: string
    # Year of publication
    PubYear:
      type: string
    # Identifier of the source
    SourceID:
      type: string
    # Code representing the source
    SourceCode:
      type: string
    # Subtitle of the article
    Subtitle:
      type: string
    # Title of the article
    Title:
      type: string
    # Volume number of the journal or publication
    Volume:
      type: string

# parse a HTML document by specifying the query selectors needed to extract
# the keys defined on 'keys.parsing'
parsing:
  # name of the parsing category
  lquery:
    # name of the parsing plugin
    type: 'htmlq'
    # the beginning of each document
    record_separator: '<article>'
    # the end of each document
    record_finalizer: '</article>'
    keys:
      AU:  # Author
        selectors:
          # Example:
          # <div class="docAuthors">John Doe</div>
          # try to use .docAuthors to find the author
          - query: 'div.docAuthors'
            type: 'text'
          # alternatively, use the last .sm-margin-bottomNews element
          - query: '.sm-margin-bottomNews'
            chain:
              # select only the last item
              - method: 'eq'
                params: -1
            type: 'text'
      AN:  # Aside Notes
        selectors:
          # Example:
          # <aside>
          #   <div>Note(s): This is an important aside note.</div>
          # </aside>
          - query: 'aside'
            chain:
              # select those contains the string 'Note'
              - method: 'filter'
                params: ':contains("Note")'
              # keep only the first element
              - method: 'eq'
                params: 0
            type: 'text'
      AB:  # Content
        selectors:
          # Example:
          # <div class="DocText">
          #   <p>At the beginning of the COVID-19 ...</p>
          # </div>
          # try to use .DocText
          - query: 'div.DocText'
            type: 'text'
          # alternatively, .docOcurrContainer
          - query: 'div.docOcurrContainer'
            type: 'text'
      DS:  # Copyright info
        selectors:
          # Example:
          # <div class="Doc-LegalInfo">
          #   <small>© 2022 Acme University. All rights reserved.</small>
          # </div>
          - query: 'div.Doc-LegalInfo small'
            chain:
              # select only the first element
              - method: 'eq'
                params: 0
            type: 'text'
      HD:  # Header
        selectors:
          # Example:
          # <div class="rdp__DocHeader">
          #   <span class="DocHeader">Thursday, December 22, 2022 13887 mots, p. 819(40)</span>
          # </div>
          - query: '.DocHeader'
            type: 'text'
      IB:  # Info Box
        selectors:
          # Example:
          # <div class="rdp__newsdoc">
          #   Information box content here
          # </div>
          - query: 'div.rdp__newsdoc'
            type: 'text'
      PID: # Publication ID
        selectors:
          # Example:
          # <div class="publiC-lblNodoc">news·20221222·XXXXX·000000000</div>
          - query: '.publiC-lblNodoc'
            type: 'text'
      SC:  # Code of the source
        selectors:
          # Example:
          # <span sourcecode="XXXXX"></span>
          - query: 'span[sourcecode]'
            # we specify that we need the attribute 'sourcecode' instant of
            # the content of the element 'span'
            type: 'attr'
            attribute: 'sourcecode'
      SD:  # Other source date
        selectors:
          # Example:
          # <div class="apd-sources-date">2022-12-22</div>
          - query: '.apd-sources-date'
            # for a single document, having multiple results for this selector
            # each item retrieved is preserved in a list
            type: 'list_text'
      SN:  # Other source name
        selectors:
          # Example:
          # <div class="apd-sources-date">2022-12-22</div>
          # <td>
          #   <div class="source-name-APD">Source Name 1</div>
          #   <div class="source-name-APD">Source Name 2</div>
          # </td>
          # <div class="apd-sources-date">2022-12-23</div>
          # <td>
          #   <div class="source-name-APD">Source Name 3</div>
          # </td>
          - query: '.apd-sources-date'
            # for a single document, having multiple results for this selector
            # each item retrieved is preserved in a list
            type: 'list_text'
            # for each item retrieved, the following chain of actions is performed
            item_chain:
              # select the closest parent 'td' element
              - method: 'closest'
                params: 'td'
              # move to the next sibling 'td' element
              - method: 'next_all'
                params: 'td'
              # find all elements with the class '.source-name-APD' within the
              # current context
              - method: 'find'
                params: '.source-name-APD'
      SO:  # Publication Name
        selectors:
          # Example:
          # <div class="rdp__DocPublicationName">
          #   <span class="DocPublicationName">Acme Law Review, Vol. 14, no 4</span>
          # </div>
          - query: '.DocPublicationName'
            type: 'text'
      ST:  # Subtitle
        selectors:
          # Example:
          # <div class="rdp__subtitle">Subtitle goes here</div>
          - query: '.rdp__subtitle'
            type: 'text'
      TI:  # Title, use the first non empty text found
        selectors:
          # Example:
          # <div class="titreArticle">
          #   <p class="titreArticleVisu">Title goes here</p>
          # </div>
          # try to use .titreArticle
          - query: '.titreArticle'
            type: 'text'
          # alternatively, .grandTitre
          - query: '.grandTitre'
            type: 'text'
          # as last-chance, .titreArticleVisu
          - query: '.titreArticleVisu'
            type: 'text'

# specifies how to convert the parsed keys into output keys
mapping:
  # from a single key source to multiple key targets
  source_targets:
    # AU |-> Author
    AU:
      - target: 'Author'
        plugins:
          # remove "email protected" from Author text
          - transform: 'regex_sub'
            params:
              regex: "\\s+\\[?email protected\\]?"
              repl: ''
    # AN |-> Notes
    AN:
      - target: 'Notes'
        plugins:
          # replace 1 or more new lines by a space
          - transform: 'regex_sub'
            params:
              regex: "\\s+"
              repl: ' '
          # from Aside Notes, keep only them starting by the string 'Note(s)'
          - transform: 'regex_match_filter'
            params:
              regex: '^Note(\(s\)\s*:\s*)?(.*)$'
              value: '{{_[1]}}'
    # AB |-> Content
    AB:
      - target: 'Content'
        plugins:
          # replace 1 or more new lines by a space
          - transform: 'regex_sub'
            params:
              regex: "\\s+"
              repl: ' '
    # DS |-> LegalInfo
    DS:
      - target: 'LegalInfo'
    # IB |-> InfoBox
    IB:
      - target: 'InfoBox'
        plugins:
          # replace 1 or more new lines by a space
          - transform: 'regex_sub'
            params:
              regex: "\\s+"
              repl: ' '
    # HD |-> Header
    HD:
      - target: 'Header'
    # SO |-> Journal
    SO:
      - target: 'Journal'
        plugins:
          # select Journal name
          - transform: 'regex_format'
            params:
              regex: '^([^,]+)\s*,?\s*(.*)$'
              value: '{{_[0]}}'
      - target: 'Volume'
        plugins:
          # select Volume
          - transform: 'regex_format'
            params:
              regex: '^([^,]+)\s*,\s*(.*[Vv]ol.*)$'
              value: '{{_[1]}}'
              ignore_unmatch: true
              fallback: ''
    # PID |-> ID, PubClass, PubDate, SourceID, ArticleID
    PID:
      - target: 'ID'
      - target: 'PubClass'
        plugins:
          # convert PID in the form 'foo\u00b720231019\u00b7bar\u00b7zaz'
          # into 'fo'
          - transform: 'regex_format'
            params:
              regex: '\b([^·]+)·([^·]+)·([^·]+)·([^·]+)\b'
              value: '{{_[0]}}'
      - target: 'PubDate'
        plugins:
          # convert PID in the form 'foo\u00b720231019\u00b7bar\u00b7zaz'
          # into 'YYYY/MM/dd'
          - transform: 'regex_format'
            params:
              regex: '\b([^·]+)·([^·]+)·([^·]+)·([^·]+)\b'
              value: '{{_[1]}}'
          # parse string as year-month-day (e.g. 2023-10-19)
          - transform: 'date_format'
            params:
              format: '%Y-%m-%d'
              # as fallback keep only the first 4 characters
              fallback: '{{ _[0][0:4] }}'
      - target: 'SourceID'
        plugins:
          # convert PID in the form 'foo\u00b720231019\u00b7bar\u00b7zaz'
          # into 'bar'
          - transform: 'regex_format'
            params:
              regex: '\b([^·]+)·([^·]+)·([^·]+)·([^·]+)\b'
              value: '{{_[2]}}'
      - target: 'ArticleID'
        plugins:
          # convert PID in the form 'foo\u00b720231019\u00b7bar\u00b7zaz'
          # into 'bar:zaz'
          - transform: 'regex_format'
            params:
              regex: '\b([^·]+)·([^·]+)·([^·]+)·([^·]+)\b'
              value: '{{_[2]}}:{{_[3]}}'
    # SC |-> SourceCode
    SC:
      - target: 'SourceCode'
    # SD |-> OtherSourceDate
    SD:
      - target: 'OtherSourceDate'
        plugins:
          # each item on the parsed SD increases the rank
          - transform: 'data_ranker'
    # SN |-> OtherSourceName
    SN:
      - target: 'OtherSourceName'
        plugins:
          # each item on the parsed SN increases the rank
          - transform: 'data_ranker'
            params:
              # each subitem on the parsed SN increases the parserank
              parserank: true
    # ST |-> SubTitle
    ST:
      - target: 'Subtitle'
    # TI |-> Title
    TI:
      - target: 'Title'
  # a single key target from a string template
  target_template:
    # PubDateSource:
    #   template: '{{OtherSourceDate}}: {{OtherSourceName}}'
    # |-> ISIpubdate
    ISIpubdate:
      # template with a dynamic variable
      template: '{{PubDate}}'
    # |-> PubType
    PubType:
      # template with a literal value for all nodes
      template: 'J'
    # |-> PubSource
    PubSource:
      # template with a literal value for all nodes
      template: 'Europresse'
    # |-> PubYear
    PubYear:
      # template with a dynamic variable, followed by a
      # mapping plugin able to parse and format dates
      template: '{{PubDate}}'
      plugins:
        # parse string as year (e.g. 2023)
        - transform: 'date_format'
          params:
            format: '%Y'
            # as fallback keep only the first 4 characters
            fallback: '{{ _[0][0:4] }}'

# specifies curating tasks to be done after/before
# ingesting, parsing, mapping or storing
curating:
  # curation actions to be taken before ingesting data
  before_ingesting:
    plugins:
      # list of plugins to be executed before ingesting
      - encode: 'convert'
        # parameters for the encoding plugin
        params:
          # encoding settings
          encode:
            # source encoding. 'guess' will attempt to detect
            # the encoding
            from: 'guess'
            # target encoding to convert the files to
            to: 'utf-8'
          # policy to apply. 'only-non-complaint' means only files
          # that don't meet the expected encoding and newline
          # criteria will be processed
          policy: 'only-non-complaint'
          # newline character to use in the output files. 'LF'
          # stands for Line Feed (Unix-style newlines)
          # valid values are [LF|CR|CRLF]
          newline: 'LF'
          # temporary directory settings
          transient:
            # base directory for creating temporary directories
            basedir: '/tmp'
            # indicates whether the temporary directories should
            # be cleaned up after processing
            cleanable: true
          # enable or disable this plugin
          enabled: true

  # curation actions to be taken after mapping data
  after_mapping:
    plugins:
      - match: 'elg_node_matcher'
        params:
          # name of the node to process
          node: 'OtherSourceName'
          # path to the registry file
          registry: 'europresse-journal-name.csv'
          # strict: clear non fully matched nodes
          # moderate: keep partially matched nodes
          # relaxed: keep non matched nodes
          filter_mode: 'relaxed'
          # normalize functions to be applied on each node item before matching
          normalize:
            # lowercase item data
            - lowercase: true
            # collapse multiple spaces
            - collapse: true
            # remove leading and trailing whitespace
            - trim: true
          # cache options
          cache:
            # directory for cache storage
            dir: './cache'
            # store the compiled registry in the cache
            store_compiled_registry: true
            # store the match results in the cache
            store_result: false
            # use the cached compiled registry
            use_cached_registry: true
            # use the cached match results
            use_cached_result: false
          # match options
          matches:
            # policy for handling ambiguous matches
            # options: keep, warn, ignore
            ambiguous_policy: 'warn'
            # delimiter for separating ambiguous matches
            ambiguous_delimiter: ' *** '
          # verbosity level
          # options: info, error, none
          verbose: 'error'
          # enable or disable the plugin
          enabled: true

  # curation actions to be taken before storing mapped data
  before_storing:
    plugins:
      - deduplicate: 'hash_key_deduplicator'
        params:
          hash_key: '{{ID}}'

# specifies how to convert the mapped data into output files
storing:
  # create a .json file
  cortext.json:
    plugins:
      # apply the 'render_template' plugin to transform the data using
      # the specified template
      - plain: 'render_template'
        params:
          # template file to be used for rendering the JSON output
          filename: 'cortext.json.tpl'

  # create both .sqlite and .db files
  cortext.sqlite:
    plugins:
      # apply the 'render_template' plugin to transform the data using
      # the specified template
      - plain: 'render_template'
        params:
          # template file to be used for rendering the SQLite script
          filename: 'cortext.sqlite.tpl'
      # apply the 'execute_sqlite_script' plugin to execute the rendered
      # SQLite script
      - binary: 'execute_sqlite_script'
