#!/bin/bash

# This example illustrates a simple technique to automatically upload a workflow's output files to
# Amazon S3. It suggests recursively uploading the "out" directory generated by miniwdl run, which
# contains tidy symlinks to the output files in their original locations (typically scattered
# throughout the run subdirectories of individual tasks and sub-workflows).

S3_DEST="s3://YOUR-BUCKET/miniwdl_upload_output_files_test/"

set -euo pipefail

# example workflow which outputs a couple of files
cat << 'EOF' > /tmp/hello.wdl
    version 1.0
    workflow hello_caller {
        input {
            Array[String] who
        }
        scatter (name in who) {
            call hello {
                input:
                    who = name
            }
        }
        output {
            Array[File] messages = hello.message
        }
    }
    task hello {
        input {
            String who
        }
        command {
            echo "Hello, ~{who}!" | tee message.txt 1>&2
        }
        output {
            File message = "message.txt"
        }
        runtime {
            docker: "ubuntu:18.04"
            memory: "1G"
        }
    }
EOF

# miniwdl run and capture its stdout JSON
miniwdl_stdout=$(miniwdl run /tmp/hello.wdl who=Alice who=Bob --dir=/tmp)

# upload output files to S3 by recursively walking the generated out directory and
# following the symlinks therein
dir_to_upload=$(jq -r .dir <(echo "${miniwdl_stdout}"))
dir_to_upload="${dir_to_upload}/out/"
aws s3 sync --follow-symlinks "$dir_to_upload" "$S3_DEST"

# Rewrite File paths in the outputs JSON to the new S3 URIs.
python3 - <<EOF
import json
import os
def rewrite(v):
    if isinstance(v, str) and v.startswith("${dir_to_upload}"):
        v = os.path.normpath(v)
        if v.startswith("${dir_to_upload}") and os.path.islink(v):
            return "${S3_DEST}" + os.path.relpath(v, "${dir_to_upload}")
    if isinstance(v, list):
        return [rewrite(u) for u in v]
    if isinstance(v, dict):
        return dict((k, rewrite(u)) for (k,u) in v.items())
    return v
outp=json.loads("""
${miniwdl_stdout}
""")
print(json.dumps(rewrite(outp["outputs"]), indent=2))
EOF
# Note: the JSON format doesn't clearly distinguish String and File values, so there's a risk of
# erroneously rewriting String values that look like File paths. In practice this risk is minimal
# because miniwdl generates absolute paths for File values, which the above script checks for.
# Still, a more precise method could be implemented by plugging into the miniwdl runtime, where the
# detailed output types are known.
