//------------------------------------------------------------------------------
/*
    This file is part of rippled: https://github.com/ripple/rippled
    Copyright 2016 Ripple Labs Inc.

    Permission to use, copy, modify, and/or distribute this software for any
    purpose  with  or without fee is hereby granted, provided that the above
    copyright notice and this permission notice appear in all copies.

    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================

#include <ripple/app/misc/ValidatorSite.h>
#include <ripple/basics/Slice.h>
#include <ripple/basics/base64.h>
#include <ripple/basics/strHex.h>
#include <ripple/protocol/HashPrefix.h>
#include <ripple/protocol/PublicKey.h>
#include <ripple/protocol/SecretKey.h>
#include <ripple/protocol/Sign.h>
#include <ripple/protocol/digest.h>
#include <ripple/protocol/jss.h>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/asio.hpp>
#include <boost/range/adaptor/transformed.hpp>
#include <chrono>
#include <date/date.h>
#include <test/jtx.h>
#include <test/jtx/TrustedPublisherServer.h>
#include <test/unit_test/FileDirGuard.h>

namespace ripple {
namespace test {
namespace detail {
constexpr const char*
realValidatorContents()
{
    return R"vl({
        "public_key": "ED2677ABFFD1B33AC6FBC3062B71F1E8397C1505E1C42C64D11AD1B28FF73F4734",
        "manifest": "JAAAAAFxIe0md6v/0bM6xvvDBitx8eg5fBUF4cQsZNEa0bKP9z9HNHMh7V0AnEi5D4odY9X2sx+cY8B3OHNjJvMhARRPtTHmWnAhdkDFcg53dAQS1WDMQDLIs2wwwHpScrUnjp1iZwwTXVXXsaRxLztycioto3JgImGdukXubbrjeqCNU02f7Y/+6w0BcBJA3M0EOU+39hmB8vwfgernXZIDQ1+o0dnuXjX73oDLgsacwXzLBVOdBpSAsJwYD+nW8YaSacOHEsWaPlof05EsAg==",
        "blob" : " ",
        "signature" : "9FF30EDC4DED7ABCD0D36389B7C716EED4B5E4F043902853534EBAC7BE966BB3813D5CF25E4DADA5E657CCF019FFD11847FD3CC44B5559A6FCEEE4C3DCFF8D0E",
        "version": 1
}
)vl";
}

auto constexpr default_expires = std::chrono::seconds{3600};
auto constexpr default_effective_overlap = std::chrono::seconds{30};
}  // namespace detail

class ValidatorSite_test : public beast::unit_test::suite
{
private:
    using Validator = TrustedPublisherServer::Validator;

    void
    testConfigLoad()
    {
        testcase("Config Load");

        using namespace jtx;

        Env env(*this, envconfig(), nullptr, beast::severities::kDisabled);
        auto trustedSites =
            std::make_unique<ValidatorSite>(env.app(), env.journal);

        // load should accept empty sites list
        std::vector<std::string> emptyCfgSites;
        BEAST_EXPECT(trustedSites->load(emptyCfgSites));

        // load should accept valid validator site uris
        std::vector<std::string> cfgSites({
            "http://ripple.com/", "http://ripple.com/validators",
                "http://ripple.com:8080/validators",
                "http://207.261.33.37/validators",
                "http://207.261.33.37:8080/validators",
                "https://ripple.com/validators",
                "https://ripple.com:443/validators",
                "file:///etc/opt/ripple/validators.txt",
                "file:///C:/Lib/validators.txt"
#if !_MSC_VER
                ,
                "file:///"
#endif
        });
        BEAST_EXPECT(trustedSites->load(cfgSites));

        // load should reject validator site uris with invalid schemes
        std::vector<std::string> badSites({"ftp://ripple.com/validators"});
        BEAST_EXPECT(!trustedSites->load(badSites));

        badSites[0] = "wss://ripple.com/validators";
        BEAST_EXPECT(!trustedSites->load(badSites));

        badSites[0] = "ripple.com/validators";
        BEAST_EXPECT(!trustedSites->load(badSites));

        // Host names are not supported for file URLs
        badSites[0] = "file://ripple.com/vl.txt";
        BEAST_EXPECT(!trustedSites->load(badSites));

        // Even local host names are not supported for file URLs
        badSites[0] = "file://localhost/home/user/vl.txt";
        BEAST_EXPECT(!trustedSites->load(badSites));

        // Nor IP addresses
        badSites[0] = "file://127.0.0.1/home/user/vl.txt";
        BEAST_EXPECT(!trustedSites->load(badSites));

        // File URL path can not be empty
        badSites[0] = "file://";
        BEAST_EXPECT(!trustedSites->load(badSites));

#if _MSC_VER  // Windows paths strip off the leading /, leaving the path empty
        // File URL path can not be a directory
        // (/ is the only path we can reasonably assume is a directory)
        badSites[0] = "file:///";
        BEAST_EXPECT(!trustedSites->load(badSites));
#endif
    }

    struct FetchListConfig
    {
        std::string path;
        std::string msg;
        bool ssl;
        bool failFetch = false;
        bool failApply = false;
        int serverVersion = 1;
        std::chrono::seconds expiresFromNow = detail::default_expires;
        std::chrono::seconds effectiveOverlap =
            detail::default_effective_overlap;
        int expectedRefreshMin = 0;
    };
    void
    testFetchList(
        detail::DirGuard const& good,
        std::vector<FetchListConfig> const& paths)
    {
        testcase << "Fetch list - "
                 << boost::algorithm::join(
                        paths |
                            boost::adaptors::transformed(
                                [](FetchListConfig const& cfg) {
                                    return cfg.path +
                                        (cfg.ssl ? " [https] v" : " [http] v") +
                                        std::to_string(cfg.serverVersion) +
                                        " " + cfg.msg;
                                }),
                        ", ");

        using namespace jtx;
        using namespace std::chrono_literals;

        Env env(*this, [&]() {
            auto p = test::jtx::envconfig();
            p->legacy("database_path", good.subdir().string());
            return p;
        }());
        auto& trustedKeys = env.app().validators();
        env.timeKeeper().set(env.timeKeeper().now() + 30s);

        test::StreamSink sink;
        beast::Journal journal{sink};

        PublicKey emptyLocalKey;
        std::vector<std::string> emptyCfgKeys;
        struct publisher
        {
            publisher(FetchListConfig const& c) : cfg{c}
            {
            }
            std::shared_ptr<TrustedPublisherServer> server;
            std::vector<Validator> list;
            std::string uri;
            FetchListConfig const& cfg;
            bool isRetry;
        };
        std::vector<publisher> servers;

        auto constexpr listSize = 20;
        std::vector<std::string> cfgPublishers;

        for (auto const& cfg : paths)
        {
            servers.push_back(cfg);
            auto& item = servers.back();
            item.isRetry = cfg.path == "/bad-resource";
            item.list.reserve(listSize);
            while (item.list.size() < listSize)
                item.list.push_back(TrustedPublisherServer::randomValidator());

            NetClock::time_point const expires =
                env.timeKeeper().now() + cfg.expiresFromNow;
            NetClock::time_point const effective2 =
                expires - cfg.effectiveOverlap;
            NetClock::time_point const expires2 =
                effective2 + cfg.expiresFromNow;
            item.server = make_TrustedPublisherServer(
                env.app().getIOService(),
                item.list,
                expires,
                {{effective2, expires2}},
                cfg.ssl,
                cfg.serverVersion);
            std::string pubHex = strHex(item.server->publisherPublic());
            cfgPublishers.push_back(pubHex);

            if (item.cfg.failFetch)
            {
                // Create a cache file
                auto const name = good.subdir() / ("cache." + pubHex);
                std::ofstream o(name.string());
                o << "{}";
            }

            std::stringstream uri;
            uri << (cfg.ssl ? "https://" : "http://")
                << item.server->local_endpoint() << cfg.path;
            item.uri = uri.str();
        }

        BEAST_EXPECT(
            trustedKeys.load(emptyLocalKey, emptyCfgKeys, cfgPublishers));

        // Normally, tests will only need a fraction of this time,
        // but sometimes DNS resolution takes an inordinate amount
        // of time, so the test will just wait.
        auto sites = std::make_unique<ValidatorSite>(env.app(), journal, 12s);

        std::vector<std::string> uris;
        for (auto const& u : servers)
            uris.push_back(u.uri);
        sites->load(uris);
        sites->start();
        sites->join();

        auto const jv = sites->getJson();
        for (auto const& u : servers)
        {
            for (auto const& val : u.list)
            {
                BEAST_EXPECT(
                    trustedKeys.listed(val.masterPublic) != u.cfg.failApply);
                BEAST_EXPECT(
                    trustedKeys.listed(val.signingPublic) != u.cfg.failApply);
            }

            Json::Value myStatus;
            for (auto const& vs : jv[jss::validator_sites])
                if (vs[jss::uri].asString().find(u.uri) != std::string::npos)
                    myStatus = vs;
            BEAST_EXPECTS(
                myStatus[jss::last_refresh_message].asString().empty() !=
                    u.cfg.failFetch,
                to_string(myStatus) + "\n" + sink.messages().str());

            if (!u.cfg.msg.empty())
            {
                BEAST_EXPECTS(
                    sink.messages().str().find(u.cfg.msg) != std::string::npos,
                    sink.messages().str());
            }

            if (u.cfg.expectedRefreshMin)
            {
                BEAST_EXPECTS(
                    myStatus[jss::refresh_interval_min].asInt() ==
                        u.cfg.expectedRefreshMin,
                    to_string(myStatus));
            }

            if (u.cfg.failFetch)
            {
                using namespace std::chrono;
                std::stringstream nextRefreshStr{
                    myStatus[jss::next_refresh_time].asString()};
                system_clock::time_point nextRefresh;
                date::from_stream(nextRefreshStr, "%Y-%b-%d %T", nextRefresh);
                BEAST_EXPECT(!nextRefreshStr.fail());
                auto now = system_clock::now();
                BEAST_EXPECTS(
                    nextRefresh <= now + (u.isRetry ? seconds{30} : minutes{5}),
                    "Now: " + to_string(now) + ", NR: " + nextRefreshStr.str());
            }
        }
    }

    void
    testFileList(std::vector<std::pair<std::string, std::string>> const& paths)
    {
        testcase << "File list - " << paths[0].first
                 << (paths.size() > 1 ? ", " + paths[1].first : "");

        using namespace jtx;

        Env env(*this);

        test::StreamSink sink;
        beast::Journal journal{sink};

        struct publisher
        {
            std::string uri;
            std::string expectMsg;
            bool shouldFail;
        };
        std::vector<publisher> servers;

        for (auto const& cfg : paths)
        {
            servers.push_back({});
            auto& item = servers.back();
            item.shouldFail = !cfg.second.empty();
            item.expectMsg = cfg.second;

            std::stringstream uri;
            uri << "file://" << cfg.first;
            item.uri = uri.str();
        }

        auto sites = std::make_unique<ValidatorSite>(env.app(), journal);

        std::vector<std::string> uris;
        for (auto const& u : servers)
            uris.push_back(u.uri);
        sites->load(uris);
        sites->start();
        sites->join();

        for (auto const& u : servers)
        {
            auto const jv = sites->getJson();
            Json::Value myStatus;
            for (auto const& vs : jv[jss::validator_sites])
                if (vs[jss::uri].asString().find(u.uri) != std::string::npos)
                    myStatus = vs;
            BEAST_EXPECTS(
                myStatus[jss::last_refresh_message].asString().empty() !=
                    u.shouldFail,
                to_string(myStatus));
            if (u.shouldFail)
            {
                BEAST_EXPECTS(
                    sink.messages().str().find(u.expectMsg) !=
                        std::string::npos,
                    sink.messages().str());
            }
        }
    }

    void
    testFileURLs()
    {
        auto fullPath = [](detail::FileDirGuard const& guard) {
            auto absPath = absolute(guard.file()).string();
            if (absPath.front() != '/')
                absPath.insert(absPath.begin(), '/');
            return absPath;
        };
        {
            // Create a file with a real validator list
            detail::FileDirGuard good(
                *this, "test_val", "vl.txt", detail::realValidatorContents());
            // Create a file with arbitrary content
            detail::FileDirGuard hello(
                *this, "test_val", "helloworld.txt", "Hello, world!");
            // Create a file with malformed Json
            detail::FileDirGuard json(
                *this,
                "test_val",
                "json.txt",
                R"json({ "version": 2, "extra" : "value" })json");
            auto const goodPath = fullPath(good);
            auto const helloPath = fullPath(hello);
            auto const jsonPath = fullPath(json);
            auto const missingPath = jsonPath + ".bad";
            testFileList({
                {goodPath, ""},
                {helloPath,
                 "Unable to parse JSON response from  file://" + helloPath},
                {jsonPath,
                 "Missing fields in JSON response from  file://" + jsonPath},
                {missingPath, "Problem retrieving from file://" + missingPath},
            });
        }
    }

public:
    void
    run() override
    {
        testConfigLoad();

        detail::DirGuard good(*this, "test_fetch");
        for (auto ssl : {true, false})
        {
            // fetch single site
            testFetchList(good, {{"/validators", "", ssl}});
            testFetchList(good, {{"/validators2", "", ssl}});
            // fetch multiple sites
            testFetchList(
                good, {{"/validators", "", ssl}, {"/validators", "", ssl}});
            testFetchList(
                good, {{"/validators", "", ssl}, {"/validators2", "", ssl}});
            testFetchList(
                good, {{"/validators2", "", ssl}, {"/validators", "", ssl}});
            testFetchList(
                good, {{"/validators2", "", ssl}, {"/validators2", "", ssl}});
            // fetch single site with single redirects
            testFetchList(good, {{"/redirect_once/301", "", ssl}});
            testFetchList(good, {{"/redirect_once/302", "", ssl}});
            testFetchList(good, {{"/redirect_once/307", "", ssl}});
            testFetchList(good, {{"/redirect_once/308", "", ssl}});
            // one redirect, one not
            testFetchList(
                good,
                {{"/validators", "", ssl}, {"/redirect_once/302", "", ssl}});
            testFetchList(
                good,
                {{"/validators2", "", ssl}, {"/redirect_once/302", "", ssl}});
            // UNLs with a "gap" between validUntil of one and validFrom of the
            // next
            testFetchList(
                good,
                {{"/validators2",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  std::chrono::seconds{-90}}});
            // fetch single site with undending redirect (fails to load)
            testFetchList(
                good,
                {{"/redirect_forever/301",
                  "Exceeded max redirects",
                  ssl,
                  true,
                  true}});
            // two that redirect forever
            testFetchList(
                good,
                {{"/redirect_forever/307",
                  "Exceeded max redirects",
                  ssl,
                  true,
                  true},
                 {"/redirect_forever/308",
                  "Exceeded max redirects",
                  ssl,
                  true,
                  true}});
            // one undending redirect, one not
            testFetchList(
                good,
                {{"/validators", "", ssl},
                 {"/redirect_forever/302",
                  "Exceeded max redirects",
                  ssl,
                  true,
                  true}});
            // one undending redirect, one not
            testFetchList(
                good,
                {{"/validators2", "", ssl},
                 {"/redirect_forever/302",
                  "Exceeded max redirects",
                  ssl,
                  true,
                  true}});
            // invalid redir Location
            testFetchList(
                good,
                {{"/redirect_to/ftp://invalid-url/302",
                  "Invalid redirect location",
                  ssl,
                  true,
                  true}});
            testFetchList(
                good,
                {{"/redirect_to/file://invalid-url/302",
                  "Invalid redirect location",
                  ssl,
                  true,
                  true}});
            // invalid json
            testFetchList(
                good,
                {{"/validators/bad",
                  "Unable to parse JSON response",
                  ssl,
                  true,
                  true}});
            testFetchList(
                good,
                {{"/validators2/bad",
                  "Unable to parse JSON response",
                  ssl,
                  true,
                  true}});
            // error status returned
            testFetchList(
                good,
                {{"/bad-resource", "returned bad status", ssl, true, true}});
            // location field missing
            testFetchList(
                good,
                {{"/redirect_nolo/308",
                  "returned a redirect with no Location",
                  ssl,
                  true,
                  true}});
            // json fields missing
            testFetchList(
                good,
                {{"/validators/missing",
                  "Missing fields in JSON response",
                  ssl,
                  true,
                  true}});
            testFetchList(
                good,
                {{"/validators2/missing",
                  "Missing fields in JSON response",
                  ssl,
                  true,
                  true}});
            // timeout
            testFetchList(
                good, {{"/sleep/13", "took too long", ssl, true, true}});
            // bad manifest format using known versions
            // * Retrieves a v1 formatted list claiming version 2
            testFetchList(
                good, {{"/validators", "Missing fields", ssl, true, true, 2}});
            // * Retrieves a v2 formatted list claiming version 1
            testFetchList(
                good, {{"/validators2", "Missing fields", ssl, true, true, 0}});
            // bad manifest version
            // Because versions other than 1 are treated as v2, the v1
            // list won't have the blobs_v2 fields, and thus will claim to have
            // missing fields
            testFetchList(
                good, {{"/validators", "Missing fields", ssl, true, true, 4}});
            testFetchList(
                good,
                {{"/validators2",
                  "1 unsupported version",
                  ssl,
                  false,
                  true,
                  4}});
            using namespace std::chrono_literals;
            // get expired validator list
            testFetchList(
                good,
                {{"/validators",
                  "Applied 1 expired validator list(s)",
                  ssl,
                  false,
                  false,
                  1,
                  0s}});
            testFetchList(
                good,
                {{"/validators2",
                  "Applied 1 expired validator list(s)",
                  ssl,
                  false,
                  false,
                  1,
                  0s,
                  -1s}});
            // force an out-of-range validUntil value
            testFetchList(
                good,
                {{"/validators",
                  "1 invalid validator list(s)",
                  ssl,
                  false,
                  true,
                  1,
                  std::chrono::seconds{Json::Value::maxInt + 1}}});
            // force an out-of-range validUntil value on the future list
            // The first list is accepted. The second fails. The parser
            // returns the "best" result, so this looks like a success.
            testFetchList(
                good,
                {{"/validators2",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  std::chrono::seconds{Json::Value::maxInt - 300},
                  299s}});
            // force an out-of-range validFrom value
            // The first list is accepted. The second fails. The parser
            // returns the "best" result, so this looks like a success.
            testFetchList(
                good,
                {{"/validators2",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  std::chrono::seconds{Json::Value::maxInt - 300},
                  301s}});
            // force an out-of-range validUntil value on _both_ lists
            testFetchList(
                good,
                {{"/validators2",
                  "2 invalid validator list(s)",
                  ssl,
                  false,
                  true,
                  1,
                  std::chrono::seconds{Json::Value::maxInt + 1},
                  std::chrono::seconds{Json::Value::maxInt - 6000}}});
            // verify refresh intervals are properly clamped
            testFetchList(
                good,
                {{"/validators/refresh/0",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  detail::default_effective_overlap,
                  1}});  // minimum of 1 minute
            testFetchList(
                good,
                {{"/validators2/refresh/0",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  detail::default_effective_overlap,
                  1}});  // minimum of 1 minute
            testFetchList(
                good,
                {{"/validators/refresh/10",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  detail::default_effective_overlap,
                  10}});  // 10 minutes is fine
            testFetchList(
                good,
                {{"/validators2/refresh/10",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  detail::default_effective_overlap,
                  10}});  // 10 minutes is fine
            testFetchList(
                good,
                {{"/validators/refresh/2000",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  detail::default_effective_overlap,
                  60 * 24}});  // max of 24 hours
            testFetchList(
                good,
                {{"/validators2/refresh/2000",
                  "",
                  ssl,
                  false,
                  false,
                  1,
                  detail::default_expires,
                  detail::default_effective_overlap,
                  60 * 24}});  // max of 24 hours
        }
        using namespace boost::filesystem;
        for (auto const& file : directory_iterator(good.subdir()))
        {
            remove_all(file);
        }

        testFileURLs();
    }
};

BEAST_DEFINE_TESTSUITE_PRIO(ValidatorSite, app, ripple, 2);

}  // namespace test
}  // namespace ripple
