#  IBM Confidential
#  PID 5900-BAF
#  Copyright StreamSets Inc., an IBM Company 2025

"""This file contains an empty pipeline definition, and a method to create a pipeline definition for new flows."""

from copy import deepcopy
from uuid import uuid4

EMPTY_PIPELINE_DEFINITION = {
    "schemaVersion": 6,
    "version": 50,
    "pipelineId": "use a flow's executor id here",
    "title": "use a flow's executor id here",
    "description": "description",  # ignore since it's only in the datacollector
    "uuid": "create a new uuid here",
    "configuration": [
        {"name": "transformerEmrConnection.retryPolicyConfig.throttledBaseDelay", "value": 500},
        {"name": "googleCloudCredentialsConfig.credentialsProvider", "value": "DEFAULT_PROVIDER"},
        {"name": "googleCloudConfig.customRegion", "value": None},
        {"name": "googleCloudConfig.workerCount", "value": 2},
        {"name": "udfSchema", "value": None},
        {"name": "transformerEmrConnection.defineBootstrapActions", "value": False},
        {"name": "emrServerlessConnection.stopApplication", "value": True},
        {"name": "udfDb", "value": None},
        {"name": "transformerEmrConnection.awsConfig.credentialMode", "value": "WITH_CREDENTIALS"},
        {"name": "livyConfig.baseUrlHD", "value": "https://localhost/livy/"},
        {"name": "useSnowflakeConnection", "value": "false"},
        {"name": "emrServerlessConnection.retryPolicyConfig.throttledBaseDelay", "value": 500},
        {"name": "transformerEmrConnection.provisionedProductName", "value": None},
        {"name": "googleCloudCredentialsConfig.path", "value": None},
        {"name": "shouldRetry", "value": True},
        {"name": "transformerEmrConnection.clusterId", "value": None},
        {"name": "emrServerlessConnection.emrApplicationTags", "value": []},
        {"name": "transformerEmrConnection.productId", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.oauthClientSecret", "value": None},
        {"name": "transformerEmrConnection.awsConfig.sessionDuration", "value": 3600},
        {"name": "clusterConfig.yarnKerberosKeytabBase64Bytes", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.customUrl", "value": "jdbc:snowflake://[HOST]:[PORT]"},
        {"name": "emrServerlessConnection.applicationByNameAndTags", "value": False},
        {"name": "snowflakeCommonConfig.overrideRole", "value": None},
        {"name": "runnerIdleTIme", "value": 60},
        {"name": "emrServerlessConnectionSelection", "value": "MANUAL"},
        {"name": "executionMode", "value": "STANDALONE"},
        {"name": "snowflakeCommonConfig.snowflakeConnection.usePrivateLinkURL", "value": False},
        {"name": "snowflakeCommonConfig.snowflakeConnection.account", "value": None},
        {"name": "snowflakeCommonConfig.overrideWarehouse", "value": None},
        {"name": "warehouse", "value": None},
        {"name": "emrServerlessConnection.awsConfig.roleARN", "value": "arn:aws:iam::<account-id>:role/role-name"},
        {"name": "errorInformationLevel", "value": "FULL_ERROR_INFORMATION"},
        {"name": "schema", "value": None},
        {"name": "role", "value": None},
        {"name": "clusterConfig.useYarnKerberosKeytab", "value": False},
        {"name": "googleCloudConfig.gcsStagingUri", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.snowflakeAuthenticationMethod", "value": "KEYPAIR_CONTENT"},
        {"name": "logLevel", "value": "INFO"},
        {"name": "googleCloudConfig.region", "value": None},
        {"name": "googleCloudConfig.network", "value": None},
        {"name": "retryAttempts", "value": -1},
        {"name": "snowflakeCommonConfig.connectionSelection", "value": "MANUAL"},
        {"name": "databricksConfig.enableExternalResourceForExecutors", "value": True},
        {"name": "transformerEmrConnection.customTags", "value": []},
        {"name": "emrServerlessConnection.awsConfig.profileName", "value": None},
        {"name": "errorRecordPolicy", "value": "ORIGINAL_RECORD"},
        {"name": "clusterConfig.deployMode", "value": "CLIENT"},
        {"name": "transformerEmrConnection.terminateCluster", "value": True},
        {"name": "skipMetrics", "value": False},
        {"name": "snowflakeCommonConfig.snowflakeConnection.database", "value": None},
        {"name": "transformerEmrConnection.generateProductName", "value": False},
        {"name": "snowflakeCommonConfig.snowflakeConnection.password", "value": None},
        {"name": "emrServerlessConnection.retryPolicyConfig.baseDelay", "value": 100},
        {"name": "livyConfig.password", "value": None},
        {"name": "transformerRetryAttempts", "value": 3},
        {"name": "transformerEmrConnection.clusterTags", "value": []},
        {"name": "db", "value": None},
        {"name": "transformerEmrConnection.s3LogUri", "value": None},
        {"name": "googleCloudConfig.clusterName", "value": None},
        {"name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.customSigningRegion", "value": None},
        {"name": "transformerEmrConnection.awsConfig.awsAccessKeyId", "value": None},
        {"name": "googleCloudConfig.subnet", "value": None},
        {"name": "notifyOnStates", "value": ["RUN_ERROR", "STOPPED", "FINISHED"]},
        {"name": "emrServerlessConnection.s3LogUri", "value": None},
        {"name": "livyConfig.username", "value": None},
        {"name": "transformerEmrConnection.awsConfig.setSessionTags", "value": True},
        {"name": "transformerEmrConnection.loggingEnabled", "value": True},
        {"name": "googleCloudCredentialsConfig.credentialsFileContent", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.schema", "value": None},
        {"name": "googleCloudConfig.workerType", "value": None},
        {"name": "transformerEmrConnection.clusterByNameAndTags", "value": False},
        {"name": "transformerEmrConnection.bootstrapActionScripts", "value": []},
        {"name": "cdeConfig.resourceFilePrefix", "value": "streamsets"},
        {"name": "transformerEmrConnection.clusterPrefix", "value": None},
        {
            "name": "testOriginStage",
            "value": "streamsets-datacollector-dev-lib::"
            "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3",
        },
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.customSigningRegion", "value": None},
        {"name": "emrServerlessConnection.maximumMemory", "value": 3000},
        {"name": "transformerEmrConnection.region", "value": "US_WEST_2"},
        {"name": "clusterConfig.dataCacheReplicas", "value": 1},
        {"name": "transformerEmrConnectionSelection", "value": "MANUAL"},
        {"name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.signingRegion", "value": "US_WEST_2"},
        {"name": "transformerEmrConnection.serviceAccessSecurityGroup", "value": None},
        {"name": "ludicrousMode", "value": False},
        {"name": "databricksConfig.useUnityCatalog", "value": False},
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.regionalEndpoint", "value": None},
        {"name": "shouldCreateFailureSnapshot", "value": True},
        {"name": "databricksConfig.token", "value": None},
        {"name": "transformerEmrConnection.primaryInstanceType", "value": None},
        {"name": "emailIDs", "value": []},
        {"name": "transformerEmrConnection.ec2SubnetId", "value": None},
        {"name": "emrServerlessConnection.applicationPrefix", "value": None},
        {"name": "transformerEmrConnection.stepConcurrency", "value": 10},
        {"name": "cdeConfig.workloadPassword", "value": None},
        {
            "name": "stopEventStage",
            "value": "streamsets-datacollector-basic-lib::"
            "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1",
        },
        {"name": "clusterConfig.hadoopUserName", "value": None},
        {"name": "edgeHttpUrl", "value": "http://localhost:18633"},
        {"name": "transformerEMRConfig.executionRole", "value": None},
        {"name": "transformerEmrConnection.visibleToAllUsers", "value": True},
        {"name": "transformerEmrConnection.s3StagingUri", "value": None},
        {"name": "databricksConfig.initScriptsContents", "value": []},
        {"name": "transformerEmrConnection.secondaryInstanceTypeCustom", "value": None},
        {"name": "emrServerlessConnection.emrApplicationName", "value": None},
        {"name": "transformerEmrConnection.clusterName", "value": None},
        {"name": "clusterConfig.sparkPrimaryUrl", "value": "local[*]"},
        {"name": "emrServerlessConnection.awsConfig.roleSessionName", "value": None},
        {"name": "transformerEmrConnection.bootstrapActionSource", "value": "IN_S3"},
        {"name": "snowflakeCommonConfig.snowflakeConnection.oauthClientId", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.privateKeyPassword", "value": None},
        {"name": "googleCloudConfig.networkType", "value": None},
        {"name": "transformerEmrConnection.secondarySecurityGroup", "value": None},
        {"name": "transformerEmrConnection.awsConfig.externalID", "value": None},
        {"name": "googleCloudConfig.primaryType", "value": None},
        {"name": "clusterConfig.yarnKerberosKeytabSource", "value": "PROPERTIES_FILE"},
        {"name": "triggerInterval", "value": 2000},
        {"name": "transformerEmrConnection.primarySecurityGroup", "value": None},
        {"name": "emrServerlessConnection.subnetIds", "value": []},
        {"name": "emrServerlessConnection.maximumCpu", "value": 400},
        {"name": "transformerEmrConnection.serviceRole", "value": "EMR_DefaultRole"},
        {
            "name": "databricksConfig.clusterConfig",
            "value": '{\n    "num_workers": 8,\n    "spark_version": "14.3.x-scala2.12",'
            '\n    "node_type_id": "i3.xlarge"\n}',
        },
        {"name": "googleCloudCredentialsConfig.projectId", "value": None},
        {"name": "livyConfig.baseUrlBDC", "value": "https://localhost:30443/gateway/default/livy/v1/"},
        {"name": "transformerEmrConnection.customRegion", "value": None},
        {"name": "transformerEmrConnection.secondaryInstanceType", "value": None},
        {"name": "cdeConfig.resource", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.privateKeyPemContent", "value": None},
        {"name": "cdeConfig.apiUrl", "value": None},
        {"name": "connectionString", "value": None},
        {"name": "databricksConfig.provisionNewCluster", "value": True},
        {
            "name": "statsAggregatorStage",
            "value": "streamsets-datacollector-basic-lib::"
            "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1",
        },
        {"name": "udfConfigBeans", "value": []},
        {"name": "databricksConfig.password", "value": None},
        {"name": "transformerEmrConnection.jobFlowRole", "value": "EMR_EC2_DefaultRole"},
        {"name": "snowflakeCommonConfig.overrideSchema", "value": None},
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.nonRegionalEndpoint", "value": None},
        {"name": "clusterConfig.stagingDir", "value": "/streamsets"},
        {"name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.regionalEndpoint", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.user", "value": None},
        {"name": "emrServerlessConnection.region", "value": "US_WEST_2"},
        {"name": "cdeConfig.workloadUser", "value": None},
        {"name": "emrServerlessConnection.createNewApplication", "value": False},
        {"name": "databricksConfig.clusterId", "value": None},
        {"name": "deliveryGuarantee", "value": "AT_LEAST_ONCE"},
        {"name": "maxRunners", "value": 0},
        {"name": "databricksConfig.credentialType", "value": "TOKEN"},
        {"name": "emrServerlessConnection.awsConfig.credentialMode", "value": "WITH_CREDENTIALS"},
        {"name": "transformerEmrConnection.emrClusterOption", "value": "EXISTING_CLUSTER"},
        {"name": "transformerEmrConnection.retryPolicyConfig.maxErrorRetry", "value": 6},
        {"name": "transformerEmrConnection.terminateProvisionedProduct", "value": True},
        {"name": "rateLimit", "value": 0},
        {"name": "transformerEmrConnection.retryPolicyConfig.maxBackoff", "value": 20000},
        {"name": "transformerEmrConnection.instanceCount", "value": 2},
        {"name": "emrServerlessConnection.s3StagingUri", "value": None},
        {
            "name": "badRecordsHandling",
            "value": "streamsets-datacollector-basic-lib::"
            "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1",
        },
        {"name": "transformerEmrConnection.awsConfig.awsSecretAccessKey", "value": None},
        {"name": "clusterConfig.yarnKerberosKeytab", "value": None},
        {"name": "databricksConfig.username", "value": None},
        {"name": "googleCloudConfig.tags", "value": []},
        {"name": "emrServerlessConnection.customTags", "value": []},
        {"name": "transformerEmrConnection.awsConfig.isAssumeRole", "value": False},
        {"name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.region", "value": "US_WEST_2"},
        {"name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.nonRegionalEndpoint", "value": None},
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.customRegion", "value": None},
        {"name": "googleCloudConfig.create", "value": False},
        {"name": "emrServerlessConnection.awsConfig.externalID", "value": None},
        {"name": "transformerEMRConfig.encryption", "value": "NONE"},
        {"name": "emrServerlessConnection.loggingEnabled", "value": True},
        {"name": "googleCloudConfig.version", "value": "1.4-ubuntu18"},
        {"name": "snowflakeCommonConfig.snowflakeConnection.snowflakeRegion", "value": "AWS_US_WEST_2"},
        {"name": "snowflakeCommonConfig.snowflakeConnection.organization", "value": None},
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.signingRegion", "value": "US_WEST_2"},
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.regionDefinitionType", "value": "NOT_SPECIFIED"},
        {"name": "constants", "value": []},
        {"name": "snowflakeCommonConfig.snowflakeConnection.customSnowflakeRole", "value": None},
        {"name": "emrServerlessConnection.emrVersion", "value": "emr-6.8.0"},
        {"name": "clusterConfig.dataCacheLevel", "value": "MEMORY_AND_DISK"},
        {"name": "emrServerlessConnection.applicationId", "value": None},
        {"name": "databricksConfig.terminateCluster", "value": False},
        {"name": "snowflakeCommonConfig.snowflakeConnection.warehouse", "value": None},
        {"name": "transformerEmrConnection.bootstrapActions", "value": []},
        {"name": "transformerEmrConnection.primaryInstanceTypeCustom", "value": None},
        {"name": "googleCloudConfig.clusterLabels", "value": []},
        {"name": "snowflakeCommonConfig.snowflakeConnection.privateKeyPemPath", "value": None},
        {"name": "emrServerlessConnection.awsConfig.isAssumeRole", "value": False},
        {"name": "transformerEmrConnection.sshEc2KeyId", "value": None},
        {
            "name": "preprocessScript",
            "value": "/*\nThe following script define a method"
            "\nthat increments an integer by 1 "
            "\nand registers it as a UDF with "
            "\nthe SparkSession, which can be accessed"
            '\nusing the variable named "spark":'
            "\ndef inc(i: Integer): Integer = {\n  i + 1\n}"
            '\nspark.udf.register ("inc", inc _)\n\n*/',
        },
        {"name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.customRegion", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.useCustomUrl", "value": False},
        {"name": "databricksConfig.baseUrl", "value": None},
        {
            "name": "startEventStage",
            "value": "streamsets-datacollector-basic-lib::"
            "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1",
        },
        {"name": "transformerEmrConnection.parameters", "value": []},
        {"name": "webhookConfigs", "value": []},
        {"name": "snowflakeCommonConfig.snowflakeConnection.customSnowflakeRegion", "value": None},
        {"name": "transformerEmrConnection.awsConfig.profileName", "value": None},
        {"name": "transformerEMRConfig.kmsKeyId", "value": None},
        {"name": "transformerEmrConnection.emrVersion", "value": None},
        {"name": "clusterConfig.callbackUrl", "value": None},
        {"name": "transformerEmrConnection.awsConfig.roleSessionName", "value": None},
        {"name": "clusterConfig.sparkAppName", "value": "${pipeline:title()}"},
        {
            "name": "transformerEmrConnection.awsConfig.awsSTSRegionConfig.regionDefinitionType",
            "value": "NOT_SPECIFIED",
        },
        {"name": "emrServerlessConnection.retryPolicyConfig.maxErrorRetry", "value": 6},
        {"name": "transformerEmrConnection.retryPolicyConfig.baseDelay", "value": 100},
        {"name": "emrServerlessConnection.awsConfig.awsSTSRegionConfig.region", "value": "US_WEST_2"},
        {"name": "clusterConfig.clusterType", "value": "LOCAL"},
        {"name": "clusterConfig.yarnKerberosPrincipal", "value": "name@DOMAIN"},
        {"name": "snowflakeCommonConfig.snowflakeConnection.oauthToken", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.useSnowflakeOrganizationURL", "value": True},
        {"name": "emrServerlessConnection.awsConfig.awsAccessKeyId", "value": None},
        {"name": "googleCloudConfig.terminate", "value": None},
        {"name": "cdeConfig.authTokenUrl", "value": None},
        {"name": "sparkConfigs", "value": []},
        {"name": "emrServerlessConnection.maximumDisk", "value": 20000},
        {"name": "transformerEmrConnection.versionName", "value": None},
        {"name": "emrServerlessConnection.runtimeRoleArn", "value": "arn:aws:iam::<account-id>:role/role-name"},
        {"name": "snowflakeCommonConfig.snowflakeConnection.connectionProperties", "value": None},
        {"name": "emrServerlessConnection.customRegion", "value": None},
        {"name": "emrServerlessConnection.retryPolicyConfig.maxBackoff", "value": 20000},
        {"name": "emrServerlessConnection.awsConfig.setSessionTags", "value": True},
        {"name": "emrServerlessConnection.awsConfig.sessionDuration", "value": 3600},
        {"name": "googleCloudConfig.clusterPrefix", "value": None},
        {"name": "snowflakeCommonConfig.overrideDb", "value": None},
        {"name": "emrServerlessConnection.awsConfig.awsSecretAccessKey", "value": None},
        {"name": "snowflakeCommonConfig.snowflakeConnection.oauthRedirectUri", "value": None},
        {"name": "transformerEmrConnection.awsConfig.roleARN", "value": "arn:aws:iam::<account-id>:role/role-name"},
        {"name": "emrServerlessConnection.securityGroupIds", "value": []},
    ],
    "uiInfo": {},
    "fragments": [],
    "stages": [],
    "errorStage": {
        "instanceName": "errorStageStageInstance",
        "library": "streamsets-datacollector-basic-lib",
        "stageName": "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
        "stageVersion": "1",
        "configuration": [],
        "uiInfo": {"stageType": "TARGET", "label": "Error -Discard"},
        "inputLanes": [],
        "outputLanes": [],
        "eventLanes": [],
        "services": [],
    },
    "info": {
        "pipelineId": "use a flow's executor id here",
        "title": "use a flow's executor id here",
        "description": "description",  # ignore since it's only in the datacollector
        "created": 1758061153670,
        "lastModified": 1758061153670,
        "creator": "use the current user's iam_id here",
        "lastModifier": "use the current user's iam_id here",
        "lastRev": "0",
        "uuid": "create a new uuid here",
        "valid": False,
        "metadata": None,
        "name": "use a flow's executor id here",
        "sdcVersion": "7.0.0-SNAPSHOT",
        "sdcId": "dc74ef9f-0da1-4ee1-ac7b-177e95601edc",
    },
    "metadata": None,
    "statsAggregatorStage": {
        "instanceName": "statsAggregatorStageInstance",
        "library": "streamsets-datacollector-basic-lib",
        "stageName": "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
        "stageVersion": "1",
        "configuration": [],
        "uiInfo": {
            "stageType": "TARGET",
            "label": "Stats Aggregator -Write Directly to Control Hub - "
            "statistics are not aggregated across Data Collectors",
        },
        "inputLanes": [],
        "outputLanes": [],
        "eventLanes": [],
        "services": [],
    },
    "startEventStages": [],
    "stopEventStages": [],
    "testOriginStage": {
        "instanceName": "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage",
        "library": "streamsets-datacollector-dev-lib",
        "stageName": "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
        "stageVersion": "4",
        "configuration": [
            {"name": "eventData", "value": None},
            {"name": "numberOfThreads", "value": 1},
            {"name": "rawData", "value": '{\n  "f1": "abc",\n  "f2": "xyz",\n  "f3": "lmn"\n}'},
            {"name": "stopAfterFirstBatch", "value": False},
            {"name": "stageOnRecordError", "value": "TO_ERROR"},
        ],
        "uiInfo": {"stageType": "SOURCE", "label": "Test Origin - Dev Raw Data Source"},
        "inputLanes": [],
        "outputLanes": ["com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1"],
        "eventLanes": [],
        "services": [
            {
                "service": "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
                "serviceVersion": 4,
                "configuration": [
                    {"name": "displayFormats", "value": "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML"},
                    {"name": "dataFormat", "value": "JSON"},
                    {"name": "dataFormatConfig.compression", "value": "NONE"},
                    {"name": "dataFormatConfig.compressionLibrary", "value": "AUTODETECT"},
                    {"name": "dataFormatConfig.filePatternInArchive", "value": "*"},
                    {"name": "dataFormatConfig.charset", "value": "UTF-8"},
                    {"name": "dataFormatConfig.removeCtrlChars", "value": False},
                    {"name": "dataFormatConfig.textMaxLineLen", "value": 1024},
                    {"name": "dataFormatConfig.useCustomDelimiter", "value": False},
                    {"name": "dataFormatConfig.customDelimiter", "value": "\\r\\n"},
                    {"name": "dataFormatConfig.includeCustomDelimiterInTheText", "value": False},
                    {"name": "dataFormatConfig.jsonContent", "value": "MULTIPLE_OBJECTS"},
                    {"name": "dataFormatConfig.jsonMaxObjectLen", "value": 4096},
                    {"name": "dataFormatConfig.csvHeader", "value": "NO_HEADER"},
                    {"name": "dataFormatConfig.csvFileFormat", "value": "CSV"},
                    {"name": "dataFormatConfig.csvCustomDelimiter", "value": "|"},
                    {"name": "dataFormatConfig.multiCharacterFieldDelimiter", "value": "||"},
                    {"name": "dataFormatConfig.multiCharacterLineDelimiter", "value": "${str:unescapeJava('\\\\n')}"},
                    {"name": "dataFormatConfig.csvCustomEscape", "value": "\\"},
                    {"name": "dataFormatConfig.csvCustomQuote", "value": '"'},
                    {"name": "dataFormatConfig.csvUnivocityFieldSeparator", "value": ","},
                    {"name": "dataFormatConfig.csvUnivocityEscape", "value": "\\"},
                    {"name": "dataFormatConfig.csvUnivocityQuote", "value": '"'},
                    {"name": "dataFormatConfig.csvUnivocityLineSeparator", "value": "${str:unescapeJava('\\\\n')}"},
                    {"name": "dataFormatConfig.csvUnivocityAllowComments", "value": False},
                    {"name": "dataFormatConfig.csvUnivocityCommentMarker", "value": "#"},
                    {"name": "dataFormatConfig.csvEnableComments", "value": False},
                    {"name": "dataFormatConfig.csvCommentMarker", "value": "#"},
                    {"name": "dataFormatConfig.csvSkipStartLines", "value": 0},
                    {"name": "dataFormatConfig.csvParser", "value": "LEGACY_PARSER"},
                    {"name": "dataFormatConfig.csvAllowExtraColumns", "value": False},
                    {"name": "dataFormatConfig.csvExtraColumnPrefix", "value": "_extra_"},
                    {"name": "dataFormatConfig.csvMaxObjectLen", "value": 1024},
                    {"name": "dataFormatConfig.csvUnivocityMaxColumns", "value": 1000},
                    {"name": "dataFormatConfig.csvUnivocityMaxCharsPerColumn", "value": 1000},
                    {"name": "dataFormatConfig.csvUnivocitySkipEmptyLines", "value": False},
                    {"name": "dataFormatConfig.csvIgnoreEmptyLines", "value": False},
                    {"name": "dataFormatConfig.csvRecordType", "value": "LIST_MAP"},
                    {"name": "dataFormatConfig.parseNull", "value": False},
                    {"name": "dataFormatConfig.nullConstant", "value": "\\\\N"},
                    {"name": "dataFormatConfig.xmlRecordElement", "value": None},
                    {"name": "dataFormatConfig.preserveRootElement", "value": False},
                    {"name": "dataFormatConfig.includeFieldXpathAttributes", "value": False},
                    {"name": "dataFormatConfig.xPathNamespaceContext", "value": []},
                    {"name": "dataFormatConfig.outputFieldAttributes", "value": False},
                    {"name": "dataFormatConfig.xmlMaxObjectLen", "value": 4096},
                    {"name": "dataFormatConfig.logMode", "value": "COMMON_LOG_FORMAT"},
                    {"name": "dataFormatConfig.logMaxObjectLen", "value": 1024},
                    {"name": "dataFormatConfig.retainOriginalLine", "value": False},
                    {"name": "dataFormatConfig.customLogFormat", "value": '%h %l %u %t "%r" %>s %b'},
                    {
                        "name": "dataFormatConfig.regex",
                        "value": "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] "
                        '"(\\S+) (\\S+) (\\S+)" (\\d{3}) (\\d+)',
                    },
                    {"name": "dataFormatConfig.fieldPathsToGroupName", "value": [{"fieldPath": "/", "group": 1}]},
                    {"name": "dataFormatConfig.grokPatternDefinition", "value": None},
                    {"name": "dataFormatConfig.grokPattern", "value": "%{COMMONAPACHELOG}"},
                    {"name": "dataFormatConfig.onParseError", "value": "ERROR"},
                    {"name": "dataFormatConfig.maxStackTraceLines", "value": 50},
                    {"name": "dataFormatConfig.enableLog4jCustomLogFormat", "value": False},
                    {"name": "dataFormatConfig.log4jCustomLogFormat", "value": "%r [%t] %-5p %c %x - %m%n"},
                    {"name": "dataFormatConfig.avroSchemaSource", "value": None},
                    {"name": "dataFormatConfig.avroSchema", "value": None},
                    {"name": "dataFormatConfig.schemaRegistryUrls", "value": []},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.securityOption", "value": "PLAINTEXT"},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.truststoreType", "value": "JKS"},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.truststoreFile", "value": None},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.truststorePassword", "value": None},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.keystoreType", "value": "JKS"},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.keystoreFile", "value": None},
                    {"name": "dataFormatConfig.schemaRegistrySecurityConfig.keystorePassword", "value": None},
                    {"name": "dataFormatConfig.basicAuth", "value": None},
                    {"name": "dataFormatConfig.schemaLookupMode", "value": "SUBJECT"},
                    {"name": "dataFormatConfig.subject", "value": None},
                    {"name": "dataFormatConfig.schemaId", "value": None},
                    {"name": "dataFormatConfig.avroSkipUnionIndex", "value": False},
                    {"name": "dataFormatConfig.protoDescriptorFile", "value": None},
                    {"name": "dataFormatConfig.messageType", "value": None},
                    {"name": "dataFormatConfig.isDelimited", "value": False},
                    {"name": "dataFormatConfig.binaryMaxObjectLen", "value": 1024},
                    {"name": "dataFormatConfig.datagramMode", "value": "SYSLOG"},
                    {"name": "dataFormatConfig.typesDbPath", "value": None},
                    {"name": "dataFormatConfig.convertTime", "value": False},
                    {"name": "dataFormatConfig.excludeInterval", "value": False},
                    {"name": "dataFormatConfig.authFilePath", "value": None},
                    {"name": "dataFormatConfig.netflowOutputValuesMode", "value": "RAW_AND_INTERPRETED"},
                    {"name": "dataFormatConfig.maxTemplateCacheSize", "value": -1},
                    {"name": "dataFormatConfig.templateCacheTimeoutMs", "value": -1},
                    {"name": "dataFormatConfig.netflowOutputValuesModeDatagram", "value": "RAW_AND_INTERPRETED"},
                    {"name": "dataFormatConfig.maxTemplateCacheSizeDatagram", "value": -1},
                    {"name": "dataFormatConfig.templateCacheTimeoutMsDatagram", "value": -1},
                    {"name": "dataFormatConfig.wholeFileMaxObjectLen", "value": 8192},
                    {"name": "dataFormatConfig.rateLimit", "value": "-1"},
                    {"name": "dataFormatConfig.verifyChecksum", "value": False},
                    {"name": "dataFormatConfig.excelHeader", "value": None},
                    {"name": "dataFormatConfig.excelSkipCellsWithNoHeader", "value": False},
                    {"name": "dataFormatConfig.excelIncludeCellsWithEmptyValue", "value": False},
                    {"name": "dataFormatConfig.excelReadAllSheets", "value": False},
                    {"name": "dataFormatConfig.maximumFileSize", "value": 100},
                    {"name": "dataFormatConfig.excelSheetNames", "value": []},
                ],
            }
        ],
    },
    "valid": False,
    "issues": {
        "pipelineIssues": [
            {
                "instanceName": None,
                "serviceName": None,
                "level": "PIPELINE",
                "configGroup": None,
                "configName": None,
                "message": "VALIDATION_0001 - The pipeline is empty",
                "errorCode": "VALIDATION_0001",
                "humanReadableMessage": "The pipeline is empty",
                "technicalMessage": "The pipeline is empty",
                "remediationSteps": "",
                "count": 1,
                "additionalInfo": None,
            }
        ],
        "stageIssues": {},
        "issueCount": 1,
    },
    "previewable": False,
}


def create_pipeline_definition_for_new_flow(flow_executor_id: str, iam_id: str) -> dict:
    """Create an empty pipeline definition for a new flow, similar to what a data collector would.

    Args:
        flow_executor_id: A flow's id on a data collector
        iam_id: IAM ID of the user
    Returns:
        A `dict` which contains the pipeline definition for a new flow.
    """
    pipeline_uuid = str(uuid4())
    new_definition = deepcopy(EMPTY_PIPELINE_DEFINITION)

    # update the flow executor id
    new_definition["pipelineId"] = flow_executor_id
    new_definition["title"] = flow_executor_id
    new_definition["info"]["pipelineId"] = flow_executor_id
    new_definition["info"]["title"] = flow_executor_id
    new_definition["info"]["name"] = flow_executor_id

    # update the uuid
    new_definition["uuid"] = pipeline_uuid
    new_definition["info"]["uuid"] = pipeline_uuid

    # update iam id
    new_definition["info"]["creator"] = iam_id
    new_definition["info"]["lastModifier"] = iam_id

    return new_definition
