Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

# Imports operators dynamically while keeping the package API clean, 

# abstracting the underlying modules 

from airflow.utils import import_module_attrs as _import_module_attrs 

 

# These need to be integrated first as other operators depend on them 

_import_module_attrs(globals(), { 

    'check_operator': [ 

        'CheckOperator', 

        'ValueCheckOperator', 

        'IntervalCheckOperator', 

    ], 

}) 

 

_operators = { 

    'bash_operator': ['BashOperator'], 

    'python_operator': [ 

        'PythonOperator', 

        'BranchPythonOperator', 

        'ShortCircuitOperator', 

    ], 

    'hive_operator': ['HiveOperator'], 

    'presto_check_operator': [ 

        'PrestoCheckOperator', 

        'PrestoValueCheckOperator', 

        'PrestoIntervalCheckOperator', 

    ], 

    'dummy_operator': ['DummyOperator'], 

    'email_operator': ['EmailOperator'], 

    'hive_to_samba_operator': ['Hive2SambaOperator'], 

    'mysql_operator': ['MySqlOperator'], 

    'sqlite_operator': ['SqliteOperator'], 

    'mysql_to_hive': ['MySqlToHiveTransfer'], 

    'postgres_operator': ['PostgresOperator'], 

    'sensors': [ 

        'BaseSensorOperator', 

        'ExternalTaskSensor', 

        'HdfsSensor', 

        'HivePartitionSensor', 

        'HttpSensor', 

        'MetastorePartitionSensor', 

        'S3KeySensor', 

        'S3PrefixSensor', 

        'SqlSensor', 

        'TimeDeltaSensor', 

        'TimeSensor', 

        'WebHdfsSensor', 

    ], 

    'subdag_operator': ['SubDagOperator'], 

    'hive_stats_operator': ['HiveStatsCollectionOperator'], 

    's3_to_hive_operator': ['S3ToHiveTransfer'], 

    'hive_to_mysql': ['HiveToMySqlTransfer'], 

    's3_file_transform_operator': ['S3FileTransformOperator'], 

    'http_operator': ['SimpleHttpOperator'], 

    'hive_to_druid': ['HiveToDruidTransfer'], 

    'jdbc_operator': ['JdbcOperator'], 

    'mssql_operator': ['MsSqlOperator'], 

    'mssql_to_hive': ['MsSqlToHiveTransfer'], 

    'slack_operator': ['SlackAPIOperator', 'SlackAPIPostOperator'], 

    'generic_transfer': ['GenericTransfer'], 

} 

 

_import_module_attrs(globals(), _operators) 

from airflow.models import BaseOperator 

 

 

def integrate_plugins(): 

    """Integrate plugins to the context""" 

    from airflow.plugins_manager import operators as _operators 

    for _operator in _operators: 

        globals()[_operator.__name__] = _operator