Coverage for gramex\services\watcher.py: 93%

# Coverage for gramex\services\watcher.py : 93%

## 75 statements 69 run 6 missing 0 excluded 2 partial

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1'''

2 The file watch service uses `watchdoc <https://pythonhosted.org/watchdog/>`_ to

3 monitor files, and run functions when the file changes.

4'''

5

6 import os

7 import six

8 import atexit

9 from fnmatch import fnmatch

10 from orderedattrdict import AttrDict

11 from watchdog.observers import Observer

12 from watchdog.events import FileSystemEventHandler

13 from gramex.config import app_log

14

15# Terminology:

16# observer - a single instance class that has all scheduler related behavior

17# watch - an instance that watches a single *folder*

18# handler - an instance that handles events - associated with a single *name*

19

20# There's only one observer. Start it at the beginning and schedule stuff later

21 observer = Observer()

22 observer.start()

23 atexit.register(observer.stop)

24

25 handlers = {} # (handler, watch) -> (folder, name)

26 watches = AttrDict() # watches[folder] = ObservedWatch

27

28 if six.PY2: 28 ↛ 29line 28 didn't jump to line 29, because the condition on line 28 was never true

29 PermissionError = RuntimeError

30

31

32 class FileEventHandler(FileSystemEventHandler):

33 '''

34 Each FileEventHandler is associated with a set of events from a config.

35 It maps a set of paths to these events.

36 '''

37 def __init__(self, patterns, **events):

38 super(FileEventHandler, self).__init__()

39 self.patterns = patterns

40 self.__dict__.update(events)

41

42 def dispatch(self, event):

43 path = os.path.abspath(event.src_path)

44 if any(fnmatch(path, pattern) for pattern in self.patterns):

45 super(FileEventHandler, self).dispatch(event)

46

47

48 def watch(name, paths, **events):

49 '''

50 Watch one or more paths, and trigger an event function.

51

52 Example::

53

54 watch('test', ['test.txt'],

55 on_modified: lambda event: logging.info('Modified test.txt'),

56 on_created: lambda event: logging.info('Created test.txt'))

57

58 When ``test.txt`` is modified or created, it logs one of the above messages.

59

60 To replace the same handler with another, use the same ``name``::

61

62 watch('test', ['test.txt'],

63 on_deleted: lambda event: logging.info('Deleted test.txt'))

64

65 Now, when ``test.txt`` is deleted, it logs a message. But when ``test.txt``

66 is created or modified, no message is shown, since the old handler has been

67 replaced.

68

69 To remove this watch, call ``unwatch('test')``.

70

71 :arg string name: Unique name of the watch. To replace an existing watch,

72 re-use the same name.

73 :arg list paths: List of relative or absolute paths to watch. The paths

74 can be strings or ``pathlib.Path`` objects.

75 :arg function on_modified(event): Called when any path is modified.

76 :arg function on_created(event): Called when any path is created.

77 :arg function on_deleted(event): Called when any path is deleted.

78 :arg function on_moved(event): Called when any path is moved.

79 :arg function on_any_event(event): Called on any of the above events.

80 '''

81 # Create a series of schedules and handlers

82 unwatch(name)

83

84 patterns = set() # List of absolute path patterns

85 folders = set() # List of folders matching these paths

86 for path in paths:

87 # paths can be pathlib.Path or str. Convert to str before proceeding

88 path = os.path.abspath(str(path))

89 if os.path.isdir(path):

90 patterns.add(os.path.join(path, '*'))

91 folders.add(path)

92 else:

93 patterns.add(path)

94 folders.add(os.path.dirname(path))

95

96 handler = FileEventHandler(patterns, **events)

97

98 for folder in folders:

99 _folder, watch = get_watch(folder, watches)

100 # If a watch for this folder (or a parent) exists, use that folder's watch instead

101 if watch is not None:

102 observer.add_handler_for_watch(handler, watch)

103 folder = _folder

104 # If it's a new folder, create a new watch for it

105 elif os.path.exists(folder): 105 ↛ 112line 105 didn't jump to line 112, because the condition on line 105 was never false

106 try:

107 watch = watches[folder] = observer.schedule(handler, folder, recursive=True)

108 except PermissionError:

109 app_log.warning('No permission to watch changes on %s', folder)

110 continue

111 else:

112 app_log.warning('watch directory %s does not exist', folder)

113 continue

114 # If EXISTING sub-folders of folder have watches, consolidate into this watch

115 consolidate_watches(folder, watch)

116 # Keep track of all handler-watch associations

117 handlers[handler, watch] = (folder, name)

118 release_unscheduled_watches()

119

120

121 def unwatch(name):

122 '''

123 Removes all handler-watch associations for a watch name

124 '''

125 for (_handler, _watch), (_folder, _name) in list(handlers.items()):

126 if _name == name:

127 del handlers[_handler, _watch]

128 observer.remove_handler_for_watch(_handler, _watch)

129

130

131 def get_watch(folder, watches):

132 '''

133 Check if a folder already has a scheduled watch. If so, return it.

134 Else return None.

135 '''

136 for watched_folder, watch in watches.items():

137 if folder.startswith(watched_folder):

138 return watched_folder, watch

139 return None, None

140

141

142 def consolidate_watches(folder, watch):

143 '''If folder is a parent of watched folders, migrate those handlers to this watch'''

144 for (_handler, _watch), (_folder, _name) in list(handlers.items()):

145 if _folder.startswith(folder) and _folder != folder:

146 del handlers[_handler, _watch]

147 observer.remove_handler_for_watch(_handler, _watch)

148 handlers[_handler, watch] = (folder, _name)

149 observer.add_handler_for_watch(_handler, watch)

150

151

152 def release_unscheduled_watches():

153 watched_folders = {folder for folder, name in handlers.values()}

154 for folder, watch in list(watches.items()):

155 if folder not in watched_folders:

156 observer.unschedule(watch)

157 del watches[folder]