Coverage for src/edwh_restic_plugin/tasks.py: 0%

1 import json

2 import os

3 import subprocess

4 import typing

5

6 import invoke

7 from edwh.tasks import DOCKER_COMPOSE

8 from invoke import Context, task

9 from termcolor import cprint

10

11 from .env import DOTENV, read_dotenv, set_env_value

12 from .helpers import _require_restic

13 from .repositories import Repository, registrations

14 from .restictypes import DockerContainer

15

16

17 def cli_repo(connection_choice: str = None, restichostname: str = None) -> Repository:

18 """

19 Create a repository object and set up the connection to the backend.

20 :param connection_choice: choose where you want to store the repo (local, SFTP, B2, swift)

21 :param restichostname: which hostname to force for restic, or blank for default.

22 :return: repository object

23 """

24 env = read_dotenv(DOTENV)

25 if restichostname:

26 set_env_value(DOTENV, "RESTICHOSTNAME", restichostname)

27

28 options = registrations.to_ordered_dict()

29

30 connection_lowercase = ""

31 if connection_choice is None:

32 # search for the most important backup and use it as default

33 for option in options:

34 if f"{option.upper()}_NAME" in env:

35 connection_lowercase = option.lower()

36 break

37 else:

38 connection_lowercase = connection_choice.lower()

39

40 if not (repoclass := registrations.get(connection_lowercase)):

41 _options = ", ".join(list(options))

42 raise ValueError(f"Invalid connection type {connection_choice}. Please use one of {_options}!")

43

44 print("Use connection: ", connection_lowercase)

45 repo = repoclass()

46 repo.setup()

47 return repo

48

49

50@task

51 def require_restic(c):

52 _require_restic(c)

53

54

55@task(aliases=("setup", "init"))

56 def configure(c, connection_choice=None, restichostname=None):

57 """Setup or update the backup command for your environment.

58 connection_choice: choose where you want to store the repo (local, SFTP, B2, swift)

59 restichostname: which hostname to force for restic, or blank for default.

60 """

61

62 # It has been decided to create a main path called 'backups' for each repository.

63 # This can be changed or removed if desired.

64 # A password is only passed with a few functions.

65 cli_repo(connection_choice, restichostname).configure(c)

66

67

68@task

69 def backup(c, target: str = "", connection_choice: str = None, message: str = None, verbose: bool = True):

70 """Performs a backup operation using restic on a local or remote/cloud file system.

71

72 Args:

73 c (Context)

74 target (str): The target of the backup (e.g. 'files', 'stream'; default is all types).

75 connection_choice (str): The name of the connection to use for the backup.

76 Defaults to None, which means the default connection will be used.

77 message (str): A message to attach to the backup snapshot.

78 Defaults to None, which means no message will be attached.

79 verbose (bool): If True, outputs more information about the backup process. Defaults to False.

80

81 Raises:

82 Exception: If an error occurs during the backup process.

83

84 """

85 # After 'backup', a file path can be specified.In this script, a test file is chosen at './test/testbestand'.

86 # It can be replaced with the desired path over which restic should perform a backup.

87 # The option --verbose provides more information about the backup that is made.It can be removed if desired.

88

89 # By using additions, it is possible to specify what should be included:

90 # --exclude ,Specified one or more times to exclude one or more items.

91 # --iexclude, Same as --exclude but ignores the case of paths.

92 # --exclude-caches, Specified once to exclude folders containing this special file.

93 # --exclude-file, Specified one or more times to exclude items listed in a given file.

94 # --iexclude-file, Same as exclude-file but ignores cases like in --iexclude.

95 # --exclude-if-present 'foo', Specified one or more times to exclude a folder's content if it contains.

96 # a file called 'foo' (optionally having a given header, no wildcards for the file name supported).

97 # --exclude-larger-than 'size', Specified once to excludes files larger than the given size.

98 # Please see 'restic help backup' for more specific information about each exclude option.

99

100 cli_repo(connection_choice).backup(c, verbose, target, message)

101

102

103@task

104 def restore(c, connection_choice: str = None, snapshot: str = "latest", target: str = "", verbose: bool = True):

105 """

106 The restore function restores the latest backed-up files by default and puts them in a restore folder.

107

108 IMPORTANT: please provide -t for the path where the restore should go. Also remember to include -c for the service

109 where the backup is stored.

110

111 :type c: Context

112 :param connection_choice: the service where the files are backed up, e.g., 'local' or 'os' (= openstack).

113 :param snapshot: the ID where the files are backed up, default value is 'latest'.

114 :param target: The target of the backup (e.g. 'files', 'stream'; default is all types).

115 :param verbose: display verbose logs (inv restore -v).

116 :return: None

117 """

118 # For restore, --target is the location where the restore should be placed, --path is the file/path that should be

119 # retrieved from the repository.

120 # 'which_restore' is a user input to enable restoring an earlier backup (default = latest).

121 # Stop the postgres services.

122 c.run(f"{DOCKER_COMPOSE} stop -t 1 pg-0 pg-1 pgpool", warn=True, hide=True)

123

124 # Get the volumes that are being used.

125 docker_inspect: invoke.Result = c.run("docker inspect pg-0 pg-1", hide=True, warn=True)

126 if docker_inspect.ok:

127 # Only if ok, because if pg-0 and pg-1 do not exist, this does not exist either, and nothing needs to be removed

128 inspected: list[DockerContainer] = json.loads(docker_inspect.stdout)

129 volumes_to_remove: list[str] = []

130 for service in inspected:

131 volumes_to_remove.extend(mount["Name"] for mount in service["Mounts"] if mount["Type"] == "volume")

132 # Remove the containers before a volume can be removed.

133 c.run(f"{DOCKER_COMPOSE} rm -f pg-0 pg-1")

134 # Remove the volumes.

135 for volume_name in volumes_to_remove:

136 c.run(f"docker volume rm {volume_name}")

137

138 cli_repo(connection_choice).restore(c, verbose, target, snapshot)

139 # print("`inv up` to restart the services.")

140

141

142@task(iterable=["tag"])

143 def snapshots(c, connection_choice: str = None, tag: list[str] = None, n: int = 1, verbose: bool = False):

144 """

145 With this you can see per repo which repo is made when and where, \

146 the repo-id can be used at inv restore as an option

147

148 :type c: Context

149 :param connection_choice: service

150 :param tag: files, stream ect

151 :param n: amount of snapshot to view, default=1(latest)

152 :param verbose: show which commands are being executed?

153 :return: None

154 """

155 # if tags is None set tag to default tags

156 if tag is None:

157 tag = ["files", "stream"]

158

159 cli_repo(connection_choice).snapshot(c, tags=tag, n=n, verbose=verbose)

160

161

162 def interactive(conn: Repository):

163 subprocess.run(["/bin/bash", "--norc", "--noprofile"], env=os.environ | {"PS1": f"{conn! r} $ "})

164

165

166@task(pre=[require_restic])

167 def run(c, connection_choice: str = None, command: typing.Optional[str] = None):

168 """

169 This function prepares for restic and runs the input command until the user types "exit".

170

171 :type c: Context

172 :param connection_choice: The connection name of the repository.

173 :param command: restic subcommand to use (default: none, prompt interactively)

174 """

175 conn = cli_repo(connection_choice)

176 conn.prepare_for_restic(c)

177

178 if command:

179 if not command.startswith("restic "):

180 command = f"restic {command}"

181 print(c.run(command, hide=True, warn=True, pty=True))

182 else:

183 interactive(conn)

184

185

186@task()

187 def env(c, connection_choice: str = None):

188 """

189

190 :type c: Context

191 :param connection_choice: The connection name of the repository.

192 """

193 from copy import deepcopy

194

195 old = deepcopy(os.environ)

196 cli_repo(connection_choice).prepare_for_restic(c)

197 new = os.environ

198 for k, v in new.items():

199 if k not in old or old[k] != v:

200 print(f"export {k.upper()}={v}")

201

202@task()

203 def forget(c: Context, connection: str = None):

204 # https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy

205 repo = cli_repo(connection)

206

207 print(

208 repo.forget(c)

209 )