|
| 1 | +"""Typer application for backup Columnstore data.""" |
| 2 | +import logging |
| 3 | +import sys |
| 4 | +from datetime import datetime |
| 5 | +from typing_extensions import Annotated |
| 6 | + |
| 7 | +import typer |
| 8 | + |
| 9 | +from cmapi_server.process_dispatchers.base import BaseDispatcher |
| 10 | +from mcs_cluster_tool.constants import MCS_BACKUP_MANAGER_SH |
| 11 | +from mcs_cluster_tool.decorators import handle_output |
| 12 | +from mcs_cluster_tool.helpers import cook_sh_arg |
| 13 | + |
| 14 | + |
| 15 | +logger = logging.getLogger('mcs_cli') |
| 16 | +# pylint: disable=unused-argument, too-many-arguments, too-many-locals |
| 17 | +# pylint: disable=invalid-name, line-too-long |
| 18 | + |
| 19 | + |
| 20 | +@handle_output |
| 21 | +def backup( |
| 22 | + bl: Annotated[ |
| 23 | + str, |
| 24 | + typer.Option( |
| 25 | + '-bl', '--backup-location', |
| 26 | + help=( |
| 27 | + 'What directory to store the backups on this machine or the target machine.\n' |
| 28 | + 'Consider write permissions of the scp user and the user running this script.\n' |
| 29 | + 'Mariadb-backup will use this location as a tmp dir for S3 and remote backups temporarily.\n' |
| 30 | + 'Example: /mnt/backups/' |
| 31 | + ) |
| 32 | + ) |
| 33 | + ] = '/tmp/backups/', |
| 34 | + bd: Annotated[ |
| 35 | + str, |
| 36 | + typer.Option( |
| 37 | + '-bd', '--backup-destination', |
| 38 | + help=( |
| 39 | + 'Are the backups going to be stored on the same machine this ' |
| 40 | + 'script is running on or another server - if Remote you need ' |
| 41 | + 'to setup scp=' |
| 42 | + 'Options: "Local" or "Remote"' |
| 43 | + ) |
| 44 | + ) |
| 45 | + ] = 'Local', |
| 46 | + scp: Annotated[ |
| 47 | + str, |
| 48 | + typer.Option( |
| 49 | + '-scp', |
| 50 | + help=( |
| 51 | + 'Used only if --backup-destination="Remote".\n' |
| 52 | + 'The user/credentials that will be used to scp the backup ' |
| 53 | + 'files\n' |
| 54 | + |
| 55 | + ) |
| 56 | + ) |
| 57 | + ] = '', |
| 58 | + bb: Annotated[ |
| 59 | + str, |
| 60 | + typer.Option( |
| 61 | + '-bb', '--backup-bucket', |
| 62 | + help=( |
| 63 | + 'Only used if --storage=S3\n' |
| 64 | + 'Name of the bucket to store the columnstore backups.\n' |
| 65 | + 'Example: "s3://my-cs-backups"' |
| 66 | + ) |
| 67 | + ) |
| 68 | + ] = '', |
| 69 | + url: Annotated[ |
| 70 | + str, |
| 71 | + typer.Option( |
| 72 | + '-url', '--endpoint-url', |
| 73 | + help=( |
| 74 | + 'Used by on premise S3 vendors.\n' |
| 75 | + 'Example: "http://127.0.0.1:8000"' |
| 76 | + ) |
| 77 | + ) |
| 78 | + ] = '', |
| 79 | + nv_ssl: Annotated[ |
| 80 | + bool, |
| 81 | + typer.Option( |
| 82 | + '-nv-ssl/-v-ssl','--no-verify-ssl/--verify-ssl', |
| 83 | + help='Skips verifying ssl certs, useful for onpremise s3 storage.' |
| 84 | + ) |
| 85 | + ] = False, |
| 86 | + s: Annotated[ |
| 87 | + str, |
| 88 | + typer.Option( |
| 89 | + '-s', '--storage', |
| 90 | + help=( |
| 91 | + 'What storage topogoly is being used by Columnstore - found ' |
| 92 | + 'in /etc/columnstore/storagemanager.cnf.\n' |
| 93 | + 'Options: "LocalStorage" or "S3"' |
| 94 | + ) |
| 95 | + ) |
| 96 | + ] = 'LocalStorage', |
| 97 | + i: Annotated[ |
| 98 | + str, |
| 99 | + typer.Option( |
| 100 | + '-i', '--incremental', |
| 101 | + help=( |
| 102 | + 'Adds columnstore deltas to an existing full backup. ' |
| 103 | + 'Backup folder to apply increment could be a value or ' |
| 104 | + '"auto_most_recent" - the incremental backup applies to ' |
| 105 | + 'last full backup.' |
| 106 | + ), |
| 107 | + show_default=False |
| 108 | + ) |
| 109 | + ] = '', |
| 110 | + ha: Annotated[ |
| 111 | + bool, |
| 112 | + typer.Option( |
| 113 | + '-ha/-no-ha', '--highavilability/--no-highavilability', |
| 114 | + help=( |
| 115 | + 'Hint wether shared storage is attached @ below on all nodes ' |
| 116 | + 'to see all data\n' |
| 117 | + ' HA LocalStorage ( /var/lib/columnstore/dataX/ )\n' |
| 118 | + ' HA S3 ( /var/lib/columnstore/storagemanager/ )' |
| 119 | + ) |
| 120 | + ) |
| 121 | + ] = False, |
| 122 | + f: Annotated[ |
| 123 | + str, |
| 124 | + typer.Option( |
| 125 | + '-f', '--config-file', |
| 126 | + help='Path to backup configuration file to load variables from.', |
| 127 | + show_default=False |
| 128 | + ) |
| 129 | + ] = '', |
| 130 | + sbrm: Annotated[ |
| 131 | + bool, |
| 132 | + typer.Option( |
| 133 | + '-sbrm/-no-sbrm', '--skip-save-brm/--no-skip-save-brm', |
| 134 | + help=( |
| 135 | + 'Skip saving brm prior to running a backup - ' |
| 136 | + 'ideal for dirty backups.' |
| 137 | + ) |
| 138 | + ) |
| 139 | + ] = False, |
| 140 | + spoll: Annotated[ |
| 141 | + bool, |
| 142 | + typer.Option( |
| 143 | + '-spoll/-no-spoll', '--skip-polls/--no-skip-polls', |
| 144 | + help='Skip sql checks confirming no write/cpimports running.' |
| 145 | + ) |
| 146 | + ] = False, |
| 147 | + slock: Annotated[ |
| 148 | + bool, |
| 149 | + typer.Option( |
| 150 | + '-slock/-no-slock', '--skip-locks/--no-skip-locks', |
| 151 | + help='Skip issuing write locks - ideal for dirty backups.' |
| 152 | + ) |
| 153 | + ] = False, |
| 154 | + smdb: Annotated[ |
| 155 | + bool, |
| 156 | + typer.Option( |
| 157 | + '-smdb/-no-smdb', '--skip-mariadb-backup/--no-skip-mariadb-backup', |
| 158 | + help=( |
| 159 | + 'Skip running a mariadb-backup for innodb data - ideal for ' |
| 160 | + 'incremental dirty backups.' |
| 161 | + ) |
| 162 | + ) |
| 163 | + ] = False, |
| 164 | + sb: Annotated[ |
| 165 | + bool, |
| 166 | + typer.Option( |
| 167 | + '-sb/-no-sb', '--skip-bucket-data/--no-skip-bucket-data', |
| 168 | + help='Skip taking a copy of the columnstore data in the bucket.' |
| 169 | + ) |
| 170 | + ] = False, |
| 171 | + pi: Annotated[ |
| 172 | + int, |
| 173 | + typer.Option( |
| 174 | + '-pi', '--poll-interval', |
| 175 | + help=( |
| 176 | + 'Number of seconds between poll checks for active writes & ' |
| 177 | + 'cpimports.' |
| 178 | + ) |
| 179 | + ) |
| 180 | + ] = 5, |
| 181 | + pmw: Annotated[ |
| 182 | + int, |
| 183 | + typer.Option( |
| 184 | + '-pmw', '--poll-max-wait', |
| 185 | + help=( |
| 186 | + 'Max number of minutes for polling checks for writes to wait ' |
| 187 | + 'before exiting as a failed backup attempt.' |
| 188 | + ) |
| 189 | + ) |
| 190 | + ] = 60, |
| 191 | + q: Annotated[ |
| 192 | + bool, |
| 193 | + typer.Option( |
| 194 | + '-q/-no-q', '--quiet/--no-quiet', |
| 195 | + help='Silence verbose copy command outputs.' |
| 196 | + ) |
| 197 | + ] = False, |
| 198 | + c: Annotated[ |
| 199 | + str, |
| 200 | + typer.Option( |
| 201 | + '-c', '--compress', |
| 202 | + help='Compress backup in X format - Options: [ pigz ].', |
| 203 | + show_default=False |
| 204 | + ) |
| 205 | + ] = '', |
| 206 | + P: Annotated[ |
| 207 | + int, |
| 208 | + typer.Option( |
| 209 | + '-P', '--parallel', |
| 210 | + help=( |
| 211 | + 'Determines if columnstore data directories will have ' |
| 212 | + 'multiple rsync running at the same time for different ' |
| 213 | + 'subfolders to parallelize writes. ' |
| 214 | + 'Ignored if "-c/--compress" argument not set.' |
| 215 | + ) |
| 216 | + ) |
| 217 | + ] = 4, |
| 218 | + nb: Annotated[ |
| 219 | + str, |
| 220 | + typer.Option( |
| 221 | + '-nb', '--name-backup', |
| 222 | + help='Define the name of the backup - default: $(date +%m-%d-%Y)' |
| 223 | + ) |
| 224 | + ] = datetime.now().strftime('%m-%d-%Y'), |
| 225 | + m: Annotated[ |
| 226 | + str, |
| 227 | + typer.Option( |
| 228 | + '-m', '--mode', |
| 229 | + help=( |
| 230 | + 'Modes ["direct","indirect"] - direct backups run on the ' |
| 231 | + 'columnstore nodes themselves. indirect run on another ' |
| 232 | + 'machine that has read-only mounts associated with ' |
| 233 | + 'columnstore/mariadb\n' |
| 234 | + ), |
| 235 | + hidden=True |
| 236 | + ) |
| 237 | + ] = 'direct', |
| 238 | + r: Annotated[ |
| 239 | + int, |
| 240 | + typer.Option( |
| 241 | + '-r', '--retention-days', |
| 242 | + help=( |
| 243 | + 'Retain backups created within the last X days, ' |
| 244 | + 'default 0 == keep all backups.' |
| 245 | + ) |
| 246 | + ) |
| 247 | + ] = 0, |
| 248 | +): |
| 249 | + """Backup Columnstore and/or MariDB data.""" |
| 250 | + |
| 251 | + # Local Storage Examples: |
| 252 | + # ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage |
| 253 | + # ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage -P 8 |
| 254 | + # ./$0 backup -bl /tmp/backups/ -bd Local -s LocalStorage --incremental 02-18-2022 |
| 255 | + # ./$0 backup -bl /tmp/backups/ -bd Remote -scp [email protected] -s LocalStorage |
| 256 | + |
| 257 | + # S3 Examples: |
| 258 | + # ./$0 backup -bb s3://my-cs-backups -s S3 |
| 259 | + # ./$0 backup -bb s3://my-cs-backups -c pigz --quiet -sb |
| 260 | + # ./$0 backup -bb gs://my-cs-backups -s S3 --incremental 02-18-2022 |
| 261 | + # ./$0 backup -bb s3://my-onpremise-bucket -s S3 -url http://127.0.0.1:8000 |
| 262 | + |
| 263 | + # Cron Example: |
| 264 | + # */60 */24 * * * root bash /root/$0 -bb s3://my-cs-backups -s S3 >> /root/csBackup.log 2>&1 |
| 265 | + |
| 266 | + arguments = [] |
| 267 | + for arg_name, value in locals().items(): |
| 268 | + sh_arg = cook_sh_arg(arg_name, value) |
| 269 | + if sh_arg is None: |
| 270 | + continue |
| 271 | + arguments.append(sh_arg) |
| 272 | + cmd = f'{MCS_BACKUP_MANAGER_SH} backup {" ".join(arguments)}' |
| 273 | + success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout) |
| 274 | + return {'success': success} |
| 275 | + |
| 276 | + |
| 277 | +@handle_output |
| 278 | +def dbrm_backup( |
| 279 | + m: Annotated[ |
| 280 | + str, |
| 281 | + typer.Option( |
| 282 | + '-m', '--mode', |
| 283 | + help=( |
| 284 | + '"loop" or "once" ; Determines if this script runs in a ' |
| 285 | + 'forever loop sleeping -i minutes or just once.' |
| 286 | + ), |
| 287 | + ) |
| 288 | + ] = 'once', |
| 289 | + i: Annotated[ |
| 290 | + int, |
| 291 | + typer.Option( |
| 292 | + '-i', '--interval', |
| 293 | + help='Number of minutes to sleep when --mode=loop.' |
| 294 | + ) |
| 295 | + ] = 90, |
| 296 | + r: Annotated[ |
| 297 | + int, |
| 298 | + typer.Option( |
| 299 | + '-r', '--retention-days', |
| 300 | + help=( |
| 301 | + 'Retain dbrm backups created within the last X days, ' |
| 302 | + 'the rest are deleted' |
| 303 | + ) |
| 304 | + ) |
| 305 | + ] = 7, |
| 306 | + p: Annotated[ |
| 307 | + str, |
| 308 | + typer.Option( |
| 309 | + '-p', '--path', |
| 310 | + help='Path of where to save the dbrm backups on disk.' |
| 311 | + ) |
| 312 | + ] = '/tmp/dbrm_backups', |
| 313 | + nb: Annotated[ |
| 314 | + str, |
| 315 | + typer.Option( |
| 316 | + '-nb', '--name-backup', |
| 317 | + help='Custom name to prefex dbrm backups with.' |
| 318 | + ) |
| 319 | + ] = 'dbrm_backup', |
| 320 | + q: Annotated[ |
| 321 | + bool, |
| 322 | + typer.Option( |
| 323 | + '-q/-no-q', '--quiet/--no-quiet', |
| 324 | + help='Silence verbose copy command outputs.' |
| 325 | + ) |
| 326 | + ] = False, |
| 327 | + ssm: Annotated[ |
| 328 | + bool, |
| 329 | + typer.Option( |
| 330 | + '-ssm/-no-ssm', '--skip-storage-manager/--no-skip-storage-manager', |
| 331 | + help='Skip backing up storagemanager directory.' |
| 332 | + ) |
| 333 | + ] = False, |
| 334 | +): |
| 335 | + """Columnstore DBRM Backup.""" |
| 336 | + |
| 337 | + # Default: ./$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups |
| 338 | + |
| 339 | + # Examples: |
| 340 | + # ./$0 dbrm_backup --mode loop --interval 90 --retention-days 7 --path /mnt/dbrm_backups |
| 341 | + # ./$0 dbrm_backup --mode once --retention-days 7 --path /mnt/dbrm_backups -nb my-one-off-backup |
| 342 | + |
| 343 | + # Cron Example: |
| 344 | + # */60 */3 * * * root bash /root/$0 dbrm_backup -m once --retention-days 7 --path /tmp/dbrm_backups >> /tmp/dbrm_backups/cs_backup.log 2>&1 |
| 345 | + arguments = [] |
| 346 | + for arg_name, value in locals().items(): |
| 347 | + sh_arg = cook_sh_arg(arg_name, value) |
| 348 | + if sh_arg is None: |
| 349 | + continue |
| 350 | + arguments.append(sh_arg) |
| 351 | + cmd = f'{MCS_BACKUP_MANAGER_SH} dbrm_backup {" ".join(arguments)}' |
| 352 | + success, _ = BaseDispatcher.exec_command(cmd, stdout=sys.stdout) |
| 353 | + return {'success': success} |
0 commit comments