Package CedarBackup3 :: Package extend :: Module postgresql
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.extend.postgresql

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2006,2010,2015 Kenneth J. Pronovici. 
 12  # Copyright (c) 2006 Antoine Beaupre. 
 13  # All rights reserved. 
 14  # 
 15  # This program is free software; you can redistribute it and/or 
 16  # modify it under the terms of the GNU General Public License, 
 17  # Version 2, as published by the Free Software Foundation. 
 18  # 
 19  # This program is distributed in the hope that it will be useful, 
 20  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 21  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 22  # 
 23  # Copies of the GNU General Public License are available from 
 24  # the Free Software Foundation website, http://www.gnu.org/. 
 25  # 
 26  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 27  # 
 28  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 29  #            Antoine Beaupre <anarcat@koumbit.org> 
 30  # Language : Python 3 (>= 3.4) 
 31  # Project  : Official Cedar Backup Extensions 
 32  # Purpose  : Provides an extension to back up PostgreSQL databases. 
 33  # 
 34  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 35  # This file was created with a width of 132 characters, and NO tabs. 
 36   
 37  ######################################################################## 
 38  # Module documentation 
 39  ######################################################################## 
 40   
 41  """ 
 42  Provides an extension to back up PostgreSQL databases. 
 43   
 44  This is a Cedar Backup extension used to back up PostgreSQL databases via the 
 45  Cedar Backup command line.  It requires a new configurations section 
 46  <postgresql> and is intended to be run either immediately before or immediately 
 47  after the standard collect action.  Aside from its own configuration, it 
 48  requires the options and collect configuration sections in the standard Cedar 
 49  Backup configuration file. 
 50   
 51  The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with 
 52  the PostgreSQL product.  Output can be compressed using C{gzip} or C{bzip2}. 
 53  Administrators can configure the extension either to back up all databases or 
 54  to back up only specific databases.  The extension assumes that the current 
 55  user has passwordless access to the database since there is no easy way to pass 
 56  a password to the C{pg_dump} client. This can be accomplished using appropriate 
 57  voodoo in the C{pg_hda.conf} file. 
 58   
 59  Note that this code always produces a full backup.  There is currently no 
 60  facility for making incremental backups. 
 61   
 62  You should always make C{/etc/cback3.conf} unreadble to non-root users once you 
 63  place postgresql configuration into it, since postgresql configuration will 
 64  contain information about available PostgreSQL databases and usernames. 
 65   
 66  Use of this extension I{may} expose usernames in the process listing (via 
 67  C{ps}) when the backup is running if the username is specified in the 
 68  configuration. 
 69   
 70  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 71  @author: Antoine Beaupre <anarcat@koumbit.org> 
 72  """ 
 73   
 74  ######################################################################## 
 75  # Imported modules 
 76  ######################################################################## 
 77   
 78  # System modules 
 79  import os 
 80  import logging 
 81  from gzip import GzipFile 
 82  from bz2 import BZ2File 
 83  from functools import total_ordering 
 84   
 85  # Cedar Backup modules 
 86  from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode 
 87  from CedarBackup3.xmlutil import readFirstChild, readString, readStringList, readBoolean 
 88  from CedarBackup3.config import VALID_COMPRESS_MODES 
 89  from CedarBackup3.util import resolveCommand, executeCommand 
 90  from CedarBackup3.util import ObjectTypeList, changeOwnership 
 91   
 92   
 93  ######################################################################## 
 94  # Module-wide constants and variables 
 95  ######################################################################## 
 96   
 97  logger = logging.getLogger("CedarBackup3.log.extend.postgresql") 
 98  POSTGRESQLDUMP_COMMAND = [ "pg_dump", ] 
 99  POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ] 
100 101 102 ######################################################################## 103 # PostgresqlConfig class definition 104 ######################################################################## 105 106 @total_ordering 107 -class PostgresqlConfig(object):
108 109 """ 110 Class representing PostgreSQL configuration. 111 112 The PostgreSQL configuration information is used for backing up PostgreSQL databases. 113 114 The following restrictions exist on data in this class: 115 116 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}. 117 - The 'all' flag must be 'Y' if no databases are defined. 118 - The 'all' flag must be 'N' if any databases are defined. 119 - Any values in the databases list must be strings. 120 121 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, user, 122 all, databases 123 """ 124
125 - def __init__(self, user=None, compressMode=None, all=None, databases=None): # pylint: disable=W0622
126 """ 127 Constructor for the C{PostgresqlConfig} class. 128 129 @param user: User to execute backup as. 130 @param compressMode: Compress mode for backed-up files. 131 @param all: Indicates whether to back up all databases. 132 @param databases: List of databases to back up. 133 """ 134 self._user = None 135 self._compressMode = None 136 self._all = None 137 self._databases = None 138 self.user = user 139 self.compressMode = compressMode 140 self.all = all 141 self.databases = databases
142
143 - def __repr__(self):
144 """ 145 Official string representation for class instance. 146 """ 147 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
148
149 - def __str__(self):
150 """ 151 Informal string representation for class instance. 152 """ 153 return self.__repr__()
154
155 - def __eq__(self, other):
156 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 157 return self.__cmp__(other) == 0
158
159 - def __lt__(self, other):
160 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 161 return self.__cmp__(other) < 0
162
163 - def __gt__(self, other):
164 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 165 return self.__cmp__(other) > 0
166
167 - def __cmp__(self, other):
168 """ 169 Original Python 2 comparison operator. 170 @param other: Other object to compare to. 171 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 172 """ 173 if other is None: 174 return 1 175 if self.user != other.user: 176 if str(self.user or "") < str(other.user or ""): 177 return -1 178 else: 179 return 1 180 if self.compressMode != other.compressMode: 181 if str(self.compressMode or "") < str(other.compressMode or ""): 182 return -1 183 else: 184 return 1 185 if self.all != other.all: 186 if self.all < other.all: 187 return -1 188 else: 189 return 1 190 if self.databases != other.databases: 191 if self.databases < other.databases: 192 return -1 193 else: 194 return 1 195 return 0
196
197 - def _setUser(self, value):
198 """ 199 Property target used to set the user value. 200 """ 201 if value is not None: 202 if len(value) < 1: 203 raise ValueError("User must be non-empty string.") 204 self._user = value
205
206 - def _getUser(self):
207 """ 208 Property target used to get the user value. 209 """ 210 return self._user
211
212 - def _setCompressMode(self, value):
213 """ 214 Property target used to set the compress mode. 215 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}. 216 @raise ValueError: If the value is not valid. 217 """ 218 if value is not None: 219 if value not in VALID_COMPRESS_MODES: 220 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES) 221 self._compressMode = value
222
223 - def _getCompressMode(self):
224 """ 225 Property target used to get the compress mode. 226 """ 227 return self._compressMode
228
229 - def _setAll(self, value):
230 """ 231 Property target used to set the 'all' flag. 232 No validations, but we normalize the value to C{True} or C{False}. 233 """ 234 if value: 235 self._all = True 236 else: 237 self._all = False
238
239 - def _getAll(self):
240 """ 241 Property target used to get the 'all' flag. 242 """ 243 return self._all
244
245 - def _setDatabases(self, value):
246 """ 247 Property target used to set the databases list. 248 Either the value must be C{None} or each element must be a string. 249 @raise ValueError: If the value is not a string. 250 """ 251 if value is None: 252 self._databases = None 253 else: 254 for database in value: 255 if len(database) < 1: 256 raise ValueError("Each database must be a non-empty string.") 257 try: 258 saved = self._databases 259 self._databases = ObjectTypeList(str, "string") 260 self._databases.extend(value) 261 except Exception as e: 262 self._databases = saved 263 raise e
264
265 - def _getDatabases(self):
266 """ 267 Property target used to get the databases list. 268 """ 269 return self._databases
270 271 user = property(_getUser, _setUser, None, "User to execute backup as.") 272 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.") 273 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.") 274 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.") 275
276 277 ######################################################################## 278 # LocalConfig class definition 279 ######################################################################## 280 281 @total_ordering 282 -class LocalConfig(object):
283 284 """ 285 Class representing this extension's configuration document. 286 287 This is not a general-purpose configuration object like the main Cedar 288 Backup configuration object. Instead, it just knows how to parse and emit 289 PostgreSQL-specific configuration values. Third parties who need to read and 290 write configuration related to this extension should access it through the 291 constructor, C{validate} and C{addConfig} methods. 292 293 @note: Lists within this class are "unordered" for equality comparisons. 294 295 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, 296 postgresql, validate, addConfig 297 """ 298
299 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
300 """ 301 Initializes a configuration object. 302 303 If you initialize the object without passing either C{xmlData} or 304 C{xmlPath} then configuration will be empty and will be invalid until it 305 is filled in properly. 306 307 No reference to the original XML data or original path is saved off by 308 this class. Once the data has been parsed (successfully or not) this 309 original information is discarded. 310 311 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 312 method will be called (with its default arguments) against configuration 313 after successfully parsing any passed-in XML. Keep in mind that even if 314 C{validate} is C{False}, it might not be possible to parse the passed-in 315 XML document if lower-level validations fail. 316 317 @note: It is strongly suggested that the C{validate} option always be set 318 to C{True} (the default) unless there is a specific need to read in 319 invalid configuration from disk. 320 321 @param xmlData: XML data representing configuration. 322 @type xmlData: String data. 323 324 @param xmlPath: Path to an XML file on disk. 325 @type xmlPath: Absolute path to a file on disk. 326 327 @param validate: Validate the document after parsing it. 328 @type validate: Boolean true/false. 329 330 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 331 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 332 @raise ValueError: If the parsed configuration document is not valid. 333 """ 334 self._postgresql = None 335 self.postgresql = None 336 if xmlData is not None and xmlPath is not None: 337 raise ValueError("Use either xmlData or xmlPath, but not both.") 338 if xmlData is not None: 339 self._parseXmlData(xmlData) 340 if validate: 341 self.validate() 342 elif xmlPath is not None: 343 with open(xmlPath) as f: 344 xmlData = f.read() 345 self._parseXmlData(xmlData) 346 if validate: 347 self.validate()
348
349 - def __repr__(self):
350 """ 351 Official string representation for class instance. 352 """ 353 return "LocalConfig(%s)" % (self.postgresql)
354
355 - def __str__(self):
356 """ 357 Informal string representation for class instance. 358 """ 359 return self.__repr__()
360
361 - def __eq__(self, other):
362 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 363 return self.__cmp__(other) == 0
364
365 - def __lt__(self, other):
366 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 367 return self.__cmp__(other) < 0
368
369 - def __gt__(self, other):
370 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 371 return self.__cmp__(other) > 0
372
373 - def __cmp__(self, other):
374 """ 375 Original Python 2 comparison operator. 376 Lists within this class are "unordered" for equality comparisons. 377 @param other: Other object to compare to. 378 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 379 """ 380 if other is None: 381 return 1 382 if self.postgresql != other.postgresql: 383 if self.postgresql < other.postgresql: 384 return -1 385 else: 386 return 1 387 return 0
388
389 - def _setPostgresql(self, value):
390 """ 391 Property target used to set the postgresql configuration value. 392 If not C{None}, the value must be a C{PostgresqlConfig} object. 393 @raise ValueError: If the value is not a C{PostgresqlConfig} 394 """ 395 if value is None: 396 self._postgresql = None 397 else: 398 if not isinstance(value, PostgresqlConfig): 399 raise ValueError("Value must be a C{PostgresqlConfig} object.") 400 self._postgresql = value
401
402 - def _getPostgresql(self):
403 """ 404 Property target used to get the postgresql configuration value. 405 """ 406 return self._postgresql
407 408 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.") 409
410 - def validate(self):
411 """ 412 Validates configuration represented by the object. 413 414 The compress mode must be filled in. Then, if the 'all' flag 415 I{is} set, no databases are allowed, and if the 'all' flag is 416 I{not} set, at least one database is required. 417 418 @raise ValueError: If one of the validations fails. 419 """ 420 if self.postgresql is None: 421 raise ValueError("PostgreSQL section is required.") 422 if self.postgresql.compressMode is None: 423 raise ValueError("Compress mode value is required.") 424 if self.postgresql.all: 425 if self.postgresql.databases is not None and self.postgresql.databases != []: 426 raise ValueError("Databases cannot be specified if 'all' flag is set.") 427 else: 428 if self.postgresql.databases is None or len(self.postgresql.databases) < 1: 429 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
430
431 - def addConfig(self, xmlDom, parentNode):
432 """ 433 Adds a <postgresql> configuration section as the next child of a parent. 434 435 Third parties should use this function to write configuration related to 436 this extension. 437 438 We add the following fields to the document:: 439 440 user //cb_config/postgresql/user 441 compressMode //cb_config/postgresql/compress_mode 442 all //cb_config/postgresql/all 443 444 We also add groups of the following items, one list element per 445 item:: 446 447 database //cb_config/postgresql/database 448 449 @param xmlDom: DOM tree as from C{impl.createDocument()}. 450 @param parentNode: Parent that the section should be appended to. 451 """ 452 if self.postgresql is not None: 453 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql") 454 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user) 455 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode) 456 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all) 457 if self.postgresql.databases is not None: 458 for database in self.postgresql.databases: 459 addStringNode(xmlDom, sectionNode, "database", database)
460
461 - def _parseXmlData(self, xmlData):
462 """ 463 Internal method to parse an XML string into the object. 464 465 This method parses the XML document into a DOM tree (C{xmlDom}) and then 466 calls a static method to parse the postgresql configuration section. 467 468 @param xmlData: XML data to be parsed 469 @type xmlData: String data 470 471 @raise ValueError: If the XML cannot be successfully parsed. 472 """ 473 (xmlDom, parentNode) = createInputDom(xmlData) 474 self._postgresql = LocalConfig._parsePostgresql(parentNode)
475 476 @staticmethod
477 - def _parsePostgresql(parent):
478 """ 479 Parses a postgresql configuration section. 480 481 We read the following fields:: 482 483 user //cb_config/postgresql/user 484 compressMode //cb_config/postgresql/compress_mode 485 all //cb_config/postgresql/all 486 487 We also read groups of the following item, one list element per 488 item:: 489 490 databases //cb_config/postgresql/database 491 492 @param parent: Parent node to search beneath. 493 494 @return: C{PostgresqlConfig} object or C{None} if the section does not exist. 495 @raise ValueError: If some filled-in value is invalid. 496 """ 497 postgresql = None 498 section = readFirstChild(parent, "postgresql") 499 if section is not None: 500 postgresql = PostgresqlConfig() 501 postgresql.user = readString(section, "user") 502 postgresql.compressMode = readString(section, "compress_mode") 503 postgresql.all = readBoolean(section, "all") 504 postgresql.databases = readStringList(section, "database") 505 return postgresql
506
507 508 ######################################################################## 509 # Public functions 510 ######################################################################## 511 512 ########################### 513 # executeAction() function 514 ########################### 515 516 -def executeAction(configPath, options, config):
517 """ 518 Executes the PostgreSQL backup action. 519 520 @param configPath: Path to configuration file on disk. 521 @type configPath: String representing a path on disk. 522 523 @param options: Program command-line options. 524 @type options: Options object. 525 526 @param config: Program configuration. 527 @type config: Config object. 528 529 @raise ValueError: Under many generic error conditions 530 @raise IOError: If a backup could not be written for some reason. 531 """ 532 logger.debug("Executing PostgreSQL extended action.") 533 if config.options is None or config.collect is None: 534 raise ValueError("Cedar Backup configuration is not properly filled in.") 535 local = LocalConfig(xmlPath=configPath) 536 if local.postgresql.all: 537 logger.info("Backing up all databases.") 538 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user, 539 config.options.backupUser, config.options.backupGroup, None) 540 if local.postgresql.databases is not None and local.postgresql.databases != []: 541 logger.debug("Backing up %d individual databases.", len(local.postgresql.databases)) 542 for database in local.postgresql.databases: 543 logger.info("Backing up database [%s].", database) 544 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user, 545 config.options.backupUser, config.options.backupGroup, database) 546 logger.info("Executed the PostgreSQL extended action successfully.")
547
548 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
549 """ 550 Backs up an individual PostgreSQL database, or all databases. 551 552 This internal method wraps the public method and adds some functionality, 553 like figuring out a filename, etc. 554 555 @param targetDir: Directory into which backups should be written. 556 @param compressMode: Compress mode to be used for backed-up files. 557 @param user: User to use for connecting to the database. 558 @param backupUser: User to own resulting file. 559 @param backupGroup: Group to own resulting file. 560 @param database: Name of database, or C{None} for all databases. 561 562 @return: Name of the generated backup file. 563 564 @raise ValueError: If some value is missing or invalid. 565 @raise IOError: If there is a problem executing the PostgreSQL dump. 566 """ 567 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode) 568 with outputFile: 569 backupDatabase(user, outputFile, database) 570 if not os.path.exists(filename): 571 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename) 572 changeOwnership(filename, backupUser, backupGroup)
573
574 #pylint: disable=R0204 575 -def _getOutputFile(targetDir, database, compressMode):
576 """ 577 Opens the output file used for saving the PostgreSQL dump. 578 579 The filename is either C{"postgresqldump.txt"} or 580 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is 581 added if C{compress} is C{True}. 582 583 @param targetDir: Target directory to write file in. 584 @param database: Name of the database (if any) 585 @param compressMode: Compress mode to be used for backed-up files. 586 587 @return: Tuple of (Output file object, filename), file opened in binary mode for use with executeCommand() 588 """ 589 if database is None: 590 filename = os.path.join(targetDir, "postgresqldump.txt") 591 else: 592 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database) 593 if compressMode == "gzip": 594 filename = "%s.gz" % filename 595 outputFile = GzipFile(filename, "wb") 596 elif compressMode == "bzip2": 597 filename = "%s.bz2" % filename 598 outputFile = BZ2File(filename, "wb") 599 else: 600 outputFile = open(filename, "wb") 601 logger.debug("PostgreSQL dump file will be [%s].", filename) 602 return (outputFile, filename)
603
604 605 ############################ 606 # backupDatabase() function 607 ############################ 608 609 -def backupDatabase(user, backupFile, database=None):
610 """ 611 Backs up an individual PostgreSQL database, or all databases. 612 613 This function backs up either a named local PostgreSQL database or all local 614 PostgreSQL databases, using the passed in user for connectivity. 615 This is I{always} a full backup. There is no facility for incremental 616 backups. 617 618 The backup data will be written into the passed-in back file. Normally, 619 this would be an object as returned from C{open()}, but it is possible to 620 use something like a C{GzipFile} to write compressed output. The caller is 621 responsible for closing the passed-in backup file. 622 623 @note: Typically, you would use the C{root} user to back up all databases. 624 625 @param user: User to use for connecting to the database. 626 @type user: String representing PostgreSQL username. 627 628 @param backupFile: File use for writing backup. 629 @type backupFile: Python file object as from C{open()} or C{file()}. 630 631 @param database: Name of the database to be backed up. 632 @type database: String representing database name, or C{None} for all databases. 633 634 @raise ValueError: If some value is missing or invalid. 635 @raise IOError: If there is a problem executing the PostgreSQL dump. 636 """ 637 args = [] 638 if user is not None: 639 args.append('-U') 640 args.append(user) 641 642 if database is None: 643 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND) 644 else: 645 command = resolveCommand(POSTGRESQLDUMP_COMMAND) 646 args.append(database) 647 648 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0] 649 if result != 0: 650 if database is None: 651 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result) 652 else: 653 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
654