Skip to content

persistent checkpoint storage #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 10, 2021
Merged

persistent checkpoint storage #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a0b22f
re-create checkpoints for incremental backup
ccrssaa May 2, 2021
24efc95
persistent checkpoint storage
ccrssaa May 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 82 additions & 1 deletion libvirtnbdbackup/libvirthelper/libvirthelper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import random
import logging
import libvirt
import glob
import os
from xml.etree import ElementTree

# this is required so libvirt.py does not report errors to stderr
Expand Down Expand Up @@ -238,10 +240,21 @@ def checkpointExists(self, domObj, checkpointName):
"""
return domObj.checkpointLookupByName(checkpointName)

def removeAllCheckpoints(self, domObj, checkpointList):
def removeAllCheckpoints(self, domObj, checkpointList, args):
""" Remove all existing checkpoints for a virtual machine,
used during FULL backup to reset checkpoint chain
"""

# clean persistent storage in args.checkpointdir
logging.debug('Cleaning up persistent storage {:s}' . format(args.checkpointdir))
try:
for checkpointFile in glob.glob('{:s}/*.xml' . format(args.checkpointdir)):
logging.debug('Remove checkpoint file {:s}' . format(checkpointFile))
os.remove(checkpointFile)
except Exception as e:
logging.error('Unable to clean persistent storage {:s}: {}' . format(args.checkpointdir, e))
sys.exit(1)

if checkpointList is None:
cpts = domObj.listAllCheckpoints()
if cpts:
Expand All @@ -260,3 +273,71 @@ def stopBackup(self, domObj, diskTarget):
""" Cancel the backup task using block job abort
"""
return domObj.blockJobAbort(diskTarget)

def redefineCheckpoints(self, domObj, args):
'''redefine checkpoints from persistent storage'''

# get list of all .xml files in checkpointdir
logging.debug('Loading checkpoint list from: {:s}' . format(args.checkpointdir))
try:
l = glob.glob('{:s}/*.xml' . format(args.checkpointdir))
except Exception as e:
logging.error('Unable to get checkpoint list from {:s}: {}' . format(args.checkpointdir, e))
sys.exit(1)

# sort to ensure that parent checkpoint created first
for checkpointFile in sorted(l):

# load checkpoint config and get root element of the tree
# config will be used later for checkpoint creation (if necessary)
logging.debug('Loading checkpoint config from: {:s}' . format(checkpointFile))
try:
with open(checkpointFile, 'r') as f:
checkpointConfig = f.read()
root = ElementTree.fromstring(checkpointConfig)
except Exception as e:
logging.error('Unable to load checkpoint config from {:s}: {}' . format(checkpointFile, e))
sys.exit(1)

# get checkpoint name
logging.debug('Get checkpoint name from XML')
try:
checkpointName = root.find('name').text
except Exception as e:
logging.error('Unable to find checkpoint name: {}' . format(e))
sys.exit(1)

# try to get checkpoint
logging.debug('Checking presense of {:s} checkpoint' . format(checkpointName))
try:
c = domObj.checkpointLookupByName(checkpointName)
# process next file if checkpoint already exists
logging.debug('Checkpoint {:s} found, skipping to the next file' . format(checkpointName))
continue
except libvirt.libvirtError as e:
# ignore VIR_ERR_NO_DOMAIN_CHECKPOINT, report other errors
if e.get_error_code() != libvirt.VIR_ERR_NO_DOMAIN_CHECKPOINT:
logging.error('libvirt error: {}' . format(e))
sys.exit(1)

# finally redefine missing checkpoint
logging.debug('Redefine missing checkpoint {:s}' . format(checkpointName))
try:
domObj.checkpointCreateXML(checkpointConfig, libvirt.VIR_DOMAIN_CHECKPOINT_CREATE_REDEFINE)
except Exception as e:
logging.error('Unable to redefine checkpoint {:s}: {}' . format(checkpointName, e))
sys.exit(1)

def backupCheckpoint(self, domObj, args, checkpointName):
'''save checkpoint config to persistent storage'''

# save checkpoint config to args.checkpointdir
checkpointFile = '{:s}/{:s}.xml' . format(args.checkpointdir, checkpointName)
logging.debug('Saving checkpoint {:s} config to: {:s}' . format(checkpointName, checkpointFile))
try:
with open(checkpointFile, 'w') as f:
c = domObj.checkpointLookupByName(checkpointName)
f.write(c.getXMLDesc())
except Exception as e:
logging.error('Unable to save checkpoint config to file {:s}: {}' . format(checkpointFile, e))
sys.exit(1)
17 changes: 12 additions & 5 deletions virtnbdbackup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ def main():
parser.add_argument(
"-o", "--output", required=True, type=str,
help="Output target directory")
parser.add_argument(
'-C', '--checkpointdir', required=True, type=str,
Copy link
Owner

@abbbi abbbi May 3, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If backing up multiple virtual machines you need one checkpoint directory per virtual machine, bcs the checkpoint name is quite hardcoded now. Wouldnt it make sense to store the checkpoint xml in the same directory as the backup files then? Otherwise, maybe it makes sense to have the virtual machines name included in the checkpoint name, but that would break existing backup chains.

Copy link
Author

@ccrssaa ccrssaa May 3, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If backing up multiple virtual machines you need one checkpoint directory per virtual machine,

Yes

bcs the checkpoint name is quite hardcoded now. Wouldnt it make sense to store the checkpoint xml in the same directory as the backup files then?

Let's consider scenario:

  1. Have full and incremental backups in directory /backup/domain.20210502
  2. Domain stopped and migrated to another host
  3. Want to make full backup to /backup/domain.20210503

We'll need to specify "previous backup directory" to access saved checkpoint configs
It is possible but will add unnecessary complexity to backup scripts using virtnbdbackup IMHO

Otherwise, maybe it makes sense to have the virtual machines name included in the checkpoint name, but that would break existing backup chains.

Oops forgot to mention I've assumed that checkpointdir contains only one domain checkpoints, exactly "one checkpoint directory per virtual machine"

For example:

root@d02l:/backup/virtnbdbackup# virtnbdbackup -d vm-template -l inc -o /backup/vm-template.20210503 -C /var/lib/libvirt/images.drbd0/checkpoints/vm-template
2021-05-03 15:17:54 INFO common - printVersion: Version: 0.19 Arguments: /usr/local/bin/virtnbdbackup -d vm-template -l inc -o /backup/vm-template.20210503 -C /var/lib/libvirt/images.drbd0/checkpoints/vm-template
...
2021-05-03 15:17:55 INFO virtnbdbackup - main: Finished
root@d02l:/backup/virtnbdbackup# ls -l /var/lib/libvirt/images.drbd0/checkpoints/vm-template
total 36
-rw-r--r-- 1 root root 7843 May  3 12:42 virtnbdbackup.0.xml
-rw-r--r-- 1 root root 7899 May  3 12:43 virtnbdbackup.1.xml
-rw-r--r-- 1 root root 7899 May  3 15:17 virtnbdbackup.2.xml
root@d02l:/backup/virtnbdbackup#

with VM name already present in storage path, add one more VM name to checkpoint woud be a tautology

Copy link
Owner

@abbbi abbbi May 3, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Have full and incremental backups in directory /backup/domain.20210502
  2. Domain stopped and migrated to another host
  3. Want to make full backup to /backup/domain.20210503

We'll need to specify "previous backup directory" to access saved checkpoint configs

yes, thats true.. it needs to redefine the checkpoints if not existant so it can remove them correctly before the next full backup execution.. to have libvirt and qcow files correctly in sync.

help='Persistent libvirt checkpoint storage directory')
parser.add_argument(
"-S", "--scratchdir", default="/var/tmp", required=False, type=str,
help="Target directory for temporary scratch file")
Expand Down Expand Up @@ -184,28 +187,30 @@ def main():
logging.warn('%s', e)
sys.exit(1)

checkpointName = lib.checkpointName
checkpointName = '{:s}.0' . format(lib.checkpointName)
parentCheckpoint = False
checkpoints = []
cptFile = '%s/%s.cpt' % (args.output, args.domain)
if os.path.exists(cptFile):
with open(cptFile,'r') as cptFh:
checkpoints = json.loads(cptFh.read())
# always try to redefine checkpoints
virtClient.redefineCheckpoints(domObj, args)

if args.level != "copy":
logging.info('Looking for checkpoints')
if args.level == "full" and checkpoints:
logging.info("Removing all existant checkpoints before full backup")
virtClient.removeAllCheckpoints(domObj, checkpoints)
virtClient.removeAllCheckpoints(domObj, checkpoints, args)
os.remove(cptFile)
checkpoints = []
elif args.level == "full" and len(checkpoints) < 1:
virtClient.removeAllCheckpoints(domObj,None)
virtClient.removeAllCheckpoints(domObj, None, args)
checkpoints = []

if checkpoints and args.level == "inc":
nextCpt = len(checkpoints)+1
checkpointName = "%s.%s" % (checkpointName, nextCpt)
nextCpt = len(checkpoints)
checkpointName = "%s.%s" % (lib.checkpointName, nextCpt)
if args.checkpoint != False:
logging.info("Overriding parent checkpoint: %s", args.checkpoint)
parentCheckpoint = args.checkpoint
Expand Down Expand Up @@ -245,6 +250,8 @@ def main():
checkpoints.append(checkpointName)
with open(cptFile,'w') as cFw:
cFw.write(json.dumps(checkpoints))
if args.printonly is False and args.output != "-":
virtClient.backupCheckpoint(domObj, args, checkpointName)

if args.startonly is True:
logging.info("Started backup job for debugging, exiting.")
Expand Down