Commit af9e3e38 authored by Girish Ramakrishnan's avatar Girish Ramakrishnan

apply backup retention policy

part of #441
parent d992702b
This diff is collapsed.
......@@ -57,6 +57,7 @@ var addons = require('./addons.js'),
eventlog = require('./eventlog.js'),
fs = require('fs'),
locker = require('./locker.js'),
moment = require('moment'),
mkdirp = require('mkdirp'),
once = require('once'),
path = require('path'),
......@@ -140,7 +141,12 @@ function testConfig(backupConfig, callback) {
}
const policy = backupConfig.retentionPolicy;
if (!policy) return callback(new BoxError(BoxError.BAD_FIELD, 'retentionPolicy is required', { field: 'retentionPolicy' }));
if ('keepWithinSecs' in policy && typeof policy.keepWithinSecs !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepWithinSecs must be a number', { field: 'retentionPolicy' }));
if ('keepDaily' in policy && typeof policy.keepDaily !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepDaily must be a number', { field: 'retentionPolicy' }));
if ('keepWeekly' in policy && typeof policy.keepWeekly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepWeekly must be a number', { field: 'retentionPolicy' }));
if ('keepMonthly' in policy && typeof policy.keepMonthly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepMonthly must be a number', { field: 'retentionPolicy' }));
if ('keepYearly' in policy && typeof policy.keepYearly !== 'number') return callback(new BoxError(BoxError.BAD_FIELD, 'keepYearly must be a number', { field: 'retentionPolicy' }));
api(backupConfig.provider).testConfig(backupConfig, callback);
}
......@@ -1225,6 +1231,52 @@ function ensureBackup(auditSource, callback) {
});
}
function applyBackupRetentionPolicy(backups, policy) {
assert(Array.isArray(backups));
assert.strictEqual(typeof policy, 'object');
const now = new Date();
for (const backup of backups) {
if (backup.keepReason) continue; // already kept for some other reason
if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
backup.keepReason = 'preserveSecs';
} else if ((now - backup.creationTime) < (policy.keepWithinSecs * 1000)) {
backup.keepReason = 'withinSecs';
}
}
const KEEP_FORMATS = {
keepDaily: 'Y-M-D',
keepWeekly: 'Y-W',
keepMonthly: 'Y-M',
keepYearly: 'Y'
};
for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
if (!(format in policy)) continue;
const n = policy[format]; // we want to keep "n" backups of format
if (!n) continue; // disabled rule
let lastPeriod = null, keptSoFar = 0;
for (const backup of backups) {
if (backup.keepReason) continue; // already kept for some other reason
const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
if (period === lastPeriod) continue; // already kept for this period
lastPeriod = period;
backup.keepReason = format;
if (++keptSoFar === n) break;
}
}
for (const backup of backups) {
if (backup.keepReason) debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason}`);
}
}
function cleanupBackup(backupConfig, backup, callback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backup, 'object');
......@@ -1260,31 +1312,34 @@ function cleanupBackup(backupConfig, backup, callback) {
}
}
function cleanupAppBackups(backupConfig, referencedAppBackups, callback) {
function cleanupAppBackups(backupConfig, referencedAppBackupIds, callback) {
assert.strictEqual(typeof backupConfig, 'object');
assert(Array.isArray(referencedAppBackups));
assert(Array.isArray(referencedAppBackupIds));
assert.strictEqual(typeof callback, 'function');
const now = new Date();
let removedAppBackups = [];
let removedAppBackupIds = [];
// we clean app backups of any state because the ones to keep are determined by the box cleanup code
backupdb.getByTypePaged(backupdb.BACKUP_TYPE_APP, 1, 1000, function (error, appBackups) {
if (error) return callback(error);
for (const appBackup of appBackups) { // set the reason so that policy filter can skip it
if (referencedAppBackupIds.includes(appBackup.id)) appBackup.keepReason = 'reference';
}
applyBackupRetentionPolicy(appBackups, backupConfig.retentionPolicy);
async.eachSeries(appBackups, function iterator(appBackup, iteratorDone) {
if (referencedAppBackups.indexOf(appBackup.id) !== -1) return iteratorDone();
if ((now - appBackup.creationTime) < (appBackup.preserveSecs * 1000)) return iteratorDone();
if ((now - appBackup.creationTime) < (backupConfig.retentionPolicy.keepWithinSecs * 1000)) return iteratorDone();
if (appBackup.keepReason) return iteratorDone();
debug('cleanupAppBackups: removing %s', appBackup.id);
removedAppBackups.push(appBackup.id);
removedAppBackupIds.push(appBackup.id);
cleanupBackup(backupConfig, appBackup, iteratorDone);
}, function () {
debug('cleanupAppBackups: done');
callback(null, removedAppBackups);
callback(null, removedAppBackupIds);
});
});
}
......@@ -1294,13 +1349,12 @@ function cleanupBoxBackups(backupConfig, auditSource, callback) {
assert.strictEqual(typeof auditSource, 'object');
assert.strictEqual(typeof callback, 'function');
const now = new Date();
let referencedAppBackups = [], removedBoxBackups = [];
let referencedAppBackupIds = [], removedBoxBackupIds = [];
backupdb.getByTypePaged(backupdb.BACKUP_TYPE_BOX, 1, 1000, function (error, boxBackups) {
if (error) return callback(error);
if (boxBackups.length === 0) return callback(null, { removedBoxBackups, referencedAppBackups });
if (boxBackups.length === 0) return callback(null, { removedBoxBackupIds, referencedAppBackupIds });
// search for the first valid backup
var i;
......@@ -1311,28 +1365,28 @@ function cleanupBoxBackups(backupConfig, auditSource, callback) {
// keep the first valid backup
if (i !== boxBackups.length) {
debug('cleanupBoxBackups: preserving box backup %s (%j)', boxBackups[i].id, boxBackups[i].dependsOn);
referencedAppBackups = boxBackups[i].dependsOn;
referencedAppBackupIds = boxBackups[i].dependsOn;
boxBackups.splice(i, 1);
} else {
debug('cleanupBoxBackups: no box backup to preserve');
}
applyBackupRetentionPolicy(boxBackups, backupConfig.retentionPolicy);
async.eachSeries(boxBackups, function iterator(boxBackup, iteratorNext) {
// TODO: errored backups should probably be cleaned up before retention time, but we will
// have to be careful not to remove any backup currently being created
if ((now - boxBackup.creationTime) < (backupConfig.retentionPolicy.keepWithinSecs * 1000)) {
referencedAppBackups = referencedAppBackups.concat(boxBackup.dependsOn);
if (boxBackup.keepReason) {
referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
return iteratorNext();
}
debug('cleanupBoxBackups: removing %s', boxBackup.id);
removedBoxBackups.push(boxBackup.id);
removedBoxBackupIds.push(boxBackup.id);
cleanupBackup(backupConfig, boxBackup, iteratorNext);
}, function () {
debug('cleanupBoxBackups: done');
callback(null, { removedBoxBackups, referencedAppBackups });
callback(null, { removedBoxBackupIds, referencedAppBackupIds });
});
});
}
......@@ -1401,12 +1455,12 @@ function cleanup(auditSource, progressCallback, callback) {
progressCallback({ percent: 10, message: 'Cleaning box backups' });
cleanupBoxBackups(backupConfig, auditSource, function (error, result) {
cleanupBoxBackups(backupConfig, auditSource, function (error, { removedBoxBackupIds, referencedAppBackupIds }) {
if (error) return callback(error);
progressCallback({ percent: 40, message: 'Cleaning app backups' });
cleanupAppBackups(backupConfig, result.referencedAppBackups, function (error, removedAppBackups) {
cleanupAppBackups(backupConfig, referencedAppBackupIds, function (error, removedAppBackupIds) {
if (error) return callback(error);
progressCallback({ percent: 90, message: 'Cleaning snapshots' });
......@@ -1414,7 +1468,7 @@ function cleanup(auditSource, progressCallback, callback) {
cleanupSnapshots(backupConfig, function (error) {
if (error) return callback(error);
callback(null, { removedBoxBackups: result.removedBoxBackups, removedAppBackups: removedAppBackups });
callback(null, { removedBoxBackupIds, removedAppBackupIds });
});
});
});
......
......@@ -107,7 +107,7 @@ function setBackupConfig(req, res, next) {
if ('acceptSelfSignedCerts' in req.body && typeof req.body.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
if (!req.body.retentionPolicy || typeof req.body.retentionPolicy !== 'object') return next(new HttpError(400, 'retentionPolicy is required'));
if (typeof req.body.retentionPolicy.keepWithin !== 'number') return next(400, 'keepWithin is required');
if (typeof req.body.retentionPolicy.keepWithinSecs !== 'number') return next(400, 'keepWithinSecs is required');
// testing the backup using put/del takes a bit of time at times
req.clearTimeout();
......
......@@ -62,7 +62,7 @@ function setup(done) {
},
function createSettings(callback) {
settings.setBackupConfig({ provider: 'filesystem', backupFolder: '/tmp', format: 'tgz' }, callback);
settings.setBackupConfig({ provider: 'filesystem', backupFolder: '/tmp', format: 'tgz', retentionPolicy: {} }, callback);
}
], done);
}
......
......@@ -145,7 +145,7 @@ let gDefaults = (function () {
backupFolder: '/var/backups',
format: 'tgz',
encryption: null,
retentionPolicy: { keepWithin: 2 * 24 * 60 * 60 }, // 2 days
retentionPolicy: { keepWithinSecs: 2 * 24 * 60 * 60 }, // 2 days
intervalSecs: 24 * 60 * 60 // ~1 day
};
result[exports.PLATFORM_CONFIG_KEY] = {};
......
......@@ -80,7 +80,7 @@ describe('backups', function () {
provider: 'filesystem',
password: 'supersecret',
backupFolder: BACKUP_DIR,
retentionPolicy: { keepWithin: 1 },
retentionPolicy: { keepWithinSecs: 1 },
format: 'tgz'
})
], done);
......@@ -277,25 +277,26 @@ describe('backups', function () {
describe('filesystem', function () {
var backupInfo1;
var gBackupConfig = {
var backupConfig = {
provider: 'filesystem',
backupFolder: path.join(os.tmpdir(), 'backups-test-filesystem'),
format: 'tgz'
format: 'tgz',
retentionPolicy: { keepWithinSecs: 10000 }
};
before(function (done) {
rimraf.sync(gBackupConfig.backupFolder);
rimraf.sync(backupConfig.backupFolder);
done();
});
after(function (done) {
rimraf.sync(gBackupConfig.backupFolder);
rimraf.sync(backupConfig.backupFolder);
done();
});
it('fails to set backup config for non-existing folder', function (done) {
settings.setBackupConfig(gBackupConfig, function (error) {
settings.setBackupConfig(backupConfig, function (error) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.BAD_FIELD);
......@@ -304,9 +305,9 @@ describe('backups', function () {
});
it('succeeds to set backup config', function (done) {
mkdirp.sync(gBackupConfig.backupFolder);
mkdirp.sync(backupConfig.backupFolder);
settings.setBackupConfig(gBackupConfig, function (error) {
settings.setBackupConfig(backupConfig, function (error) {
expect(error).to.be(null);
done();
......@@ -318,10 +319,9 @@ describe('backups', function () {
if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done();
createBackup(function (error, result) {
console.dir(error);
expect(error).to.be(null);
expect(fs.statSync(path.join(gBackupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(gBackupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2);
expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2);
backupInfo1 = result;
......@@ -335,9 +335,9 @@ console.dir(error);
createBackup(function (error, result) {
expect(error).to.be(null);
expect(fs.statSync(path.join(gBackupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(gBackupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup
expect(fs.statSync(path.join(gBackupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore
expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore
done();
});
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment