diff options
Diffstat (limited to 'scripts/gerrit/cherry-pick_automation/toolbox.js')
| -rw-r--r-- | scripts/gerrit/cherry-pick_automation/toolbox.js | 416 |
1 files changed, 415 insertions, 1 deletions
diff --git a/scripts/gerrit/cherry-pick_automation/toolbox.js b/scripts/gerrit/cherry-pick_automation/toolbox.js index 121367dc..55989075 100644 --- a/scripts/gerrit/cherry-pick_automation/toolbox.js +++ b/scripts/gerrit/cherry-pick_automation/toolbox.js @@ -7,6 +7,7 @@ const safeJsonStringify = require("safe-json-stringify"); const v8 = require('v8'); const postgreSQLClient = require("./postgreSQLClient"); +const { queryBranchesRe, checkBranchAndAccess, queryChange } = require("./gerritRESTTools"); const Logger = require("./logger"); const logger = new Logger(); @@ -17,7 +18,8 @@ let dbListenerCacheUpdateLockout = false; // Deep copy an object. Useful for forking processing paths with different data // than originally entered the system. -exports.deepCopy = function (obj) { +exports.deepCopy = deepCopy; +function deepCopy(obj) { return v8.deserialize(v8.serialize(obj)); } @@ -43,6 +45,418 @@ exports.findPickToBranches = function (requestUuid, message) { return branchSet; }; +// Build a waterfall of cherry picks based on the list of branches. +// Picks to older feature and release branches must be attached to newer +// feature branch picks. +// Example: for input of ["6.5.0", "6.4", "5.15.12", "5.15.11", "5.15"], +// the output will be: +// { +// '6.5.0': [], +// '6.4': ['5.15.12', '5.15.11', '5.15'] +// } +exports.waterfallCherryPicks = function (requestUuid, branchList) { + branchList = Array.from(branchList); // Might be a set. + + if (branchList.length === 0) { + logger.log(`No branches to waterfall.`, 'debug', requestUuid); + return {}; + } + + // If any of the entries of branchList are not numerical, + // then return a waterfall object with each branch as a key. + if (branchList.some((branch) => !/^(dev|master|\d+\.\d+(\.\d+)?)$/.test(branch))) { + let waterfall = {}; + for (let branch of branchList) { + waterfall[branch] = []; + } + return waterfall; + } + + branchList = sortBranches(branchList); + + let waterfall = {}; + let youngestFeatureBranch = branchList.find(isFeatureBranchOrDev); + let remainingBranches = branchList.filter((branch) => branch !== youngestFeatureBranch); + if (!youngestFeatureBranch || remainingBranches.length === 0) { + waterfall[youngestFeatureBranch || remainingBranches[0]] = []; + return waterfall; + } + + let children = remainingBranches.filter((branch) => { + let result = (isAncestor(branch, youngestFeatureBranch) + || isChild(branch, youngestFeatureBranch)); + return result; + }); + + waterfall[youngestFeatureBranch] = children; + + remainingBranches = remainingBranches.filter((branch) => !children.includes(branch)); + + for (let branch of remainingBranches) { + waterfall[branch] = []; + } + + return waterfall; +}; + +// Determine if a given branch is a child of another branch. +function isChild(maybeChild, maybeParent) { + // Account for dev and master branches. + if (maybeChild === "dev" || maybeChild === "master") { + return false; + } + if (maybeParent === "dev" || maybeParent === "master") { + return true; + } + let childParts = maybeChild.split('.').map(Number); + let parentParts = maybeParent.split('.').map(Number); + + // Major version less than the parent is always a child. + if (childParts[0] < parentParts[0]) { + return true; + } + + // Feature versions are children if lesser than the parent. This + // also catches release versions of the same or newer feature versions. + if (childParts[1] <= parentParts[1]) { + return true; + } + + // Then the release version is newer than the parent. + return false; +} + +// Determine if a given branch is an ancestor of another branch. +function isAncestor(maybeAncestor, reference) { + // Account for dev and master branches. + if (maybeAncestor === "dev" || maybeAncestor === "master") { + return true; + } + if (reference === "dev" || reference === "master") { + return false; + } + let branchParts = maybeAncestor.split('.').map(Number); + let ancestorParts = reference.split('.').map(Number); + + // Release branches like 5.15.0 cannot be ancestors of feature branches like 5.15. + if (branchParts.length > ancestorParts.length) { + return false; + } + + // Major versions that are less than the passed ancestor are ancestors. + if (branchParts[0] < reference[0]) { + return true; + } + + // Feature versions are ancestors if it is greater than the reference. + if (branchParts[1] > reference[1]) { + return true; + } + + return true; +} + +function isFeatureBranchOrDev(branch) { + return branch === "dev" || branch === "master" || branch.split('.').length === 2; +} + +// given a list of pick-to branches, determine if any gaps exist in the list. +// Return a completed list of pick-to targets with a diff. +exports.findMissingTargets = function (uuid, changeId, project, targets, callback) { + + const isTqtc = /tqtc-/.test(project); // Is the cherry-pick request coming from a tqtc repo? + const isLTS = /^(tqtc\/)?lts-/.test(decodeURIComponent(changeId).split('~')[1]); // Is the change on an LTS branch? + const prefix = isTqtc && isLTS + ? "tqtc/lts-" + : isTqtc + ? "tqtc/" + : isLTS + ? "lts-" : ""; + const bareBranch = decodeURIComponent(changeId).split('~')[1].replace(/^(tqtc\/)?(lts-)?/, ''); + let highestTarget = ""; + + if (Array.from(targets).length === 0) { + logger.log(`No targets to check.`, 'debug', uuid); + callback(false, undefined, []); + return; + } + + // targets will always be bare versions, like "5.15". + let featureBranches = new Set(); + for (let target of targets) { + // account for dev and master branches. + if (target === "dev" || target === "master") { + continue; + } + let parts = target.split('.'); + featureBranches.add(parts[0] + '.' + parts[1]); + } + highestTarget = Array.from(featureBranches).sort(sortBranches)[0]; + if (isChild(highestTarget, bareBranch)) { + // If the highest target is a child of the bare branch, then the highest target + // should be considered self. + highestTarget = bareBranch; + } + + + + let releaseBranches = {}; + for (let key of featureBranches) { + releaseBranches[key] = []; + } + for (let target of targets) { + // account for dev and master branches. + if (target === "dev" || target === "master") { + continue; + } + let parts = target.split('.'); + if (parts.length === 3) { + releaseBranches[parts[0] + '.' + parts[1]].push(target); + } + } + + // Sort by release version + for (let key of featureBranches) { + releaseBranches[key].sort((a, b) => { + let aParts = a.split('.').map(Number); + let bParts = b.split('.').map(Number); + return aParts[2] - bParts[2]; + }); + } + + // Filter result branches if the originating change had a tqtc and/or an LTS prefix. + const searchRe = `(${prefix}(?:${Array.from(featureBranches).join('|')}|[6-9].[0-9]{1,}|${prefix}dev|${prefix}master)).*` + queryBranchesRe(uuid, project, false, searchRe, undefined, (success, remoteBranches) => { + if (!success) { + return; + } + + function makeNextFeature(branch) { + return branch.split('.').slice(0, 1).join('.') + + '.' + (Number(branch.split('.')[1]) + 1); // f.e. 5.15 -> 5.16 + } + + // Use sanitized branches to determine if any gaps exist. + const bareRemoteBranches = remoteBranches.map((branch) => branch.replace(/^(tqtc\/)?(lts-)?/, '')); + + function _finishUp(error, change) { + // Always include all gaps in release branches. + for (let branch of featureBranches) { + for (let release of releaseBranches[branch]) { + const nextRelease = release.split('.').slice(0, 2).join('.') + + '.' + (Number(release.split('.')[2]) + 1); // f.e. 5.15.12 -> 5.15.13 + if (releaseBranches[branch].includes(nextRelease)) { + continue; + } + if (bareRemoteBranches.includes(nextRelease)) { + missing.push(nextRelease); + } + } + } + + if (missing.length) { + logger.log(`Missing branches: ${missing.join(', ')}`, 'info', uuid); + } + callback(Boolean(error), change, missing); + } + + let missing = []; + function _findMissingFeatures(branch, start = false) { + if (start) { + if (bareRemoteBranches.includes(branch)) { + if (branch === bareBranch) { + // Do not test the branch we're picking from. It must exist. + _findMissingFeatures(branch); // recurse normally + } else if (!targets.has(branch)) { + // Check to see if the next feature branch is still open for new changes. + checkBranchAndAccess(uuid, project, prefix + branch, "cherrypickbot", "push", undefined, + (success, hasAccess) => { + if (success && hasAccess) { + const missingChangeId = `${encodeURIComponent(project)}~` + + `${encodeURIComponent(prefix + branch)}~${changeId.split('~').pop()}` + queryChange(uuid, missingChangeId, undefined, undefined, (success) => { + if (!success) + missing.push(branch); + else + logger.log(`Skipping ${branch}. Change already exists.`, "debug", uuid); + _findMissingFeatures(branch); // recurse normally + }); + } else { + logger.log(`Skipping ${branch} because it is closed.`, "debug", uuid); + _findMissingFeatures(branch); // recurse normally + } + }); + } else { + logger.log(`Skipping ${branch} because it is already in the list.`, "debug", uuid); + _findMissingFeatures(branch); // recurse normally + } + } else { + logger.log(`Skipping ${branch} because it does not exist.`, "debug", uuid); + _findMissingFeatures(branch); // recurse normally + } + return; + } + + const nextFeature = makeNextFeature(branch); + if (bareRemoteBranches.includes(nextFeature)) { + if (nextFeature === bareBranch || featureBranches.has(nextFeature)) { + _findMissingFeatures(nextFeature); // Recurse to the next feature branch. + } else if (isChild(nextFeature, highestTarget)) { + // Only test branches which are older than/children to the current branch. + // This forces the waterfall to only work downwards from the highest + // specified branch. + // Check to see if the next feature branch is still open for new changes. + checkBranchAndAccess(uuid, project, prefix + nextFeature, "cherrypickbot", "push", undefined, + (success, hasAccess) => { + if (success && hasAccess) { + const missingChangeId = `${encodeURIComponent(project)}~` + + `${encodeURIComponent(prefix + nextFeature)}~${changeId.split('~').pop()}` + queryChange(uuid, missingChangeId, undefined, undefined, (success, data) => { + if (!success) + missing.push(prefix + nextFeature); + else { + if (data.status == "MERGED") { + logger.log(`Skipping ${prefix + nextFeature}. Merged change already exists.`, "debug", uuid); + } else { + const _next = makeNextFeature(nextFeature); + if (!bareRemoteBranches.includes(_next)) { + // This was the last and highest feature branch, which means it would + // be our first pick target (excepting dev). Since the change is open, + // we should not touch it. + // Call _finishUp() and pick to only any immediate release targets. + logger.log(`Missing immediate target ${prefix + nextFeature} has a change in ${data.status}.`, "error", uuid); + _finishUp(true, data); + return; + } + } + } + _findMissingFeatures(nextFeature); + }); + } else { + logger.log(`Skipping ${prefix + nextFeature} because it is closed.`, "debug", uuid); + _findMissingFeatures(nextFeature); + } + }); + } else { + // NextFeature exists remotely, but is out of scope based on pick targets and source branch. + _finishUp(); + } + } else { + // We've reached the end of the feature branches which are older than the highest branch. + _finishUp(); + } + } + + // Use the oldest feature version to find any gaps since then. + if (Array.from(featureBranches).length === 0) { + _finishUp(); + return; + } + try { + _findMissingFeatures(Array.from(featureBranches).sort(sortBranches)[0], true); + } catch (e) { + logger.log(`Error finding missing targets: ${e}`, 'error', uuid); + } + }); +} + +exports.sortBranches = sortBranches; +function sortBranches(branches) { + return Array.from(branches).sort((a, b) => { + // Account for dev and master branches. + if (a === "dev" || a === "master") { + return -1; + } + if (b === "dev" || b === "master") { + return 1; + } + let aParts = a.split('.').map(Number); + let bParts = b.split('.').map(Number); + + for (let i = 0; i < Math.min(aParts.length, bParts.length); i++) { + if (aParts[i] !== bParts[i]) { + return bParts[i] - aParts[i]; + } + } + + return aParts.length - bParts.length; + }); +} + +// Take a gerrit Change object and mock a change-merged event. +// Use the original merge event as the template for the mocked event. +exports.mockChangeMergedFromOther = mockChangeMergedFromOther; +function mockChangeMergedFromOther(uuid, originalMerge, targetBranch, remainingPicks, callback) { + if (remainingPicks.size == 0) { + logger.log(`No remaining picks for existing target on ${targetBranch}. Nothing to do.`, + "debug", uuid); + callback(null); + return; + } + let mockMerge = deepCopy(originalMerge); + // Assemble the fullChangeID from the project and branch. + let targetChangeId = encodeURIComponent(mockMerge.change.project) + "~" + + encodeURIComponent(targetBranch) + "~" + mockMerge.change.id; + // Query the target change from gerrit. + queryChange(uuid, targetChangeId, undefined, undefined, function (success, targetChange) { + if (!success) { + logger.log(`Error mocking change-merged event: ${targetChange}, ${targetChange}`, "error", uuid); + callback(null); + return; + } + // Replace the following properties of mockMerge with the targetChange: + // newRev + // patchSet + // change + // refName + // changeKey + // fullChangeID + // eventCreatedOn + delete mockMerge.uuid; + mockMerge.newRev = targetChange.current_revision; + mockMerge.patchSet = { + number: targetChange.revisions[targetChange.current_revision]._number, + revision: targetChange.current_revision, + parents: targetChange.revisions[targetChange.current_revision].commit.parents, + ref: `refs/changes/${targetChange._number}/${targetChange.current_revision}`, + uploader: targetChange.owner, + author: targetChange.revisions[targetChange.current_revision].commit.author, + createdOn: targetChange.created + }; + targetChange.id = targetChange.change_id; + targetChange.number = targetChange._number; + // build the url from the project and change number. + targetChange.url = `https://codereview.qt-project.org/c/${targetChange.project}/+/` + + `${targetChange._number}`; + // Replace Pick-to: footer with the remaining picks. + // If targetChange did not have a Pick-to footer, add one at the beginning + // of the footers. Footers are separated from the commit message body by the last\n\n. + const origCommitMessage = targetChange.revisions[targetChange.current_revision].commit.message; + let footerIndex = origCommitMessage.lastIndexOf("\n\n"); + if (footerIndex === -1) { + footerIndex = origCommitMessage.length; + } else { + footerIndex += 2; // Start after the last \n\n. + } + let footer = origCommitMessage.slice(footerIndex); + let pickToFooter = "Pick-to: " + Array.from(remainingPicks).join(" "); + if (footer.match(/Pick-to:.*$/m)) { + footer = footer.replace(/Pick-to:.*$/m, pickToFooter); + } else { + footer = pickToFooter + "\n\n" + footer; + } + targetChange.commitMessage = origCommitMessage.slice(0, footerIndex) + footer; + mockMerge.change = targetChange; + mockMerge.refName = `refs/heads/${targetChange.branch}`; + mockMerge.changeKey = { key: targetChange.change_id }; + mockMerge.fullChangeID = encodeURIComponent(targetChange.project) + "~" + + encodeURIComponent(targetChange.branch) + "~" + targetChange.change_id; + mockMerge.eventCreatedOn = Date.now(); + + callback(mockMerge); + }); +} + // Get all database items in the processing_queue with state "processing" exports.getAllProcessingRequests = getAllProcessingRequests; function getAllProcessingRequests(callback) { |