diff options
Diffstat (limited to 'webapp/codereview/models.py')
| -rw-r--r-- | webapp/codereview/models.py | 1480 |
1 files changed, 1480 insertions, 0 deletions
diff --git a/webapp/codereview/models.py b/webapp/codereview/models.py new file mode 100644 index 0000000000..60a0515b0a --- /dev/null +++ b/webapp/codereview/models.py @@ -0,0 +1,1480 @@ +# Copyright 2008 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""App Engine data model (schema) definition for Gerrit.""" + +# Python imports +import base64 +import datetime +import hashlib +import logging +import random +import re +import zlib + +# AppEngine imports +from google.appengine.ext import db +from google.appengine.api import memcache +from google.appengine.api import users + +# Local imports +from memcache import Key as MemCacheKey +import patching + + +DEFAULT_CONTEXT = 10 +CONTEXT_CHOICES = (3, 10, 25, 50, 75, 100) +FETCH_MAX = 1000 +MAX_DELTA_DEPTH = 10 + +LGTM_CHOICES = ( + ('lgtm', 'Looks good to me, approved.'), + ('yes', 'Looks good to me, but someone else must approve.'), + ('abstain', 'No score.'), + ('no', 'I would prefer that you didn\'t submit this.'), + ('reject', 'Do not submit.'), + ) +LIMITED_LGTM_CHOICES = [choice for choice in LGTM_CHOICES + if choice[0] != 'lgtm' and choice[0] != 'reject'] + +### GQL query cache ### + + +_query_cache = {} + +class BackedUpModel(db.Model): + """Base class for our models that keeps a property used for backup.""" + + last_backed_up = db.IntegerProperty(default=0) + + def __init__(self, *args, **kwargs): + db.Model.__init__(self, *args, **kwargs) + +def gql(cls, clause, *args, **kwds): + """Return a query object, from the cache if possible. + + Args: + cls: a BackedUpModel subclass. + clause: a query clause, e.g. 'WHERE draft = TRUE'. + *args, **kwds: positional and keyword arguments to be bound to the query. + + Returns: + A db.GqlQuery instance corresponding to the query with *args and + **kwds bound to the query. + """ + query_string = 'SELECT * FROM %s %s' % (cls.kind(), clause) + query = _query_cache.get(query_string) + if query is None: + _query_cache[query_string] = query = db.GqlQuery(query_string) + query.bind(*args, **kwds) + return query + + +### Exceptions ### + +class InvalidLgtmException(Exception): + """User is not alloewd to LGTM this change.""" + +class InvalidVerifierException(Exception): + """User is not alloewd to verify this change.""" + +class InvalidSubmitMergeException(Exception): + """The change cannot me scheduled for merge.""" + +class DeltaPatchingException(Exception): + """Applying a patch yield the wrong hash.""" + + +### Settings ### + +def _genkey(n=26): + k = ''.join(map(chr, (random.randrange(256) for i in xrange(n)))) + return base64.b64encode(k) + +class Settings(BackedUpModel): + """Global settings for the application instance.""" + + analytics = db.StringProperty() + internal_api_key = db.StringProperty() + xsrf_key = db.StringProperty() + from_email = db.StringProperty() + + _Key = MemCacheKey('Settings_Singleton') + + @classmethod + def get_settings(cls): + """Get the Settings singleton. + + If possible, get it from memcache. If it's not there, it tries to do a + normal get(). Only if that fails does it call get_or_insert, because of + possible contention errors due to get_or_insert's transaction. + """ + def read(): + result = cls.get_by_key_name('settings') + if result: + return result + else: + return cls.get_or_insert('settings', + internal_api_key=_genkey(26), + xsrf_key=_genkey(26)) + return Settings._Key.get(read) + + def put(self): + BackedUpModel.put(self) + self._Key.clear() + + +### Approval rights ### + +def _flatten_users_and_groups(users, groups): + """Returns a set of the users and the groups provided""" + result = set() + for user in users: + result.add(user) + if groups: + for group in db.get(groups): + for user in group.members: + result.add(user) + return result + + +class ApprovalRight(BackedUpModel): + """The tuple of a set of path patterns and a set of users who can approve + changes for those paths.""" + files = db.StringListProperty() + approvers_users = db.ListProperty(users.User) + approvers_groups = db.ListProperty(db.Key) + verifiers_users = db.ListProperty(users.User) + verifiers_groups = db.ListProperty(db.Key) + required = db.BooleanProperty() + + def approvers(self): + """Returns a set of the users who are approvers.""" + return _flatten_users_and_groups(self.approvers_users, + self.approvers_groups) + + def verifiers(self): + """Returns a set of the users who are verifiers.""" + return _flatten_users_and_groups(self.verifiers_users, + self.verifiers_groups) + + @classmethod + def validate_file(cls, file): + """Returns whether this is a valid file path. + + The rules: + - The length must be > 0 + - The file path must start with a '/' + - The file path must contain either 0 or 1 '...' + - If it contains one '...', it must either be last or directly + after the first '/' + + These last two limitations could be removed someday but are + good enough for now. + """ + if len(file) == 0: + return False + if file[0] != '/': + return False + (before, during, after) = file.partition("...") + if during == "" and after == "": + return True + if before != "/": + return False + if after.find("...") != -1: + return False + return True + +### Projects ### + +class Project(BackedUpModel): + """An open source project. + + Projects have owners who can set approvers and stuff. + """ + + name = db.StringProperty(required=True) + comment = db.StringProperty(required=False) + owners_users = db.ListProperty(users.User) + owners_groups = db.ListProperty(db.Key) + code_reviews = db.ListProperty(db.Key) + + @classmethod + def get_all_projects(cls): + """Return all projects""" + all = cls.all() + all.order('name') + return list(all) + + @classmethod + def get_project_for_name(cls, name): + return cls.gql('WHERE name=:name', name=name).get() + + def remove(self): + """delete this project""" + db.delete(ApprovalRight.get(self.code_reviews)) + self.delete() + + def set_code_reviews(self, approval_right_keys): + for key in self.code_reviews: + val = ApprovalRight.get(key) + if val: + db.delete(val) + self.code_reviews = approval_right_keys + + def get_code_reviews(self): + return [ApprovalRight.get(key) for key in self.code_reviews] + + def is_code_reviewed(self): + return True + + def put(self): + memcache.flush_all() + BackedUpModel.put(self) + + @classmethod + def projects_owned_by_user(cls, user): + memcache.flush_all() + key = "projects_owned_by_user_%s" % user.email() + result = memcache.get(key) + if result is None: + result = set( + Project.gql("WHERE owner_users=:user", user=user).fetch(1000)) + groups = AccountGroup.gql("WHERE members=:user", user=user).fetch(1000) + if groups: + pr = Project.gql( + "WHERE owners_groups IN :groups", + groups=[g.key() for g in groups]).fetch(1000) + result.update(pr) + memcache.set(key, [p.key() for p in result]) + return result + + +class Branch(BackedUpModel): + """A branch in a specific Project.""" + + project = db.ReferenceProperty(Project, required=True) + name = db.StringProperty(required=True) # == key + + status = db.StringProperty(choices=('NEEDS_MERGE', + 'MERGING', + 'BUILDING')) + merge_submitted = db.DateTimeProperty() + to_merge = db.ListProperty(db.Key) # PatchSets + merging = db.ListProperty(db.Key) # PatchSets + waiting = db.ListProperty(db.Key) # PatchSets + + @classmethod + def get_or_insert_branch(cls, project, name): + key = 'p.%s %s' % (project.key().id(), name) + return cls.get_or_insert(key, project=project, name=name) + + @classmethod + def get_branch_for_name(cls, project, name): + key = 'p.%s %s' % (project.key().id(), name) + return cls.get_by_key_name(key) + + @property + def short_name(self): + if self.name.startswith("refs/heads/"): + return self.name[len("refs/heads/"):] + return self.name + + def is_code_reviewed(self): + return True + + def merge_patchset(self, patchset): + """Add a patchset to the end of the branch's merge queue + + This method runs in an independent transaction. + """ + ps_key = patchset.key() + def trans(key): + b = db.get(key) + if not ps_key in b.to_merge: + b.to_merge.append(ps_key) + if b.status is None: + b.status = 'NEEDS_MERGE' + b.merge_submitted = datetime.datetime.now() + b.put() + db.run_in_transaction(trans, self.key()) + + def begin_merge(self): + """Lock this branch and start merging patchsets. + + This method runs in an independent transaction. + """ + def trans(key): + b = db.get(key) + if b.status == 'NEEDS_MERGE': + b.status = 'MERGING' + b.merging.extend(b.waiting) + b.merging.extend(b.to_merge) + b.waiting = [] + b.to_merge = [] + b.put() + return b.merging + return [] + keys = db.run_in_transaction(trans, self.key()) + objs = db.get(keys) + + good = [] + torm = [] + for k, o in zip(keys, objs): + if o: + good.append(o) + else: + torm.append(k) + + if torm: + def clear_branch(key): + b = db.get(key) + + for ps_key in torm: + if ps_key in b.merging: + b.merging.remove(ps_key) + + if not good and b.status in ('MERGING', 'BUILDING'): + if b.to_merge: + b.status = 'NEEDS_MERGE' + else: + b.status = None + + b.put() + db.run_in_transaction(clear_branch, self.key()) + return good + + def finish_merge(self, success, fail, defer): + """Update our patchset lists with the results of a merge. + + This method runs in an independent transaction. + """ + def trans(key): + b = db.get(key) + + rm = [] + rm.extend(success) + rm.extend(fail) + for ps in rm: + ps_key = ps.key() + + if ps_key in b.to_merge: + b.to_merge.remove(ps_key) + if ps_key in b.merging: + b.merging.remove(ps_key) + if ps_key in b.waiting: + b.waiting.remove(ps_key) + + for ps in defer: + ps_key = ps.key() + + if ps_key in b.to_merge: + b.to_merge.remove(ps_key) + if ps_key in b.merging: + b.merging.remove(ps_key) + if ps_key not in b.waiting: + b.waiting.append(ps_key) + + b.put() + db.run_in_transaction(trans, self.key()) + + def merged(self, merged): + """Updates the branch to include pending PatchSets. + + This method runs in an independent transaction. + """ + def trans(key): + b = db.get(key) + + for ps in merged: + ps_key = ps.key() + + if ps_key in b.to_merge: + b.to_merge.remove(ps_key) + if ps_key in b.merging: + b.merging.remove(ps_key) + if ps_key in b.waiting: + b.waiting.remove(ps_key) + + if b.status in ('MERGING', 'BUILDING'): + if b.to_merge: + b.status = 'NEEDS_MERGE' + else: + b.status = None + b.put() + db.run_in_transaction(trans, self.key()) + + +### Revisions ### + +class RevisionId(BackedUpModel): + """A specific revision of a project.""" + + project = db.ReferenceProperty(Project, required=True) + id = db.StringProperty(required=True) # == key + + author_name = db.StringProperty() + author_email = db.EmailProperty() + author_when = db.DateTimeProperty() + author_tz = db.IntegerProperty() + + committer_name = db.StringProperty() + committer_email = db.EmailProperty() + committer_when = db.DateTimeProperty() + committer_tz = db.IntegerProperty() + + ancestors = db.StringListProperty() # other RevisionId.id + message = db.TextProperty() + + patchset_key = db.StringProperty() + def _get_patchset(self): + try: + return self._patchset_obj + except AttributeError: + k_str = self._patchset_key + if k_str: + self._patchset_obj = db.get(db.Key(k_str)) + else: + self._patchset_obj = None + return self._patchset_obj + + def _set_patchset(self, p): + if p is None: + self._patchset_key = None + self._patchset_obj = None + else: + self._patchset_key = str(p.key()) + self._patchset_obj = p + patchset = property(_get_patchset, _set_patchset) + + @classmethod + def get_or_insert_revision(cls, project, id, **kw): + key = 'p.%s %s' % (project.key().id(), id) + return cls.get_or_insert(key, project=project, id=id, **kw) + + @classmethod + def get_revision(cls, project, id): + key = 'p.%s %s' % (project.key().id(), id) + return cls.get_by_key_name(key) + + @classmethod + def get_for_patchset(cls, patchset): + """Get all revisions linked to a patchset. + """ + return gql(cls, 'WHERE patchset_key = :1', str(patchset.key())) + + def add_ancestor(self, other_id): + """Adds the other revision as an ancestor for this one. + + If the other rev is already in the list, does nothing. + """ + if not other_id in self.ancestors: + self.ancestors.append(other_id) + + def remove_ancestor(self, other_id): + """Removes an ancestor previously stored. + + If the other rev is not already in the list, does nothing. + """ + if other_id in self.ancestors: + self.ancestors.remove(other_id) + + def get_ancestors(self): + """Fully fetches all ancestors from the data store. + """ + p_id = self.project.key().id() + names = ['p.%s %s' % (p_id, i) for i in self.ancestors] + return [r for r in RevisionId.get_by_key_name(names) if r] + + def get_children(self): + """Obtain the revisions that depend upon this one. + """ + return gql(RevisionId, + 'WHERE project = :1 AND ancestors = :2', + self.project, self.id) + + def link_patchset(self, new_patchset): + """Uniquely connect one patchset to this revision. + + Returns True if the passed patchset is the single patchset; + False if another patchset has already been linked onto it. + """ + def trans(self_key): + c = db.get(self_key) + if c.patchset is None: + c.patchset = new_patchset + c.put() + return True + return False + return db.run_in_transaction(trans, self.key()) + + +class BuildAttempt(BackedUpModel): + """A specific build attempt.""" + + branch = db.ReferenceProperty(Branch, required=True) + revision_id = db.StringProperty(required=True) + new_changes = db.ListProperty(db.Key) # PatchSet + + started = db.DateTimeProperty(auto_now_add=True) + finished = db.BooleanProperty(default=False) + success = db.BooleanProperty() + + +### Changes, PatchSets, Patches, DeltaContents, Comments, Messages ### + +class Change(BackedUpModel): + """The major top-level entity. + + It has one or more PatchSets as its descendants. + """ + + subject = db.StringProperty(required=True) + description = db.TextProperty() + owner = db.UserProperty(required=True) + created = db.DateTimeProperty(auto_now_add=True) + modified = db.DateTimeProperty(auto_now=True) + reviewers = db.ListProperty(db.Email) + claimed = db.BooleanProperty(default=False) + cc = db.ListProperty(db.Email) + closed = db.BooleanProperty(default=False) + n_comments = db.IntegerProperty(default=0) + n_patchsets = db.IntegerProperty(default=0) + + dest_project = db.ReferenceProperty(Project, required=True) + dest_branch = db.ReferenceProperty(Branch, required=True) + + merge_submitted = db.DateTimeProperty() + merged = db.BooleanProperty(default=False) + + emailed_clean_merge = db.BooleanProperty(default=False) + emailed_missing_dependency = db.BooleanProperty(default=False) + emailed_path_conflict = db.BooleanProperty(default=False) + + merge_patchset_key = db.StringProperty() + def _get_merge_patchset(self): + try: + return self._merge_patchset_obj + except AttributeError: + k_str = self._merge_patchset_key + if k_str: + self._merge_patchset_obj = db.get(db.Key(k_str)) + else: + self._merge_patchset_obj = None + return self._merge_patchset_obj + + def _set_merge_patchset(self, p): + if p is None: + self._merge_patchset_key = None + self._merge_patchset_obj = None + else: + self._merge_patchset_key = str(p.key()) + self._merge_patchset_obj = p + merge_patchset = property(_get_merge_patchset, _set_merge_patchset) + + _is_starred = None + + @property + def is_starred(self): + """Whether the current user has this change starred.""" + if self._is_starred is not None: + return self._is_starred + account = Account.current_user_account + self._is_starred = account is not None and self.key().id() in account.stars + return self._is_starred + + def update_comment_count(self, n): + """Increment the n_comments property by n. + """ + self.n_comments += n + + @property + def num_comments(self): + """The number of non-draft comments for this change. + + This is almost an alias for self.n_comments, except that if + n_comments is None, it is computed through a query, and stored, + using n_comments as a cache. + """ + return self.n_comments + + _num_drafts = None + + @property + def num_drafts(self): + """The number of draft comments on this change for the current user. + + The value is expensive to compute, so it is cached. + """ + if self._num_drafts is None: + account = Account.current_user_account + if account is None: + self._num_drafts = 0 + else: + query = gql(Comment, + 'WHERE ANCESTOR IS :1 AND author = :2 AND draft = TRUE', + self, account.user) + self._num_drafts = query.count() + return self._num_drafts + + def new_patchset(self, **kw): + """Construct a new patchset for this change. + """ + def trans(change_key): + change = db.get(change_key) + change.n_patchsets += 1 + id = change.n_patchsets + change.put() + + patchset = PatchSet(change=change, parent=change, id=id, **kw) + patchset.put() + return patchset + return db.run_in_transaction(trans, self.key()) + + def set_review_status(self, user): + """Gets or inserts the ReviewStatus object for the suppiled user.""" + return ReviewStatus.get_or_insert_status(self, user) + + def get_review_status(self, user=None): + """Return the lgtm status for the given user if supplied. All for this + change otherwise.""" + if user: + # The owner must be checked separately because she automatically + # approves / verifies her own change and there is no ReviewStatus + # for that one. + if user == self.owner: + return [] + return ReviewStatus.get_status_for_user(self, user) + else: + return ReviewStatus.all_for_change(self) + + @classmethod + def get_reviewer_status(cls, reviews): + """Return a tuple of who has commented on the changes. + + The owner of the change is automatically added to the list + + Args: + reviews a list of ReviewStatus objects are returned from + get_review_status(). + + Returns: + A map of the LGTM_CHOICES keys to users, plus the mapping + verified_by --> the uesrs who verified it + """ + result = {} + for (k,v) in LGTM_CHOICES: + result[k] = [r.user for r in reviews if r.lgtm == k] + result["verified_by"] = [r.user for r in reviews if r.verified] + return result + + @property + def is_submitted(self): + """Return true if the change has been submitted for merge. + """ + return self.merge_submitted is not None + + def submit_merge(self, patchset): + """Schedule a specific patchset of the change to be merged. + """ + branch = self.dest_branch + if not branch: + raise InvalidSubmitMergeException, 'No branch defined' + + if self.merged: + raise InvalidSubmitMergeException, 'Already merged' + + if self.is_submitted: + raise InvalidSubmitMergeException, \ + "Already merging patch set %s" % self.merge_patchset.id + + branch.merge_patchset(patchset) + self.merge_submitted = datetime.datetime.now() + self.merge_patchset = patchset + self.emailed_clean_merge = False + self.emailed_missing_dependency = False + self.emailed_path_conflict = False + + def unsubmit_merge(self): + """Unschedule a merge of this change. + """ + if self.merged: + raise InvalidSubmitMergeException, 'Already merged' + + self.merge_submitted = None + self.merge_patchset = None + + def set_reviewers(self, reviewers): + self.reviewers = reviewers + self.claimed = len(reviewers) != 0 + + +class PatchSetFilenames(BackedUpModel): + """A list of the file names in a PatchSet. + + This is a descendant of a PatchSet. + """ + + compressed_filenames = db.BlobProperty() + + @classmethod + def _mc(cls, patchset): + return MemCacheKey("PatchSet %s filenames" % patchset.key()) + + @classmethod + def store_compressed(cls, patchset, bin): + cls(key_name = 'filenames', + compressed_filenames = db.Blob(bin), + parent = patchset).put() + cls._mc(patchset).set(cls._split(bin)) + + @classmethod + def get_list(cls, patchset): + def read(): + c = cls.get_by_key_name('filenames', parent = patchset) + if c: + return cls._split(c.compressed_filenames) + names = patchset._all_filenames() + bin = zlib.compress("\0".join(names).encode('utf_8'), 9) + cls(key_name = 'filenames', + compressed_filenames = db.Blob(bin), + parent = patchset).put() + return names + return cls._mc(patchset).get(read) + + @classmethod + def _split(cls, bin): + tmp = zlib.decompress(bin).split("\0") + return [s.decode('utf_8') for s in tmp] + + +class PatchSet(BackedUpModel): + """A set of patchset uploaded together. + + This is a descendant of an Change and has Patches as descendants. + """ + + id = db.IntegerProperty(required=True) + change = db.ReferenceProperty(Change, required=True) # == parent + message = db.StringProperty() + owner = db.UserProperty(required=True) + created = db.DateTimeProperty(auto_now_add=True) + modified = db.DateTimeProperty(auto_now=True) + revision = db.ReferenceProperty(RevisionId, required=True) + complete = db.BooleanProperty(default=False) + + _filenames = None + + @property + def filenames(self): + if self._filenames is None: + self._filenames = PatchSetFilenames.get_list(self) + return self._filenames + + def _all_filenames(self): + last = '' + names = [] + while True: + list = gql(Patch, + 'WHERE patchset = :1 AND filename > :2' + ' ORDER BY filename', + self, last).fetch(500) + if not list: + break + for p in list: + names.append(p.filename) + last = list[-1].filename + return names + + +class Message(BackedUpModel): + """A copy of a message sent out in email. + + This is a descendant of an Change. + """ + + change = db.ReferenceProperty(Change, required=True) # == parent + subject = db.StringProperty() + sender = db.EmailProperty() + recipients = db.ListProperty(db.Email) + date = db.DateTimeProperty(auto_now_add=True) + text = db.TextProperty() + + +class CachedDeltaContent(object): + """A fully inflated DeltaContent stored in memcache. + """ + def __init__(self, dc): + self.data_lines = dc.data_lines + self.patch_lines = dc.patch_lines + self.is_patch = dc.is_patch + self.is_data = dc.is_data + + @property + def data_text(self): + if self.data_lines is None: + return None + return ''.join(self.data_lines) + + @property + def patch_text(self): + if self.patch_lines is None: + return None + return ''.join(self.patch_lines) + + @classmethod + def get(cls, key): + def load(): + dc = db.get(key) + if dc: + return cls(dc) + return None + return MemCacheKey('DeltaContent:%s' % key.name(), + compress = True).get(load) + + +def _apply_patch(old_lines, patch_name, dif_lines): + new_lines = [] + chunks = patching.ParsePatchToChunks(dif_lines, patch_name) + for tag, old, new in patching.PatchChunks(old_lines, chunks): + new_lines.extend(new) + return ''.join(new_lines) + +def _blob_hash(data): + m = hashlib.sha1() + m.update("blob %d\0" % len(data)) + m.update(data) + return m.hexdigest() + + +class DeltaContent(BackedUpModel): + """Any content, such as for the old or new image of a Patch, + or the patch data itself. + + These are stored as top-level entities. + + Key: + Git blob SHA-1 of inflate(text) + -or- + Randomly generated name if this is a patch + """ + + text_z = db.BlobProperty(required=True) + depth = db.IntegerProperty(default=0, required=True) + base = db.SelfReferenceProperty() + + _data_lines = None + _data_text = None + _patch_text = None + _patch_lines = None + + @classmethod + def create_patch(cls, id, text_z): + key_name = 'patch:%s' % id + return cls.get_or_insert(key_name, + text_z = db.Blob(text_z), + depth = 0, + base = None) + + @classmethod + def create_content(cls, id, text_z, base = None): + """Create (or lookup and return an existing) content instance. + + Arguments: + id: + Git blob SHA-1 hash of the fully inflated content. + text_z: + If base is None this is the deflated content whose hash + is id. + + If base is supplied this is a patch which when applied to + base yields the content whose hash is id. + base: + The base content if text_z is a patch. + """ + key_name = 'content:%s' % id + r = cls.get_by_key_name(key_name) + if r: + return r + + if base is None: + return cls.get_or_insert(key_name, + text_z = db.Blob(text_z), + depth = 0, + base = None) + + my_text = _apply_patch(base.data_lines, + id, + zlib.decompress(text_z).splitlines(True)) + cmp_id = _blob_hash(my_text) + if id != cmp_id: + raise DeltaPatchingException() + + if base.depth < MAX_DELTA_DEPTH: + return cls.get_or_insert(key_name, + text_z = db.Blob(text_z), + depth = base.depth + 1, + base = base) + return cls.get_or_insert(key_name, + text_z = db.Blob(zlib.compress(my_text)), + depth = 0, + base = None) + + @property + def is_patch(self): + return self._base or self.key().name().startswith('patch:') + + @property + def is_data(self): + return self.key().name().startswith('content:') + + @property + def data_text(self): + if self._data_text is None: + if self._base: + base = CachedDeltaContent.get(self._base) + raw = _apply_patch(base.data_lines, + self.key().name(), + self.patch_lines) + else: + raw = zlib.decompress(self.text_z) + self._data_text = raw + return self._data_text + + @property + def data_lines(self): + if self._data_lines is None: + self._data_lines = self.data_text.splitlines(True) + return self._data_lines + + @property + def patch_text(self): + if not self.is_patch: + return None + if self._patch_text is None: + self._patch_text = zlib.decompress(self.text_z) + return self._patch_text + + @property + def patch_lines(self): + if not self.is_patch: + return None + if self._patch_lines is None: + self._patch_lines = self.patch_text.splitlines(True) + return self._patch_lines + + +class Patch(BackedUpModel): + """A single patch, i.e. a set of changes to a single file. + + This is a descendant of a PatchSet. + """ + + patchset = db.ReferenceProperty(PatchSet, required=True) # == parent + filename = db.StringProperty(required=True) + status = db.StringProperty(required=True) # 'A', 'M', 'D' + n_comments = db.IntegerProperty() + + old_data = db.ReferenceProperty(DeltaContent, collection_name='olddata_set') + new_data = db.ReferenceProperty(DeltaContent, collection_name='newdata_set') + diff_data = db.ReferenceProperty(DeltaContent, collection_name='diffdata_set') + + @classmethod + def get_or_insert_patch(cls, patchset, filename, **kw): + """Get or insert the patch for a specific file path. + + This method runs in an independent transaction. + """ + m = hashlib.sha1() + m.update(filename) + key = 'z%s' % m.hexdigest() + return cls.get_or_insert(key, + parent = patchset, + patchset = patchset, + filename = filename, + **kw) + + @classmethod + def get_patch(cls, parent, id_str): + if id_str.startswith('z'): + return cls.get_by_key_name(id_str, parent=parent); + else: + return cls.get_by_id(int(id_str), parent=parent); + + @property + def id(self): + return str(self.key().id_or_name()) + + @property + def num_comments(self): + """The number of non-draft comments for this patch. + """ + return self.n_comments + + def update_comment_count(self, n): + """Increment the n_comments property by n. + """ + self.n_comments += n + + _num_drafts = None + + @property + def num_drafts(self): + """The number of draft comments on this patch for the current user. + + The value is expensive to compute, so it is cached. + """ + if self._num_drafts is None: + user = Account.current_user_account + if user is None: + self._num_drafts = 0 + else: + query = gql(Comment, + 'WHERE patch = :1 AND draft = TRUE AND author = :2', + self, user.user) + self._num_drafts = query.count() + return self._num_drafts + + def _data(self, name): + prop = '_%s_CachedDeltaContent' % name + try: + c = getattr(self, prop) + except AttributeError: + # XXX Using internal knowledge about db package: + # Key for ReferenceProperty 'foo' is '_foo'. + + data_key = getattr(self, '_%s_data' % name, None) + if data_key: + c = CachedDeltaContent.get(data_key) + if data_key in ('diff', 'new') \ + and self._diff_data == self._new_data: + self._diff_CachedDeltaContent = c + self._new_CachedDeltaContent = c + else: + setattr(self, prop, c) + else: + c = None + setattr(self, prop, c) + return c + + @property + def patch_text(self): + """Get the patch converting old_text to new_text. + """ + return self._data('diff').patch_text + + @property + def patch_lines(self): + """The patch_text split into lines, retaining line endings. + """ + return self._data('diff').patch_lines + + @property + def old_text(self): + """Original version of the file text. + """ + d = self._data('old') + if d: + return d.data_text + return '' + + @property + def old_lines(self): + """The old_text split into lines, retaining line endings. + """ + d = self._data('old') + if d: + return d.data_lines + return [] + + @property + def new_text(self): + """Get self.new_content + """ + d = self._data('new') + if d: + return d.data_text + return '' + + @property + def new_lines(self): + """The new_text split into lines, retaining line endings. + """ + d = self._data('new') + if d: + return d.data_lines + return [] + + +class Comment(BackedUpModel): + """A Comment for a specific line of a specific file. + + This is a descendant of a Patch. + """ + + patch = db.ReferenceProperty(Patch) # == parent + message_id = db.StringProperty() # == key_name + author = db.UserProperty() + date = db.DateTimeProperty(auto_now=True) + lineno = db.IntegerProperty() + text = db.TextProperty() + left = db.BooleanProperty() + draft = db.BooleanProperty(required=True, default=True) + + def complete(self, patch): + """Set the shorttext and buckets attributes.""" + # TODO(guido): Turn these into caching proprties instead. + # TODO(guido): Properly parse the text into quoted and unquoted buckets. + self.shorttext = self.text.lstrip()[:50].rstrip() + self.buckets = [Bucket(text=self.text)] + + +class Bucket(BackedUpModel): + """A 'Bucket' of text. + + A comment may consist of multiple text buckets, some of which may be + collapsed by default (when they represent quoted text). + + NOTE: This entity is never written to the database. See Comment.complete(). + """ + # TODO(guido): Flesh this out. + + text = db.TextProperty() + + +class ReviewStatus(BackedUpModel): + """The information for whether a user has LGTMed or verified a change.""" + change = db.ReferenceProperty(Change, required=True) # == parent + user = db.UserProperty(required=True) + lgtm = db.StringProperty() + verified = db.BooleanProperty() + + @classmethod + def get_or_insert_status(cls, change, user): + key = '<%s>' % user.email + return cls.get_or_insert(key, + change=change, + user=user, + parent=change) + + @classmethod + def get_status_for_user(cls, change, user): + key = '<%s>' % user.email + return cls.get_by_key_name(key, parent=change) + + @classmethod + def all_for_change(cls, change): + return gql(ReviewStatus, + 'WHERE ANCESTOR IS :1', + change).fetch(FETCH_MAX) + + +### Contributor License Agreements ### + +class IndividualCLA: + NONE = 0 + + +### Accounts ### + + +class Account(BackedUpModel): + """Maps a user or email address to a user-selected real_name, and more. + + Nicknames do not have to be unique. + + The default real_name is generated from the email address by + stripping the first '@' sign and everything after it. The email + should not be empty nor should it start with '@' (AssertionError + error is raised if either of these happens). + + Holds a list of ids of starred changes. The expectation + that you won't have more than a dozen or so starred changes (a few + hundred in extreme cases) and the memory used up by a list of + integers of that size is very modest, so this is an efficient + solution. (If someone found a use case for having thousands of + starred changes we'd have to think of a different approach.) + + Returns whether a user is authorized to do lgtm or verify. + For now, these authorization check methods do not test which repository + the change is in. This will change. + """ + + user = db.UserProperty(required=True) + email = db.EmailProperty(required=True) # key == <email> + preferred_email = db.EmailProperty() + + created = db.DateTimeProperty(auto_now_add=True) + modified = db.DateTimeProperty(auto_now=True) + + welcomed = db.BooleanProperty(default=False) + real_name_entered = db.BooleanProperty(default=False) + real_name = db.StringProperty() + mailing_address = db.TextProperty() + mailing_address_country = db.StringProperty() + phone_number = db.StringProperty() + fax_number = db.StringProperty() + + cla_verified = db.BooleanProperty(default=False) + cla_verified_by = db.UserProperty() + cla_verified_timestamp = db.DateTimeProperty() # the first time it's set + individual_cla_version = db.IntegerProperty(default=IndividualCLA.NONE) + individual_cla_timestamp = db.DateTimeProperty() + cla_comments = db.TextProperty() + + default_context = db.IntegerProperty(default=DEFAULT_CONTEXT, + choices=CONTEXT_CHOICES) + stars = db.ListProperty(int) # Change ids of all starred changes + unclaimed_changes_projects = db.ListProperty(db.Key) + + # Current user's Account. Updated by middleware.AddUserToRequestMiddleware. + current_user_account = None + + def get_email(self): + "Gets the email that this person wants us to use -- separate from login." + if self.preferred_email: + return self.preferred_email + else: + return self.email + + def get_email_formatted(self): + return '"%s" <%s>' % (self.real_name, self.get_email()) + + @classmethod + def get_account_for_user(cls, user): + """Get the Account for a user, creating a default one if needed.""" + email = user.email() + assert email + key = '<%s>' % email + # Since usually the account already exists, first try getting it + # without the transaction implied by get_or_insert(). + account = cls.get_by_key_name(key) + if account is not None: + return account + real_name = user.nickname() + if '@' in real_name: + real_name = real_name.split('@', 1)[0] + assert real_name + return cls.get_or_insert(key, user=user, email=email, real_name=real_name) + + @classmethod + def get_account_for_email(cls, email): + """Get the Account for an email address, or return None.""" + assert email + key = '<%s>' % email + return cls.get_by_key_name(key) + + @classmethod + def get_accounts_for_emails(cls, emails): + """Get the Accounts for all email address. + """ + return cls.get_by_key_name(map(lambda x: '<%s>' % x, emails)) + + @classmethod + def get_real_name_for_email(cls, email): + """Get the real_name for an email address, possibly a default.""" + account = cls.get_account_for_email(email) + if account is not None and account.real_name: + return account.real_name + real_name = email + if '@' in real_name: + real_name = real_name.split('@', 1)[0] + assert real_name + return real_name + + @classmethod + def get_accounts_for_real_name(cls, real_name): + """Get the list of Accounts that have this real_name.""" + assert real_name + assert '@' not in real_name + return list(gql(cls, 'WHERE real_name = :1', real_name)) + + @classmethod + def get_email_for_real_name(cls, real_name): + """Turn a real_name into an email address. + + If the real_name is not unique or does not exist, this returns None. + """ + accounts = cls.get_accounts_for_real_name(real_name) + if len(accounts) != 1: + return None + return accounts[0].email + + _drafts = None + + @property + def drafts(self): + """A list of change ids that have drafts by this user. + + This is cached in memcache. + """ + if self._drafts is None: + if self._initialize_drafts(): + self._save_drafts() + return self._drafts + + def update_drafts(self, change, have_drafts=None): + """Update the user's draft status for this change. + + Args: + change: an Change instance. + have_drafts: optional bool forcing the draft status. By default, + change.num_drafts is inspected (which may query the datastore). + + The Account is written to the datastore if necessary. + """ + dirty = False + if self._drafts is None: + dirty = self._initialize_drafts() + id = change.key().id() + if have_drafts is None: + have_drafts = bool(change.num_drafts) # Beware, this may do a query. + if have_drafts: + if id not in self._drafts: + self._drafts.append(id) + dirty = True + else: + if id in self._drafts: + self._drafts.remove(id) + dirty = True + if dirty: + self._save_drafts() + + def _initialize_drafts(self): + """Initialize self._drafts from scratch. + + This mostly exists as a schema conversion utility. + + Returns: + True if the user should call self._save_drafts(), False if not. + """ + drafts = memcache.get('user_drafts:' + self.email) + if drafts is not None: + self._drafts = drafts + return False + # We're looking for the Change key id. The ancestry of comments goes: + # Change -> PatchSet -> Patch -> Comment. + change_ids = set(comment.key().parent().parent().parent().id() + for comment in gql(Comment, + 'WHERE author = :1 AND draft = TRUE', + self.user)) + self._drafts = list(change_ids) + return True + + def _save_drafts(self): + """Save self._drafts to memcache.""" + memcache.set('user_drafts:' + self.email, self._drafts, 3600) + + def can_lgtm(self): + """Returns whether the user can lgtm a given change. + + For now returns true and doesn't take the change to check. + """ + return True + + def can_verify(self): + """Returns whether the user can verify a given change. + + For now returns true and doesn't take the change to check. + """ + return True + + @classmethod + def get_all_accounts(cls): + """Return all accounts""" + all = cls.all() + all.order('real_name') + return list(all) + + +### Group ### + +AUTO_GROUPS = ['admin', 'submitters'] + +class AccountGroup(BackedUpModel): + """A set of users. Permissions are assigned to groups. + + There are some groups that can't be deleted -- like admin and all + """ + + name = db.StringProperty(required=True) + comment = db.TextProperty(required=False) + members = db.ListProperty(users.User) + + @classmethod + def get_all_groups(cls): + """Return all groups""" + all = cls.all() + all.order('name') + return list(all) + + @property + def is_auto_group(self): + """These groups can't be deleted.""" + return self.name in AUTO_GROUPS + + @classmethod + def create_groups(cls): + for group_name in AUTO_GROUPS: + def trans(): + g = cls(name=group_name, comment=( + 'Auto created %s group' % group_name)) + g.put() + q = cls.gql('WHERE name=:name', name=group_name) + if q.get() is None: + db.run_in_transaction(trans) + + @classmethod + def get_group_for_name(cls, name): + return cls.gql('WHERE name=:name', name=name).get() + + def remove(self): + """delete this group""" + def trans(group): + group.delete() + # this will do the ON DELETE CASCADE once the users are in there + db.run_in_transaction(trans, self) + + def put(self): + if self.name in ['admin', 'submitters']: + memcache.delete('group_members:%s' % self.name) + BackedUpModel.put(self) + + @classmethod + def _is_in_cached_group(cls, user, group): + if not user: + return False + cache_key = 'group_members:%s' % group + users = memcache.get(cache_key) + if not users: + g = AccountGroup.get_group_for_name(group) + if not g: + AccountGroup.create_groups() + g = AccountGroup.get_group_for_name(group) + if len(g.members) == 0: + # if there are no users in this group, everyone is in this group + # (helps with testing, upgrading and bootstrapping) + # In prod this never really happens, so don't bother caching it + return True + users = [u.email() for u in g.members] + memcache.set(cache_key, users) + return user.email() in users + + @classmethod + def is_user_admin(cls, user): + return AccountGroup._is_in_cached_group(user, 'admin') + + @classmethod + def is_user_submitter(cls, user): + return AccountGroup._is_in_cached_group(user, 'submitters') |