diff options
Diffstat (limited to 'webapp/django/db')
55 files changed, 11324 insertions, 0 deletions
diff --git a/webapp/django/db/__init__.py b/webapp/django/db/__init__.py new file mode 100644 index 0000000000..d15fd3238e --- /dev/null +++ b/webapp/django/db/__init__.py @@ -0,0 +1,63 @@ +import os +from django.conf import settings +from django.core import signals +from django.core.exceptions import ImproperlyConfigured +from django.utils.functional import curry + +__all__ = ('backend', 'connection', 'DatabaseError', 'IntegrityError') + +if not settings.DATABASE_ENGINE: + settings.DATABASE_ENGINE = 'dummy' + +try: + # Most of the time, the database backend will be one of the official + # backends that ships with Django, so look there first. + _import_path = 'django.db.backends.' + backend = __import__('%s%s.base' % (_import_path, settings.DATABASE_ENGINE), {}, {}, ['']) +except ImportError, e: + # If the import failed, we might be looking for a database backend + # distributed external to Django. So we'll try that next. + try: + _import_path = '' + backend = __import__('%s.base' % settings.DATABASE_ENGINE, {}, {}, ['']) + except ImportError, e_user: + # The database backend wasn't found. Display a helpful error message + # listing all possible (built-in) database backends. + backend_dir = os.path.join(__path__[0], 'backends') + try: + available_backends = [f for f in os.listdir(backend_dir) if not f.startswith('_') and not f.startswith('.') and not f.endswith('.py') and not f.endswith('.pyc')] + except EnvironmentError: + available_backends = [] + available_backends.sort() + if settings.DATABASE_ENGINE not in available_backends: + raise ImproperlyConfigured, "%r isn't an available database backend. Available options are: %s\nError was: %s" % \ + (settings.DATABASE_ENGINE, ", ".join(map(repr, available_backends)), e_user) + else: + raise # If there's some other error, this must be an error in Django itself. + +# Convenient aliases for backend bits. +connection = backend.DatabaseWrapper(**settings.DATABASE_OPTIONS) +DatabaseError = backend.DatabaseError +IntegrityError = backend.IntegrityError + +# Register an event that closes the database connection +# when a Django request is finished. +def close_connection(**kwargs): + connection.close() +signals.request_finished.connect(close_connection) + +# Register an event that resets connection.queries +# when a Django request is started. +def reset_queries(**kwargs): + connection.queries = [] +signals.request_started.connect(reset_queries) + +# Register an event that rolls back the connection +# when a Django request has an exception. +def _rollback_on_exception(**kwargs): + from django.db import transaction + try: + transaction.rollback_unless_managed() + except DatabaseError: + pass +signals.got_request_exception.connect(_rollback_on_exception) diff --git a/webapp/django/db/backends/__init__.py b/webapp/django/db/backends/__init__.py new file mode 100644 index 0000000000..074fa0ed70 --- /dev/null +++ b/webapp/django/db/backends/__init__.py @@ -0,0 +1,448 @@ +try: + # Only exists in Python 2.4+ + from threading import local +except ImportError: + # Import copy of _thread_local.py from Python 2.4 + from django.utils._threading_local import local +try: + set +except NameError: + # Python 2.3 compat + from sets import Set as set + +from django.db.backends import util +from django.utils import datetime_safe + +class BaseDatabaseWrapper(local): + """ + Represents a database connection. + """ + ops = None + def __init__(self, **kwargs): + self.connection = None + self.queries = [] + self.options = kwargs + + def _commit(self): + if self.connection is not None: + return self.connection.commit() + + def _rollback(self): + if self.connection is not None: + return self.connection.rollback() + + def _savepoint(self, sid): + if not self.features.uses_savepoints: + return + self.connection.cursor().execute(self.ops.savepoint_create_sql(sid)) + + def _savepoint_rollback(self, sid): + if not self.features.uses_savepoints: + return + self.connection.cursor().execute(self.ops.savepoint_rollback_sql(sid)) + + def _savepoint_commit(self, sid): + if not self.features.uses_savepoints: + return + self.connection.cursor().execute(self.ops.savepoint_commit_sql(sid)) + + def close(self): + if self.connection is not None: + self.connection.close() + self.connection = None + + def cursor(self): + from django.conf import settings + cursor = self._cursor(settings) + if settings.DEBUG: + return self.make_debug_cursor(cursor) + return cursor + + def make_debug_cursor(self, cursor): + return util.CursorDebugWrapper(cursor, self) + +class BaseDatabaseFeatures(object): + # True if django.db.backend.utils.typecast_timestamp is used on values + # returned from dates() calls. + needs_datetime_string_cast = True + uses_custom_query_class = False + empty_fetchmany_value = [] + update_can_self_select = True + interprets_empty_strings_as_nulls = False + can_use_chunked_reads = True + uses_savepoints = False + +class BaseDatabaseOperations(object): + """ + This class encapsulates all backend-specific differences, such as the way + a backend performs ordering or calculates the ID of a recently-inserted + row. + """ + def autoinc_sql(self, table, column): + """ + Returns any SQL needed to support auto-incrementing primary keys, or + None if no SQL is necessary. + + This SQL is executed when a table is created. + """ + return None + + def date_extract_sql(self, lookup_type, field_name): + """ + Given a lookup_type of 'year', 'month' or 'day', returns the SQL that + extracts a value from the given date field field_name. + """ + raise NotImplementedError() + + def date_trunc_sql(self, lookup_type, field_name): + """ + Given a lookup_type of 'year', 'month' or 'day', returns the SQL that + truncates the given date field field_name to a DATE object with only + the given specificity. + """ + raise NotImplementedError() + + def datetime_cast_sql(self): + """ + Returns the SQL necessary to cast a datetime value so that it will be + retrieved as a Python datetime object instead of a string. + + This SQL should include a '%s' in place of the field's name. + """ + return "%s" + + def deferrable_sql(self): + """ + Returns the SQL necessary to make a constraint "initially deferred" + during a CREATE TABLE statement. + """ + return '' + + def drop_foreignkey_sql(self): + """ + Returns the SQL command that drops a foreign key. + """ + return "DROP CONSTRAINT" + + def drop_sequence_sql(self, table): + """ + Returns any SQL necessary to drop the sequence for the given table. + Returns None if no SQL is necessary. + """ + return None + + def field_cast_sql(self, db_type): + """ + Given a column type (e.g. 'BLOB', 'VARCHAR'), returns the SQL necessary + to cast it before using it in a WHERE statement. Note that the + resulting string should contain a '%s' placeholder for the column being + searched against. + """ + return '%s' + + def fulltext_search_sql(self, field_name): + """ + Returns the SQL WHERE clause to use in order to perform a full-text + search of the given field_name. Note that the resulting string should + contain a '%s' placeholder for the value being searched against. + """ + raise NotImplementedError('Full-text search is not implemented for this database backend') + + def last_executed_query(self, cursor, sql, params): + """ + Returns a string of the query last executed by the given cursor, with + placeholders replaced with actual values. + + `sql` is the raw query containing placeholders, and `params` is the + sequence of parameters. These are used by default, but this method + exists for database backends to provide a better implementation + according to their own quoting schemes. + """ + from django.utils.encoding import smart_unicode, force_unicode + + # Convert params to contain Unicode values. + to_unicode = lambda s: force_unicode(s, strings_only=True) + if isinstance(params, (list, tuple)): + u_params = tuple([to_unicode(val) for val in params]) + else: + u_params = dict([(to_unicode(k), to_unicode(v)) for k, v in params.items()]) + + return smart_unicode(sql) % u_params + + def last_insert_id(self, cursor, table_name, pk_name): + """ + Given a cursor object that has just performed an INSERT statement into + a table that has an auto-incrementing ID, returns the newly created ID. + + This method also receives the table name and the name of the primary-key + column. + """ + return cursor.lastrowid + + def lookup_cast(self, lookup_type): + """ + Returns the string to use in a query when performing lookups + ("contains", "like", etc). The resulting string should contain a '%s' + placeholder for the column being searched against. + """ + return "%s" + + def max_name_length(self): + """ + Returns the maximum length of table and column names, or None if there + is no limit. + """ + return None + + def no_limit_value(self): + """ + Returns the value to use for the LIMIT when we are wanting "LIMIT + infinity". Returns None if the limit clause can be omitted in this case. + """ + # FIXME: API may need to change once Oracle backend is repaired. + raise NotImplementedError() + + def pk_default_value(self): + """ + Returns the value to use during an INSERT statement to specify that + the field should use its default value. + """ + return 'DEFAULT' + + def query_class(self, DefaultQueryClass): + """ + Given the default Query class, returns a custom Query class + to use for this backend. Returns None if a custom Query isn't used. + See also BaseDatabaseFeatures.uses_custom_query_class, which regulates + whether this method is called at all. + """ + return None + + def quote_name(self, name): + """ + Returns a quoted version of the given table, index or column name. Does + not quote the given name if it's already been quoted. + """ + raise NotImplementedError() + + def random_function_sql(self): + """ + Returns a SQL expression that returns a random value. + """ + return 'RANDOM()' + + def regex_lookup(self, lookup_type): + """ + Returns the string to use in a query when performing regular expression + lookups (using "regex" or "iregex"). The resulting string should + contain a '%s' placeholder for the column being searched against. + + If the feature is not supported (or part of it is not supported), a + NotImplementedError exception can be raised. + """ + raise NotImplementedError + + def savepoint_create_sql(self, sid): + """ + Returns the SQL for starting a new savepoint. Only required if the + "uses_savepoints" feature is True. The "sid" parameter is a string + for the savepoint id. + """ + raise NotImplementedError + + def savepoint_commit_sql(self, sid): + """ + Returns the SQL for committing the given savepoint. + """ + raise NotImplementedError + + def savepoint_rollback_sql(self, sid): + """ + Returns the SQL for rolling back the given savepoint. + """ + raise NotImplementedError + + def sql_flush(self, style, tables, sequences): + """ + Returns a list of SQL statements required to remove all data from + the given database tables (without actually removing the tables + themselves). + + The `style` argument is a Style object as returned by either + color_style() or no_style() in django.core.management.color. + """ + raise NotImplementedError() + + def sequence_reset_sql(self, style, model_list): + """ + Returns a list of the SQL statements required to reset sequences for + the given models. + + The `style` argument is a Style object as returned by either + color_style() or no_style() in django.core.management.color. + """ + return [] # No sequence reset required by default. + + def start_transaction_sql(self): + """ + Returns the SQL statement required to start a transaction. + """ + return "BEGIN;" + + def sql_for_tablespace(self, tablespace, inline=False): + """ + Returns the SQL that will be appended to tables or rows to define + a tablespace. Returns '' if the backend doesn't use tablespaces. + """ + return '' + + def prep_for_like_query(self, x): + """Prepares a value for use in a LIKE query.""" + from django.utils.encoding import smart_unicode + return smart_unicode(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_") + + def value_to_db_date(self, value): + """ + Transform a date value to an object compatible with what is expected + by the backend driver for date columns. + """ + if value is None: + return None + return datetime_safe.new_date(value).strftime('%Y-%m-%d') + + def value_to_db_datetime(self, value): + """ + Transform a datetime value to an object compatible with what is expected + by the backend driver for datetime columns. + """ + if value is None: + return None + return unicode(value) + + def value_to_db_time(self, value): + """ + Transform a datetime value to an object compatible with what is expected + by the backend driver for time columns. + """ + if value is None: + return None + return unicode(value) + + def value_to_db_decimal(self, value, max_digits, decimal_places): + """ + Transform a decimal.Decimal value to an object compatible with what is + expected by the backend driver for decimal (numeric) columns. + """ + if value is None: + return None + return util.format_number(value, max_digits, decimal_places) + + def year_lookup_bounds(self, value): + """ + Returns a two-elements list with the lower and upper bound to be used + with a BETWEEN operator to query a field value using a year lookup + + `value` is an int, containing the looked-up year. + """ + first = '%s-01-01 00:00:00' + second = '%s-12-31 23:59:59.999999' + return [first % value, second % value] + + def year_lookup_bounds_for_date_field(self, value): + """ + Returns a two-elements list with the lower and upper bound to be used + with a BETWEEN operator to query a DateField value using a year lookup + + `value` is an int, containing the looked-up year. + + By default, it just calls `self.year_lookup_bounds`. Some backends need + this hook because on their DB date fields can't be compared to values + which include a time part. + """ + return self.year_lookup_bounds(value) + +class BaseDatabaseIntrospection(object): + """ + This class encapsulates all backend-specific introspection utilities + """ + data_types_reverse = {} + + def __init__(self, connection): + self.connection = connection + + def table_name_converter(self, name): + """Apply a conversion to the name for the purposes of comparison. + + The default table name converter is for case sensitive comparison. + """ + return name + + def table_names(self): + "Returns a list of names of all tables that exist in the database." + cursor = self.connection.cursor() + return self.get_table_list(cursor) + + def django_table_names(self, only_existing=False): + """ + Returns a list of all table names that have associated Django models and + are in INSTALLED_APPS. + + If only_existing is True, the resulting list will only include the tables + that actually exist in the database. + """ + from django.db import models + tables = set() + for app in models.get_apps(): + for model in models.get_models(app): + tables.add(model._meta.db_table) + tables.update([f.m2m_db_table() for f in model._meta.local_many_to_many]) + if only_existing: + tables = [t for t in tables if t in self.table_names()] + return tables + + def installed_models(self, tables): + "Returns a set of all models represented by the provided list of table names." + from django.db import models + all_models = [] + for app in models.get_apps(): + for model in models.get_models(app): + all_models.append(model) + return set([m for m in all_models + if self.table_name_converter(m._meta.db_table) in map(self.table_name_converter, tables) + ]) + + def sequence_list(self): + "Returns a list of information about all DB sequences for all models in all apps." + from django.db import models + + apps = models.get_apps() + sequence_list = [] + + for app in apps: + for model in models.get_models(app): + for f in model._meta.local_fields: + if isinstance(f, models.AutoField): + sequence_list.append({'table': model._meta.db_table, 'column': f.column}) + break # Only one AutoField is allowed per model, so don't bother continuing. + + for f in model._meta.local_many_to_many: + sequence_list.append({'table': f.m2m_db_table(), 'column': None}) + + return sequence_list + +class BaseDatabaseClient(object): + """ + This class encapsualtes all backend-specific methods for opening a + client shell + """ + def runshell(self): + raise NotImplementedError() + +class BaseDatabaseValidation(object): + """ + This class encapsualtes all backend-specific model validation. + """ + def validate_field(self, errors, opts, f): + "By default, there is no backend-specific validation" + pass + diff --git a/webapp/django/db/backends/creation.py b/webapp/django/db/backends/creation.py new file mode 100644 index 0000000000..e7460357c8 --- /dev/null +++ b/webapp/django/db/backends/creation.py @@ -0,0 +1,404 @@ +import sys +import time +try: + set +except NameError: + # Python 2.3 compat + from sets import Set as set + +from django.conf import settings +from django.core.management import call_command + +# The prefix to put on the default database name when creating +# the test database. +TEST_DATABASE_PREFIX = 'test_' + +class BaseDatabaseCreation(object): + """ + This class encapsulates all backend-specific differences that pertain to + database *creation*, such as the column types to use for particular Django + Fields, the SQL used to create and destroy tables, and the creation and + destruction of test databases. + """ + data_types = {} + + def __init__(self, connection): + self.connection = connection + + def sql_create_model(self, model, style, known_models=set()): + """ + Returns the SQL required to create a single model, as a tuple of: + (list_of_sql, pending_references_dict) + """ + from django.db import models + + opts = model._meta + final_output = [] + table_output = [] + pending_references = {} + qn = self.connection.ops.quote_name + for f in opts.local_fields: + col_type = f.db_type() + tablespace = f.db_tablespace or opts.db_tablespace + if col_type is None: + # Skip ManyToManyFields, because they're not represented as + # database columns in this table. + continue + # Make the definition (e.g. 'foo VARCHAR(30)') for this field. + field_output = [style.SQL_FIELD(qn(f.column)), + style.SQL_COLTYPE(col_type)] + field_output.append(style.SQL_KEYWORD('%sNULL' % (not f.null and 'NOT ' or ''))) + if f.primary_key: + field_output.append(style.SQL_KEYWORD('PRIMARY KEY')) + elif f.unique: + field_output.append(style.SQL_KEYWORD('UNIQUE')) + if tablespace and f.unique: + # We must specify the index tablespace inline, because we + # won't be generating a CREATE INDEX statement for this field. + field_output.append(self.connection.ops.tablespace_sql(tablespace, inline=True)) + if f.rel: + ref_output, pending = self.sql_for_inline_foreign_key_references(f, known_models, style) + if pending: + pr = pending_references.setdefault(f.rel.to, []).append((model, f)) + else: + field_output.extend(ref_output) + table_output.append(' '.join(field_output)) + if opts.order_with_respect_to: + table_output.append(style.SQL_FIELD(qn('_order')) + ' ' + \ + style.SQL_COLTYPE(models.IntegerField().db_type()) + ' ' + \ + style.SQL_KEYWORD('NULL')) + for field_constraints in opts.unique_together: + table_output.append(style.SQL_KEYWORD('UNIQUE') + ' (%s)' % \ + ", ".join([style.SQL_FIELD(qn(opts.get_field(f).column)) for f in field_constraints])) + + full_statement = [style.SQL_KEYWORD('CREATE TABLE') + ' ' + style.SQL_TABLE(qn(opts.db_table)) + ' ('] + for i, line in enumerate(table_output): # Combine and add commas. + full_statement.append(' %s%s' % (line, i < len(table_output)-1 and ',' or '')) + full_statement.append(')') + if opts.db_tablespace: + full_statement.append(self.connection.ops.tablespace_sql(opts.db_tablespace)) + full_statement.append(';') + final_output.append('\n'.join(full_statement)) + + if opts.has_auto_field: + # Add any extra SQL needed to support auto-incrementing primary keys. + auto_column = opts.auto_field.db_column or opts.auto_field.name + autoinc_sql = self.connection.ops.autoinc_sql(opts.db_table, auto_column) + if autoinc_sql: + for stmt in autoinc_sql: + final_output.append(stmt) + + return final_output, pending_references + + def sql_for_inline_foreign_key_references(self, field, known_models, style): + "Return the SQL snippet defining the foreign key reference for a field" + qn = self.connection.ops.quote_name + if field.rel.to in known_models: + output = [style.SQL_KEYWORD('REFERENCES') + ' ' + \ + style.SQL_TABLE(qn(field.rel.to._meta.db_table)) + ' (' + \ + style.SQL_FIELD(qn(field.rel.to._meta.get_field(field.rel.field_name).column)) + ')' + + self.connection.ops.deferrable_sql() + ] + pending = False + else: + # We haven't yet created the table to which this field + # is related, so save it for later. + output = [] + pending = True + + return output, pending + + def sql_for_pending_references(self, model, style, pending_references): + "Returns any ALTER TABLE statements to add constraints after the fact." + from django.db.backends.util import truncate_name + + qn = self.connection.ops.quote_name + final_output = [] + opts = model._meta + if model in pending_references: + for rel_class, f in pending_references[model]: + rel_opts = rel_class._meta + r_table = rel_opts.db_table + r_col = f.column + table = opts.db_table + col = opts.get_field(f.rel.field_name).column + # For MySQL, r_name must be unique in the first 64 characters. + # So we are careful with character usage here. + r_name = '%s_refs_%s_%x' % (r_col, col, abs(hash((r_table, table)))) + final_output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' % \ + (qn(r_table), truncate_name(r_name, self.connection.ops.max_name_length()), + qn(r_col), qn(table), qn(col), + self.connection.ops.deferrable_sql())) + del pending_references[model] + return final_output + + def sql_for_many_to_many(self, model, style): + "Return the CREATE TABLE statments for all the many-to-many tables defined on a model" + output = [] + for f in model._meta.local_many_to_many: + output.extend(self.sql_for_many_to_many_field(model, f, style)) + return output + + def sql_for_many_to_many_field(self, model, f, style): + "Return the CREATE TABLE statements for a single m2m field" + from django.db import models + from django.db.backends.util import truncate_name + + output = [] + if f.creates_table: + opts = model._meta + qn = self.connection.ops.quote_name + tablespace = f.db_tablespace or opts.db_tablespace + if tablespace: + sql = self.connection.ops.tablespace_sql(tablespace, inline=True) + if sql: + tablespace_sql = ' ' + sql + else: + tablespace_sql = '' + else: + tablespace_sql = '' + table_output = [style.SQL_KEYWORD('CREATE TABLE') + ' ' + \ + style.SQL_TABLE(qn(f.m2m_db_table())) + ' ('] + table_output.append(' %s %s %s%s,' % + (style.SQL_FIELD(qn('id')), + style.SQL_COLTYPE(models.AutoField(primary_key=True).db_type()), + style.SQL_KEYWORD('NOT NULL PRIMARY KEY'), + tablespace_sql)) + + deferred = [] + inline_output, deferred = self.sql_for_inline_many_to_many_references(model, f, style) + table_output.extend(inline_output) + + table_output.append(' %s (%s, %s)%s' % + (style.SQL_KEYWORD('UNIQUE'), + style.SQL_FIELD(qn(f.m2m_column_name())), + style.SQL_FIELD(qn(f.m2m_reverse_name())), + tablespace_sql)) + table_output.append(')') + if opts.db_tablespace: + # f.db_tablespace is only for indices, so ignore its value here. + table_output.append(self.connection.ops.tablespace_sql(opts.db_tablespace)) + table_output.append(';') + output.append('\n'.join(table_output)) + + for r_table, r_col, table, col in deferred: + r_name = '%s_refs_%s_%x' % (r_col, col, + abs(hash((r_table, table)))) + output.append(style.SQL_KEYWORD('ALTER TABLE') + ' %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s)%s;' % + (qn(r_table), + truncate_name(r_name, self.connection.ops.max_name_length()), + qn(r_col), qn(table), qn(col), + self.connection.ops.deferrable_sql())) + + # Add any extra SQL needed to support auto-incrementing PKs + autoinc_sql = self.connection.ops.autoinc_sql(f.m2m_db_table(), 'id') + if autoinc_sql: + for stmt in autoinc_sql: + output.append(stmt) + return output + + def sql_for_inline_many_to_many_references(self, model, field, style): + "Create the references to other tables required by a many-to-many table" + from django.db import models + opts = model._meta + qn = self.connection.ops.quote_name + + table_output = [ + ' %s %s %s %s (%s)%s,' % + (style.SQL_FIELD(qn(field.m2m_column_name())), + style.SQL_COLTYPE(models.ForeignKey(model).db_type()), + style.SQL_KEYWORD('NOT NULL REFERENCES'), + style.SQL_TABLE(qn(opts.db_table)), + style.SQL_FIELD(qn(opts.pk.column)), + self.connection.ops.deferrable_sql()), + ' %s %s %s %s (%s)%s,' % + (style.SQL_FIELD(qn(field.m2m_reverse_name())), + style.SQL_COLTYPE(models.ForeignKey(field.rel.to).db_type()), + style.SQL_KEYWORD('NOT NULL REFERENCES'), + style.SQL_TABLE(qn(field.rel.to._meta.db_table)), + style.SQL_FIELD(qn(field.rel.to._meta.pk.column)), + self.connection.ops.deferrable_sql()) + ] + deferred = [] + + return table_output, deferred + + def sql_indexes_for_model(self, model, style): + "Returns the CREATE INDEX SQL statements for a single model" + output = [] + for f in model._meta.local_fields: + output.extend(self.sql_indexes_for_field(model, f, style)) + return output + + def sql_indexes_for_field(self, model, f, style): + "Return the CREATE INDEX SQL statements for a single model field" + if f.db_index and not f.unique: + qn = self.connection.ops.quote_name + tablespace = f.db_tablespace or model._meta.db_tablespace + if tablespace: + sql = self.connection.ops.tablespace_sql(tablespace) + if sql: + tablespace_sql = ' ' + sql + else: + tablespace_sql = '' + else: + tablespace_sql = '' + output = [style.SQL_KEYWORD('CREATE INDEX') + ' ' + + style.SQL_TABLE(qn('%s_%s' % (model._meta.db_table, f.column))) + ' ' + + style.SQL_KEYWORD('ON') + ' ' + + style.SQL_TABLE(qn(model._meta.db_table)) + ' ' + + "(%s)" % style.SQL_FIELD(qn(f.column)) + + "%s;" % tablespace_sql] + else: + output = [] + return output + + def sql_destroy_model(self, model, references_to_delete, style): + "Return the DROP TABLE and restraint dropping statements for a single model" + # Drop the table now + qn = self.connection.ops.quote_name + output = ['%s %s;' % (style.SQL_KEYWORD('DROP TABLE'), + style.SQL_TABLE(qn(model._meta.db_table)))] + if model in references_to_delete: + output.extend(self.sql_remove_table_constraints(model, references_to_delete, style)) + + if model._meta.has_auto_field: + ds = self.connection.ops.drop_sequence_sql(model._meta.db_table) + if ds: + output.append(ds) + return output + + def sql_remove_table_constraints(self, model, references_to_delete, style): + from django.db.backends.util import truncate_name + + output = [] + qn = self.connection.ops.quote_name + for rel_class, f in references_to_delete[model]: + table = rel_class._meta.db_table + col = f.column + r_table = model._meta.db_table + r_col = model._meta.get_field(f.rel.field_name).column + r_name = '%s_refs_%s_%x' % (col, r_col, abs(hash((table, r_table)))) + output.append('%s %s %s %s;' % \ + (style.SQL_KEYWORD('ALTER TABLE'), + style.SQL_TABLE(qn(table)), + style.SQL_KEYWORD(self.connection.ops.drop_foreignkey_sql()), + style.SQL_FIELD(truncate_name(r_name, self.connection.ops.max_name_length())))) + del references_to_delete[model] + return output + + def sql_destroy_many_to_many(self, model, f, style): + "Returns the DROP TABLE statements for a single m2m field" + qn = self.connection.ops.quote_name + output = [] + if f.creates_table: + output.append("%s %s;" % (style.SQL_KEYWORD('DROP TABLE'), + style.SQL_TABLE(qn(f.m2m_db_table())))) + ds = self.connection.ops.drop_sequence_sql("%s_%s" % (model._meta.db_table, f.column)) + if ds: + output.append(ds) + return output + + def create_test_db(self, verbosity=1, autoclobber=False): + """ + Creates a test database, prompting the user for confirmation if the + database already exists. Returns the name of the test database created. + """ + if verbosity >= 1: + print "Creating test database..." + + test_database_name = self._create_test_db(verbosity, autoclobber) + + self.connection.close() + settings.DATABASE_NAME = test_database_name + + call_command('syncdb', verbosity=verbosity, interactive=False) + + if settings.CACHE_BACKEND.startswith('db://'): + cache_name = settings.CACHE_BACKEND[len('db://'):] + call_command('createcachetable', cache_name) + + # Get a cursor (even though we don't need one yet). This has + # the side effect of initializing the test database. + cursor = self.connection.cursor() + + return test_database_name + + def _create_test_db(self, verbosity, autoclobber): + "Internal implementation - creates the test db tables." + suffix = self.sql_table_creation_suffix() + + if settings.TEST_DATABASE_NAME: + test_database_name = settings.TEST_DATABASE_NAME + else: + test_database_name = TEST_DATABASE_PREFIX + settings.DATABASE_NAME + + qn = self.connection.ops.quote_name + + # Create the test database and connect to it. We need to autocommit + # if the database supports it because PostgreSQL doesn't allow + # CREATE/DROP DATABASE statements within transactions. + cursor = self.connection.cursor() + self.set_autocommit() + try: + cursor.execute("CREATE DATABASE %s %s" % (qn(test_database_name), suffix)) + except Exception, e: + sys.stderr.write("Got an error creating the test database: %s\n" % e) + if not autoclobber: + confirm = raw_input("Type 'yes' if you would like to try deleting the test database '%s', or 'no' to cancel: " % test_database_name) + if autoclobber or confirm == 'yes': + try: + if verbosity >= 1: + print "Destroying old test database..." + cursor.execute("DROP DATABASE %s" % qn(test_database_name)) + if verbosity >= 1: + print "Creating test database..." + cursor.execute("CREATE DATABASE %s %s" % (qn(test_database_name), suffix)) + except Exception, e: + sys.stderr.write("Got an error recreating the test database: %s\n" % e) + sys.exit(2) + else: + print "Tests cancelled." + sys.exit(1) + + return test_database_name + + def destroy_test_db(self, old_database_name, verbosity=1): + """ + Destroy a test database, prompting the user for confirmation if the + database already exists. Returns the name of the test database created. + """ + if verbosity >= 1: + print "Destroying test database..." + self.connection.close() + test_database_name = settings.DATABASE_NAME + settings.DATABASE_NAME = old_database_name + + self._destroy_test_db(test_database_name, verbosity) + + def _destroy_test_db(self, test_database_name, verbosity): + "Internal implementation - remove the test db tables." + # Remove the test database to clean up after + # ourselves. Connect to the previous database (not the test database) + # to do so, because it's not allowed to delete a database while being + # connected to it. + cursor = self.connection.cursor() + self.set_autocommit() + time.sleep(1) # To avoid "database is being accessed by other users" errors. + cursor.execute("DROP DATABASE %s" % self.connection.ops.quote_name(test_database_name)) + self.connection.close() + + def set_autocommit(self): + "Make sure a connection is in autocommit mode." + if hasattr(self.connection.connection, "autocommit"): + if callable(self.connection.connection.autocommit): + self.connection.connection.autocommit(True) + else: + self.connection.connection.autocommit = True + elif hasattr(self.connection.connection, "set_isolation_level"): + self.connection.connection.set_isolation_level(0) + + def sql_table_creation_suffix(self): + "SQL to append to the end of the test table creation statements" + return '' + diff --git a/webapp/django/db/backends/dummy/__init__.py b/webapp/django/db/backends/dummy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/webapp/django/db/backends/dummy/__init__.py diff --git a/webapp/django/db/backends/dummy/base.py b/webapp/django/db/backends/dummy/base.py new file mode 100644 index 0000000000..530ea9c519 --- /dev/null +++ b/webapp/django/db/backends/dummy/base.py @@ -0,0 +1,55 @@ +""" +Dummy database backend for Django. + +Django uses this if the DATABASE_ENGINE setting is empty (None or empty string). + +Each of these API functions, except connection.close(), raises +ImproperlyConfigured. +""" + +from django.core.exceptions import ImproperlyConfigured +from django.db.backends import * +from django.db.backends.creation import BaseDatabaseCreation + +def complain(*args, **kwargs): + raise ImproperlyConfigured, "You haven't set the DATABASE_ENGINE setting yet." + +def ignore(*args, **kwargs): + pass + +class DatabaseError(Exception): + pass + +class IntegrityError(DatabaseError): + pass + +class DatabaseOperations(BaseDatabaseOperations): + quote_name = complain + +class DatabaseClient(BaseDatabaseClient): + runshell = complain + +class DatabaseIntrospection(BaseDatabaseIntrospection): + get_table_list = complain + get_table_description = complain + get_relations = complain + get_indexes = complain + +class DatabaseWrapper(object): + operators = {} + cursor = complain + _commit = complain + _rollback = ignore + + def __init__(self, *args, **kwargs): + super(DatabaseWrapper, self).__init__(*args, **kwargs) + + self.features = BaseDatabaseFeatures() + self.ops = DatabaseOperations() + self.client = DatabaseClient() + self.creation = BaseDatabaseCreation(self) + self.introspection = DatabaseIntrospection(self) + self.validation = BaseDatabaseValidation() + + def close(self): + pass diff --git a/webapp/django/db/backends/mysql/__init__.py b/webapp/django/db/backends/mysql/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/webapp/django/db/backends/mysql/__init__.py diff --git a/webapp/django/db/backends/mysql/base.py b/webapp/django/db/backends/mysql/base.py new file mode 100644 index 0000000000..07fb700eaa --- /dev/null +++ b/webapp/django/db/backends/mysql/base.py @@ -0,0 +1,227 @@ +""" +MySQL database backend for Django. + +Requires MySQLdb: http://sourceforge.net/projects/mysql-python +""" + +import re + +try: + import MySQLdb as Database +except ImportError, e: + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("Error loading MySQLdb module: %s" % e) + +# We want version (1, 2, 1, 'final', 2) or later. We can't just use +# lexicographic ordering in this check because then (1, 2, 1, 'gamma') +# inadvertently passes the version test. +version = Database.version_info +if (version < (1,2,1) or (version[:3] == (1, 2, 1) and + (len(version) < 5 or version[3] != 'final' or version[4] < 2))): + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("MySQLdb-1.2.1p2 or newer is required; you have %s" % Database.__version__) + +from MySQLdb.converters import conversions +from MySQLdb.constants import FIELD_TYPE, FLAG + +from django.db.backends import * +from django.db.backends.mysql.client import DatabaseClient +from django.db.backends.mysql.creation import DatabaseCreation +from django.db.backends.mysql.introspection import DatabaseIntrospection +from django.db.backends.mysql.validation import DatabaseValidation + +# Raise exceptions for database warnings if DEBUG is on +from django.conf import settings +if settings.DEBUG: + from warnings import filterwarnings + filterwarnings("error", category=Database.Warning) + +DatabaseError = Database.DatabaseError +IntegrityError = Database.IntegrityError + +# MySQLdb-1.2.1 supports the Python boolean type, and only uses datetime +# module for time-related columns; older versions could have used mx.DateTime +# or strings if there were no datetime module. However, MySQLdb still returns +# TIME columns as timedelta -- they are more like timedelta in terms of actual +# behavior as they are signed and include days -- and Django expects time, so +# we still need to override that. +django_conversions = conversions.copy() +django_conversions.update({ + FIELD_TYPE.TIME: util.typecast_time, + FIELD_TYPE.DECIMAL: util.typecast_decimal, + FIELD_TYPE.NEWDECIMAL: util.typecast_decimal, +}) + +# This should match the numerical portion of the version numbers (we can treat +# versions like 5.0.24 and 5.0.24a as the same). Based on the list of version +# at http://dev.mysql.com/doc/refman/4.1/en/news.html and +# http://dev.mysql.com/doc/refman/5.0/en/news.html . +server_version_re = re.compile(r'(\d{1,2})\.(\d{1,2})\.(\d{1,2})') + +# MySQLdb-1.2.1 and newer automatically makes use of SHOW WARNINGS on +# MySQL-4.1 and newer, so the MysqlDebugWrapper is unnecessary. Since the +# point is to raise Warnings as exceptions, this can be done with the Python +# warning module, and this is setup when the connection is created, and the +# standard util.CursorDebugWrapper can be used. Also, using sql_mode +# TRADITIONAL will automatically cause most warnings to be treated as errors. + +class DatabaseFeatures(BaseDatabaseFeatures): + empty_fetchmany_value = () + update_can_self_select = False + +class DatabaseOperations(BaseDatabaseOperations): + def date_extract_sql(self, lookup_type, field_name): + # http://dev.mysql.com/doc/mysql/en/date-and-time-functions.html + return "EXTRACT(%s FROM %s)" % (lookup_type.upper(), field_name) + + def date_trunc_sql(self, lookup_type, field_name): + fields = ['year', 'month', 'day', 'hour', 'minute', 'second'] + format = ('%%Y-', '%%m', '-%%d', ' %%H:', '%%i', ':%%s') # Use double percents to escape. + format_def = ('0000-', '01', '-01', ' 00:', '00', ':00') + try: + i = fields.index(lookup_type) + 1 + except ValueError: + sql = field_name + else: + format_str = ''.join([f for f in format[:i]] + [f for f in format_def[i:]]) + sql = "CAST(DATE_FORMAT(%s, '%s') AS DATETIME)" % (field_name, format_str) + return sql + + def drop_foreignkey_sql(self): + return "DROP FOREIGN KEY" + + def fulltext_search_sql(self, field_name): + return 'MATCH (%s) AGAINST (%%s IN BOOLEAN MODE)' % field_name + + def no_limit_value(self): + # 2**64 - 1, as recommended by the MySQL documentation + return 18446744073709551615L + + def quote_name(self, name): + if name.startswith("`") and name.endswith("`"): + return name # Quoting once is enough. + return "`%s`" % name + + def random_function_sql(self): + return 'RAND()' + + def sql_flush(self, style, tables, sequences): + # NB: The generated SQL below is specific to MySQL + # 'TRUNCATE x;', 'TRUNCATE y;', 'TRUNCATE z;'... style SQL statements + # to clear all tables of all data + if tables: + sql = ['SET FOREIGN_KEY_CHECKS = 0;'] + for table in tables: + sql.append('%s %s;' % (style.SQL_KEYWORD('TRUNCATE'), style.SQL_FIELD(self.quote_name(table)))) + sql.append('SET FOREIGN_KEY_CHECKS = 1;') + + # 'ALTER TABLE table AUTO_INCREMENT = 1;'... style SQL statements + # to reset sequence indices + sql.extend(["%s %s %s %s %s;" % \ + (style.SQL_KEYWORD('ALTER'), + style.SQL_KEYWORD('TABLE'), + style.SQL_TABLE(self.quote_name(sequence['table'])), + style.SQL_KEYWORD('AUTO_INCREMENT'), + style.SQL_FIELD('= 1'), + ) for sequence in sequences]) + return sql + else: + return [] + + def value_to_db_datetime(self, value): + # MySQL doesn't support microseconds + if value is None: + return None + return unicode(value.replace(microsecond=0)) + + def value_to_db_time(self, value): + # MySQL doesn't support microseconds + if value is None: + return None + return unicode(value.replace(microsecond=0)) + + def year_lookup_bounds(self, value): + # Again, no microseconds + first = '%s-01-01 00:00:00' + second = '%s-12-31 23:59:59.99' + return [first % value, second % value] + +class DatabaseWrapper(BaseDatabaseWrapper): + + operators = { + 'exact': '= %s', + 'iexact': 'LIKE %s', + 'contains': 'LIKE BINARY %s', + 'icontains': 'LIKE %s', + 'regex': 'REGEXP BINARY %s', + 'iregex': 'REGEXP %s', + 'gt': '> %s', + 'gte': '>= %s', + 'lt': '< %s', + 'lte': '<= %s', + 'startswith': 'LIKE BINARY %s', + 'endswith': 'LIKE BINARY %s', + 'istartswith': 'LIKE %s', + 'iendswith': 'LIKE %s', + } + + def __init__(self, **kwargs): + super(DatabaseWrapper, self).__init__(**kwargs) + self.server_version = None + + self.features = DatabaseFeatures() + self.ops = DatabaseOperations() + self.client = DatabaseClient() + self.creation = DatabaseCreation(self) + self.introspection = DatabaseIntrospection(self) + self.validation = DatabaseValidation() + + def _valid_connection(self): + if self.connection is not None: + try: + self.connection.ping() + return True + except DatabaseError: + self.connection.close() + self.connection = None + return False + + def _cursor(self, settings): + if not self._valid_connection(): + kwargs = { + 'conv': django_conversions, + 'charset': 'utf8', + 'use_unicode': True, + } + if settings.DATABASE_USER: + kwargs['user'] = settings.DATABASE_USER + if settings.DATABASE_NAME: + kwargs['db'] = settings.DATABASE_NAME + if settings.DATABASE_PASSWORD: + kwargs['passwd'] = settings.DATABASE_PASSWORD + if settings.DATABASE_HOST.startswith('/'): + kwargs['unix_socket'] = settings.DATABASE_HOST + elif settings.DATABASE_HOST: + kwargs['host'] = settings.DATABASE_HOST + if settings.DATABASE_PORT: + kwargs['port'] = int(settings.DATABASE_PORT) + kwargs.update(self.options) + self.connection = Database.connect(**kwargs) + cursor = self.connection.cursor() + return cursor + + def _rollback(self): + try: + BaseDatabaseWrapper._rollback(self) + except Database.NotSupportedError: + pass + + def get_server_version(self): + if not self.server_version: + if not self._valid_connection(): + self.cursor() + m = server_version_re.match(self.connection.get_server_info()) + if not m: + raise Exception('Unable to determine MySQL version from version string %r' % self.connection.get_server_info()) + self.server_version = tuple([int(x) for x in m.groups()]) + return self.server_version diff --git a/webapp/django/db/backends/mysql/client.py b/webapp/django/db/backends/mysql/client.py new file mode 100644 index 0000000000..24758867af --- /dev/null +++ b/webapp/django/db/backends/mysql/client.py @@ -0,0 +1,29 @@ +from django.db.backends import BaseDatabaseClient +from django.conf import settings +import os + +class DatabaseClient(BaseDatabaseClient): + def runshell(self): + args = [''] + db = settings.DATABASE_OPTIONS.get('db', settings.DATABASE_NAME) + user = settings.DATABASE_OPTIONS.get('user', settings.DATABASE_USER) + passwd = settings.DATABASE_OPTIONS.get('passwd', settings.DATABASE_PASSWORD) + host = settings.DATABASE_OPTIONS.get('host', settings.DATABASE_HOST) + port = settings.DATABASE_OPTIONS.get('port', settings.DATABASE_PORT) + defaults_file = settings.DATABASE_OPTIONS.get('read_default_file') + # Seems to be no good way to set sql_mode with CLI + + if defaults_file: + args += ["--defaults-file=%s" % defaults_file] + if user: + args += ["--user=%s" % user] + if passwd: + args += ["--password=%s" % passwd] + if host: + args += ["--host=%s" % host] + if port: + args += ["--port=%s" % port] + if db: + args += [db] + + os.execvp('mysql', args) diff --git a/webapp/django/db/backends/mysql/creation.py b/webapp/django/db/backends/mysql/creation.py new file mode 100644 index 0000000000..96faaacb75 --- /dev/null +++ b/webapp/django/db/backends/mysql/creation.py @@ -0,0 +1,68 @@ +from django.conf import settings +from django.db.backends.creation import BaseDatabaseCreation + +class DatabaseCreation(BaseDatabaseCreation): + # This dictionary maps Field objects to their associated MySQL column + # types, as strings. Column-type strings can contain format strings; they'll + # be interpolated against the values of Field.__dict__ before being output. + # If a column type is set to None, it won't be included in the output. + data_types = { + 'AutoField': 'integer AUTO_INCREMENT', + 'BooleanField': 'bool', + 'CharField': 'varchar(%(max_length)s)', + 'CommaSeparatedIntegerField': 'varchar(%(max_length)s)', + 'DateField': 'date', + 'DateTimeField': 'datetime', + 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)', + 'FileField': 'varchar(%(max_length)s)', + 'FilePathField': 'varchar(%(max_length)s)', + 'FloatField': 'double precision', + 'IntegerField': 'integer', + 'IPAddressField': 'char(15)', + 'NullBooleanField': 'bool', + 'OneToOneField': 'integer', + 'PhoneNumberField': 'varchar(20)', + 'PositiveIntegerField': 'integer UNSIGNED', + 'PositiveSmallIntegerField': 'smallint UNSIGNED', + 'SlugField': 'varchar(%(max_length)s)', + 'SmallIntegerField': 'smallint', + 'TextField': 'longtext', + 'TimeField': 'time', + 'USStateField': 'varchar(2)', + } + + def sql_table_creation_suffix(self): + suffix = [] + if settings.TEST_DATABASE_CHARSET: + suffix.append('CHARACTER SET %s' % settings.TEST_DATABASE_CHARSET) + if settings.TEST_DATABASE_COLLATION: + suffix.append('COLLATE %s' % settings.TEST_DATABASE_COLLATION) + return ' '.join(suffix) + + def sql_for_inline_foreign_key_references(self, field, known_models, style): + "All inline references are pending under MySQL" + return [], True + + def sql_for_inline_many_to_many_references(self, model, field, style): + from django.db import models + opts = model._meta + qn = self.connection.ops.quote_name + + table_output = [ + ' %s %s %s,' % + (style.SQL_FIELD(qn(field.m2m_column_name())), + style.SQL_COLTYPE(models.ForeignKey(model).db_type()), + style.SQL_KEYWORD('NOT NULL')), + ' %s %s %s,' % + (style.SQL_FIELD(qn(field.m2m_reverse_name())), + style.SQL_COLTYPE(models.ForeignKey(field.rel.to).db_type()), + style.SQL_KEYWORD('NOT NULL')) + ] + deferred = [ + (field.m2m_db_table(), field.m2m_column_name(), opts.db_table, + opts.pk.column), + (field.m2m_db_table(), field.m2m_reverse_name(), + field.rel.to._meta.db_table, field.rel.to._meta.pk.column) + ] + return table_output, deferred +
\ No newline at end of file diff --git a/webapp/django/db/backends/mysql/introspection.py b/webapp/django/db/backends/mysql/introspection.py new file mode 100644 index 0000000000..d8d59b73a1 --- /dev/null +++ b/webapp/django/db/backends/mysql/introspection.py @@ -0,0 +1,98 @@ +from django.db.backends import BaseDatabaseIntrospection +from MySQLdb import ProgrammingError, OperationalError +from MySQLdb.constants import FIELD_TYPE +import re + +foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)") + +class DatabaseIntrospection(BaseDatabaseIntrospection): + data_types_reverse = { + FIELD_TYPE.BLOB: 'TextField', + FIELD_TYPE.CHAR: 'CharField', + FIELD_TYPE.DECIMAL: 'DecimalField', + FIELD_TYPE.NEWDECIMAL: 'DecimalField', + FIELD_TYPE.DATE: 'DateField', + FIELD_TYPE.DATETIME: 'DateTimeField', + FIELD_TYPE.DOUBLE: 'FloatField', + FIELD_TYPE.FLOAT: 'FloatField', + FIELD_TYPE.INT24: 'IntegerField', + FIELD_TYPE.LONG: 'IntegerField', + FIELD_TYPE.LONGLONG: 'IntegerField', + FIELD_TYPE.SHORT: 'IntegerField', + FIELD_TYPE.STRING: 'CharField', + FIELD_TYPE.TIMESTAMP: 'DateTimeField', + FIELD_TYPE.TINY: 'IntegerField', + FIELD_TYPE.TINY_BLOB: 'TextField', + FIELD_TYPE.MEDIUM_BLOB: 'TextField', + FIELD_TYPE.LONG_BLOB: 'TextField', + FIELD_TYPE.VAR_STRING: 'CharField', + } + + def get_table_list(self, cursor): + "Returns a list of table names in the current database." + cursor.execute("SHOW TABLES") + return [row[0] for row in cursor.fetchall()] + + def get_table_description(self, cursor, table_name): + "Returns a description of the table, with the DB-API cursor.description interface." + cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name)) + return cursor.description + + def _name_to_index(self, cursor, table_name): + """ + Returns a dictionary of {field_name: field_index} for the given table. + Indexes are 0-based. + """ + return dict([(d[0], i) for i, d in enumerate(self.get_table_description(cursor, table_name))]) + + def get_relations(self, cursor, table_name): + """ + Returns a dictionary of {field_index: (field_index_other_table, other_table)} + representing all relationships to the given table. Indexes are 0-based. + """ + my_field_dict = self._name_to_index(cursor, table_name) + constraints = [] + relations = {} + try: + # This should work for MySQL 5.0. + cursor.execute(""" + SELECT column_name, referenced_table_name, referenced_column_name + FROM information_schema.key_column_usage + WHERE table_name = %s + AND table_schema = DATABASE() + AND referenced_table_name IS NOT NULL + AND referenced_column_name IS NOT NULL""", [table_name]) + constraints.extend(cursor.fetchall()) + except (ProgrammingError, OperationalError): + # Fall back to "SHOW CREATE TABLE", for previous MySQL versions. + # Go through all constraints and save the equal matches. + cursor.execute("SHOW CREATE TABLE %s" % self.connection.ops.quote_name(table_name)) + for row in cursor.fetchall(): + pos = 0 + while True: + match = foreign_key_re.search(row[1], pos) + if match == None: + break + pos = match.end() + constraints.append(match.groups()) + + for my_fieldname, other_table, other_field in constraints: + other_field_index = self._name_to_index(cursor, other_table)[other_field] + my_field_index = my_field_dict[my_fieldname] + relations[my_field_index] = (other_field_index, other_table) + + return relations + + def get_indexes(self, cursor, table_name): + """ + Returns a dictionary of fieldname -> infodict for the given table, + where each infodict is in the format: + {'primary_key': boolean representing whether it's the primary key, + 'unique': boolean representing whether it's a unique index} + """ + cursor.execute("SHOW INDEX FROM %s" % self.connection.ops.quote_name(table_name)) + indexes = {} + for row in cursor.fetchall(): + indexes[row[4]] = {'primary_key': (row[2] == 'PRIMARY'), 'unique': not bool(row[1])} + return indexes + diff --git a/webapp/django/db/backends/mysql/validation.py b/webapp/django/db/backends/mysql/validation.py new file mode 100644 index 0000000000..85354a8468 --- /dev/null +++ b/webapp/django/db/backends/mysql/validation.py @@ -0,0 +1,13 @@ +from django.db.backends import BaseDatabaseValidation + +class DatabaseValidation(BaseDatabaseValidation): + def validate_field(self, errors, opts, f): + "Prior to MySQL 5.0.3, character fields could not exceed 255 characters" + from django.db import models + from django.db import connection + db_version = connection.get_server_version() + if db_version < (5, 0, 3) and isinstance(f, (models.CharField, models.CommaSeparatedIntegerField, models.SlugField)) and f.max_length > 255: + errors.add(opts, + '"%s": %s cannot have a "max_length" greater than 255 when you are using a version of MySQL prior to 5.0.3 (you are using %s).' % + (f.name, f.__class__.__name__, '.'.join([str(n) for n in db_version[:3]]))) +
\ No newline at end of file diff --git a/webapp/django/db/backends/oracle/__init__.py b/webapp/django/db/backends/oracle/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/webapp/django/db/backends/oracle/__init__.py diff --git a/webapp/django/db/backends/oracle/base.py b/webapp/django/db/backends/oracle/base.py new file mode 100644 index 0000000000..b8bc6d00d5 --- /dev/null +++ b/webapp/django/db/backends/oracle/base.py @@ -0,0 +1,425 @@ +""" +Oracle database backend for Django. + +Requires cx_Oracle: http://www.python.net/crew/atuining/cx_Oracle/ +""" + +import os +import datetime +import time + +# Oracle takes client-side character set encoding from the environment. +os.environ['NLS_LANG'] = '.UTF8' +try: + import cx_Oracle as Database +except ImportError, e: + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("Error loading cx_Oracle module: %s" % e) + +from django.db.backends import * +from django.db.backends.oracle import query +from django.db.backends.oracle.client import DatabaseClient +from django.db.backends.oracle.creation import DatabaseCreation +from django.db.backends.oracle.introspection import DatabaseIntrospection +from django.utils.encoding import smart_str, force_unicode + +DatabaseError = Database.Error +IntegrityError = Database.IntegrityError + + +class DatabaseFeatures(BaseDatabaseFeatures): + empty_fetchmany_value = () + needs_datetime_string_cast = False + uses_custom_query_class = True + interprets_empty_strings_as_nulls = True + + +class DatabaseOperations(BaseDatabaseOperations): + def autoinc_sql(self, table, column): + # To simulate auto-incrementing primary keys in Oracle, we have to + # create a sequence and a trigger. + sq_name = get_sequence_name(table) + tr_name = get_trigger_name(table) + tbl_name = self.quote_name(table) + col_name = self.quote_name(column) + sequence_sql = """ + DECLARE + i INTEGER; + BEGIN + SELECT COUNT(*) INTO i FROM USER_CATALOG + WHERE TABLE_NAME = '%(sq_name)s' AND TABLE_TYPE = 'SEQUENCE'; + IF i = 0 THEN + EXECUTE IMMEDIATE 'CREATE SEQUENCE %(sq_name)s'; + END IF; + END; + /""" % locals() + trigger_sql = """ + CREATE OR REPLACE TRIGGER %(tr_name)s + BEFORE INSERT ON %(tbl_name)s + FOR EACH ROW + WHEN (new.%(col_name)s IS NULL) + BEGIN + SELECT %(sq_name)s.nextval + INTO :new.%(col_name)s FROM dual; + END; + /""" % locals() + return sequence_sql, trigger_sql + + def date_extract_sql(self, lookup_type, field_name): + # http://download-east.oracle.com/docs/cd/B10501_01/server.920/a96540/functions42a.htm#1017163 + return "EXTRACT(%s FROM %s)" % (lookup_type, field_name) + + def date_trunc_sql(self, lookup_type, field_name): + # Oracle uses TRUNC() for both dates and numbers. + # http://download-east.oracle.com/docs/cd/B10501_01/server.920/a96540/functions155a.htm#SQLRF06151 + if lookup_type == 'day': + sql = 'TRUNC(%s)' % field_name + else: + sql = "TRUNC(%s, '%s')" % (field_name, lookup_type) + return sql + + def datetime_cast_sql(self): + return "TO_TIMESTAMP(%s, 'YYYY-MM-DD HH24:MI:SS.FF')" + + def deferrable_sql(self): + return " DEFERRABLE INITIALLY DEFERRED" + + def drop_sequence_sql(self, table): + return "DROP SEQUENCE %s;" % self.quote_name(get_sequence_name(table)) + + def field_cast_sql(self, db_type): + if db_type and db_type.endswith('LOB'): + return "DBMS_LOB.SUBSTR(%s)" + else: + return "%s" + + def last_insert_id(self, cursor, table_name, pk_name): + sq_name = util.truncate_name(table_name, self.max_name_length() - 3) + cursor.execute('SELECT %s_sq.currval FROM dual' % sq_name) + return cursor.fetchone()[0] + + def lookup_cast(self, lookup_type): + if lookup_type in ('iexact', 'icontains', 'istartswith', 'iendswith'): + return "UPPER(%s)" + return "%s" + + def max_name_length(self): + return 30 + + def query_class(self, DefaultQueryClass): + return query.query_class(DefaultQueryClass, Database) + + def quote_name(self, name): + # SQL92 requires delimited (quoted) names to be case-sensitive. When + # not quoted, Oracle has case-insensitive behavior for identifiers, but + # always defaults to uppercase. + # We simplify things by making Oracle identifiers always uppercase. + if not name.startswith('"') and not name.endswith('"'): + name = '"%s"' % util.truncate_name(name.upper(), self.max_name_length()) + return name.upper() + + def random_function_sql(self): + return "DBMS_RANDOM.RANDOM" + + def regex_lookup_9(self, lookup_type): + raise NotImplementedError("Regexes are not supported in Oracle before version 10g.") + + def regex_lookup_10(self, lookup_type): + if lookup_type == 'regex': + match_option = "'c'" + else: + match_option = "'i'" + return 'REGEXP_LIKE(%%s, %%s, %s)' % match_option + + def regex_lookup(self, lookup_type): + # If regex_lookup is called before it's been initialized, then create + # a cursor to initialize it and recur. + from django.db import connection + connection.cursor() + return connection.ops.regex_lookup(lookup_type) + + def sql_flush(self, style, tables, sequences): + # Return a list of 'TRUNCATE x;', 'TRUNCATE y;', + # 'TRUNCATE z;'... style SQL statements + if tables: + # Oracle does support TRUNCATE, but it seems to get us into + # FK referential trouble, whereas DELETE FROM table works. + sql = ['%s %s %s;' % \ + (style.SQL_KEYWORD('DELETE'), + style.SQL_KEYWORD('FROM'), + style.SQL_FIELD(self.quote_name(table)) + ) for table in tables] + # Since we've just deleted all the rows, running our sequence + # ALTER code will reset the sequence to 0. + for sequence_info in sequences: + sequence_name = get_sequence_name(sequence_info['table']) + table_name = self.quote_name(sequence_info['table']) + column_name = self.quote_name(sequence_info['column'] or 'id') + query = _get_sequence_reset_sql() % {'sequence': sequence_name, + 'table': table_name, + 'column': column_name} + sql.append(query) + return sql + else: + return [] + + def sequence_reset_sql(self, style, model_list): + from django.db import models + output = [] + query = _get_sequence_reset_sql() + for model in model_list: + for f in model._meta.local_fields: + if isinstance(f, models.AutoField): + table_name = self.quote_name(model._meta.db_table) + sequence_name = get_sequence_name(model._meta.db_table) + column_name = self.quote_name(f.column) + output.append(query % {'sequence': sequence_name, + 'table': table_name, + 'column': column_name}) + break # Only one AutoField is allowed per model, so don't bother continuing. + for f in model._meta.many_to_many: + table_name = self.quote_name(f.m2m_db_table()) + sequence_name = get_sequence_name(f.m2m_db_table()) + column_name = self.quote_name('id') + output.append(query % {'sequence': sequence_name, + 'table': table_name, + 'column': column_name}) + return output + + def start_transaction_sql(self): + return '' + + def tablespace_sql(self, tablespace, inline=False): + return "%sTABLESPACE %s" % ((inline and "USING INDEX " or ""), self.quote_name(tablespace)) + + def value_to_db_time(self, value): + if value is None: + return None + if isinstance(value, basestring): + return datetime.datetime(*(time.strptime(value, '%H:%M:%S')[:6])) + return datetime.datetime(1900, 1, 1, value.hour, value.minute, + value.second, value.microsecond) + + def year_lookup_bounds_for_date_field(self, value): + first = '%s-01-01' + second = '%s-12-31' + return [first % value, second % value] + + +class DatabaseWrapper(BaseDatabaseWrapper): + + operators = { + 'exact': '= %s', + 'iexact': '= UPPER(%s)', + 'contains': "LIKEC %s ESCAPE '\\'", + 'icontains': "LIKEC UPPER(%s) ESCAPE '\\'", + 'gt': '> %s', + 'gte': '>= %s', + 'lt': '< %s', + 'lte': '<= %s', + 'startswith': "LIKEC %s ESCAPE '\\'", + 'endswith': "LIKEC %s ESCAPE '\\'", + 'istartswith': "LIKEC UPPER(%s) ESCAPE '\\'", + 'iendswith': "LIKEC UPPER(%s) ESCAPE '\\'", + } + oracle_version = None + + def __init__(self, *args, **kwargs): + super(DatabaseWrapper, self).__init__(*args, **kwargs) + + self.features = DatabaseFeatures() + self.ops = DatabaseOperations() + self.client = DatabaseClient() + self.creation = DatabaseCreation(self) + self.introspection = DatabaseIntrospection(self) + self.validation = BaseDatabaseValidation() + + def _valid_connection(self): + return self.connection is not None + + def _cursor(self, settings): + cursor = None + if not self._valid_connection(): + if len(settings.DATABASE_HOST.strip()) == 0: + settings.DATABASE_HOST = 'localhost' + if len(settings.DATABASE_PORT.strip()) != 0: + dsn = Database.makedsn(settings.DATABASE_HOST, int(settings.DATABASE_PORT), settings.DATABASE_NAME) + self.connection = Database.connect(settings.DATABASE_USER, settings.DATABASE_PASSWORD, dsn, **self.options) + else: + conn_string = "%s/%s@%s" % (settings.DATABASE_USER, settings.DATABASE_PASSWORD, settings.DATABASE_NAME) + self.connection = Database.connect(conn_string, **self.options) + cursor = FormatStylePlaceholderCursor(self.connection) + # Set oracle date to ansi date format. This only needs to execute + # once when we create a new connection. + cursor.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD' " + "NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS.FF'") + try: + self.oracle_version = int(self.connection.version.split('.')[0]) + # There's no way for the DatabaseOperations class to know the + # currently active Oracle version, so we do some setups here. + # TODO: Multi-db support will need a better solution (a way to + # communicate the current version). + if self.oracle_version <= 9: + self.ops.regex_lookup = self.ops.regex_lookup_9 + else: + self.ops.regex_lookup = self.ops.regex_lookup_10 + except ValueError: + pass + try: + self.connection.stmtcachesize = 20 + except: + # Django docs specify cx_Oracle version 4.3.1 or higher, but + # stmtcachesize is available only in 4.3.2 and up. + pass + if not cursor: + cursor = FormatStylePlaceholderCursor(self.connection) + # Default arraysize of 1 is highly sub-optimal. + cursor.arraysize = 100 + return cursor + + +class OracleParam(object): + """ + Wrapper object for formatting parameters for Oracle. If the string + representation of the value is large enough (greater than 4000 characters) + the input size needs to be set as NCLOB. Alternatively, if the parameter has + an `input_size` attribute, then the value of the `input_size` attribute will + be used instead. Otherwise, no input size will be set for the parameter when + executing the query. + """ + def __init__(self, param, charset, strings_only=False): + self.smart_str = smart_str(param, charset, strings_only) + if hasattr(param, 'input_size'): + # If parameter has `input_size` attribute, use that. + self.input_size = param.input_size + elif isinstance(param, basestring) and len(param) > 4000: + # Mark any string parameter greater than 4000 characters as an NCLOB. + self.input_size = Database.NCLOB + else: + self.input_size = None + + +class FormatStylePlaceholderCursor(Database.Cursor): + """ + Django uses "format" (e.g. '%s') style placeholders, but Oracle uses ":var" + style. This fixes it -- but note that if you want to use a literal "%s" in + a query, you'll need to use "%%s". + + We also do automatic conversion between Unicode on the Python side and + UTF-8 -- for talking to Oracle -- in here. + """ + charset = 'utf-8' + + def _format_params(self, params): + if isinstance(params, dict): + result = {} + for key, value in params.items(): + result[smart_str(key, self.charset)] = OracleParam(param, self.charset) + return result + else: + return tuple([OracleParam(p, self.charset, True) for p in params]) + + def _guess_input_sizes(self, params_list): + if isinstance(params_list[0], dict): + sizes = {} + iterators = [params.iteritems() for params in params_list] + else: + sizes = [None] * len(params_list[0]) + iterators = [enumerate(params) for params in params_list] + for iterator in iterators: + for key, value in iterator: + if value.input_size: sizes[key] = value.input_size + if isinstance(sizes, dict): + self.setinputsizes(**sizes) + else: + self.setinputsizes(*sizes) + + def _param_generator(self, params): + if isinstance(params, dict): + return dict([(k, p.smart_str) for k, p in params.iteritems()]) + else: + return [p.smart_str for p in params] + + def execute(self, query, params=None): + if params is None: + params = [] + else: + params = self._format_params(params) + args = [(':arg%d' % i) for i in range(len(params))] + # cx_Oracle wants no trailing ';' for SQL statements. For PL/SQL, it + # it does want a trailing ';' but not a trailing '/'. However, these + # characters must be included in the original query in case the query + # is being passed to SQL*Plus. + if query.endswith(';') or query.endswith('/'): + query = query[:-1] + query = smart_str(query, self.charset) % tuple(args) + self._guess_input_sizes([params]) + return Database.Cursor.execute(self, query, self._param_generator(params)) + + def executemany(self, query, params=None): + try: + args = [(':arg%d' % i) for i in range(len(params[0]))] + except (IndexError, TypeError): + # No params given, nothing to do + return None + # cx_Oracle wants no trailing ';' for SQL statements. For PL/SQL, it + # it does want a trailing ';' but not a trailing '/'. However, these + # characters must be included in the original query in case the query + # is being passed to SQL*Plus. + if query.endswith(';') or query.endswith('/'): + query = query[:-1] + query = smart_str(query, self.charset) % tuple(args) + formatted = [self._format_params(i) for i in params] + self._guess_input_sizes(formatted) + return Database.Cursor.executemany(self, query, [self._param_generator(p) for p in formatted]) + + def fetchone(self): + row = Database.Cursor.fetchone(self) + if row is None: + return row + return tuple([to_unicode(e) for e in row]) + + def fetchmany(self, size=None): + if size is None: + size = self.arraysize + return tuple([tuple([to_unicode(e) for e in r]) for r in Database.Cursor.fetchmany(self, size)]) + + def fetchall(self): + return tuple([tuple([to_unicode(e) for e in r]) for r in Database.Cursor.fetchall(self)]) + +def to_unicode(s): + """ + Convert strings to Unicode objects (and return all other data types + unchanged). + """ + if isinstance(s, basestring): + return force_unicode(s) + return s + +def _get_sequence_reset_sql(): + # TODO: colorize this SQL code with style.SQL_KEYWORD(), etc. + return """ + DECLARE + startvalue integer; + cval integer; + BEGIN + LOCK TABLE %(table)s IN SHARE MODE; + SELECT NVL(MAX(%(column)s), 0) INTO startvalue FROM %(table)s; + SELECT %(sequence)s.nextval INTO cval FROM dual; + cval := startvalue - cval; + IF cval != 0 THEN + EXECUTE IMMEDIATE 'ALTER SEQUENCE %(sequence)s MINVALUE 0 INCREMENT BY '||cval; + SELECT %(sequence)s.nextval INTO cval FROM dual; + EXECUTE IMMEDIATE 'ALTER SEQUENCE %(sequence)s INCREMENT BY 1'; + END IF; + COMMIT; + END; + /""" + +def get_sequence_name(table): + name_length = DatabaseOperations().max_name_length() - 3 + return '%s_SQ' % util.truncate_name(table, name_length).upper() + +def get_trigger_name(table): + name_length = DatabaseOperations().max_name_length() - 3 + return '%s_TR' % util.truncate_name(table, name_length).upper() diff --git a/webapp/django/db/backends/oracle/client.py b/webapp/django/db/backends/oracle/client.py new file mode 100644 index 0000000000..77fc9b9847 --- /dev/null +++ b/webapp/django/db/backends/oracle/client.py @@ -0,0 +1,13 @@ +from django.db.backends import BaseDatabaseClient +from django.conf import settings +import os + +class DatabaseClient(BaseDatabaseClient): + def runshell(self): + dsn = settings.DATABASE_USER + if settings.DATABASE_PASSWORD: + dsn += "/%s" % settings.DATABASE_PASSWORD + if settings.DATABASE_NAME: + dsn += "@%s" % settings.DATABASE_NAME + args = ["sqlplus", "-L", dsn] + os.execvp("sqlplus", args) diff --git a/webapp/django/db/backends/oracle/creation.py b/webapp/django/db/backends/oracle/creation.py new file mode 100644 index 0000000000..99c1b11a37 --- /dev/null +++ b/webapp/django/db/backends/oracle/creation.py @@ -0,0 +1,291 @@ +import sys, time +from django.conf import settings +from django.core import management +from django.db.backends.creation import BaseDatabaseCreation + +TEST_DATABASE_PREFIX = 'test_' +PASSWORD = 'Im_a_lumberjack' + +class DatabaseCreation(BaseDatabaseCreation): + # This dictionary maps Field objects to their associated Oracle column + # types, as strings. Column-type strings can contain format strings; they'll + # be interpolated against the values of Field.__dict__ before being output. + # If a column type is set to None, it won't be included in the output. + # + # Any format strings starting with "qn_" are quoted before being used in the + # output (the "qn_" prefix is stripped before the lookup is performed. + + data_types = { + 'AutoField': 'NUMBER(11)', + 'BooleanField': 'NUMBER(1) CHECK (%(qn_column)s IN (0,1))', + 'CharField': 'NVARCHAR2(%(max_length)s)', + 'CommaSeparatedIntegerField': 'VARCHAR2(%(max_length)s)', + 'DateField': 'DATE', + 'DateTimeField': 'TIMESTAMP', + 'DecimalField': 'NUMBER(%(max_digits)s, %(decimal_places)s)', + 'FileField': 'NVARCHAR2(%(max_length)s)', + 'FilePathField': 'NVARCHAR2(%(max_length)s)', + 'FloatField': 'DOUBLE PRECISION', + 'IntegerField': 'NUMBER(11)', + 'IPAddressField': 'VARCHAR2(15)', + 'NullBooleanField': 'NUMBER(1) CHECK ((%(qn_column)s IN (0,1)) OR (%(qn_column)s IS NULL))', + 'OneToOneField': 'NUMBER(11)', + 'PhoneNumberField': 'VARCHAR2(20)', + 'PositiveIntegerField': 'NUMBER(11) CHECK (%(qn_column)s >= 0)', + 'PositiveSmallIntegerField': 'NUMBER(11) CHECK (%(qn_column)s >= 0)', + 'SlugField': 'NVARCHAR2(50)', + 'SmallIntegerField': 'NUMBER(11)', + 'TextField': 'NCLOB', + 'TimeField': 'TIMESTAMP', + 'URLField': 'VARCHAR2(%(max_length)s)', + 'USStateField': 'CHAR(2)', + } + + remember = {} + + def _create_test_db(self, verbosity=1, autoclobber=False): + TEST_DATABASE_NAME = self._test_database_name(settings) + TEST_DATABASE_USER = self._test_database_user(settings) + TEST_DATABASE_PASSWD = self._test_database_passwd(settings) + TEST_DATABASE_TBLSPACE = self._test_database_tblspace(settings) + TEST_DATABASE_TBLSPACE_TMP = self._test_database_tblspace_tmp(settings) + + parameters = { + 'dbname': TEST_DATABASE_NAME, + 'user': TEST_DATABASE_USER, + 'password': TEST_DATABASE_PASSWD, + 'tblspace': TEST_DATABASE_TBLSPACE, + 'tblspace_temp': TEST_DATABASE_TBLSPACE_TMP, + } + + self.remember['user'] = settings.DATABASE_USER + self.remember['passwd'] = settings.DATABASE_PASSWORD + + cursor = self.connection.cursor() + if self._test_database_create(settings): + if verbosity >= 1: + print 'Creating test database...' + try: + self._execute_test_db_creation(cursor, parameters, verbosity) + except Exception, e: + sys.stderr.write("Got an error creating the test database: %s\n" % e) + if not autoclobber: + confirm = raw_input("It appears the test database, %s, already exists. Type 'yes' to delete it, or 'no' to cancel: " % TEST_DATABASE_NAME) + if autoclobber or confirm == 'yes': + try: + if verbosity >= 1: + print "Destroying old test database..." + self._execute_test_db_destruction(cursor, parameters, verbosity) + if verbosity >= 1: + print "Creating test database..." + self._execute_test_db_creation(cursor, parameters, verbosity) + except Exception, e: + sys.stderr.write("Got an error recreating the test database: %s\n" % e) + sys.exit(2) + else: + print "Tests cancelled." + sys.exit(1) + + if self._test_user_create(settings): + if verbosity >= 1: + print "Creating test user..." + try: + self._create_test_user(cursor, parameters, verbosity) + except Exception, e: + sys.stderr.write("Got an error creating the test user: %s\n" % e) + if not autoclobber: + confirm = raw_input("It appears the test user, %s, already exists. Type 'yes' to delete it, or 'no' to cancel: " % TEST_DATABASE_USER) + if autoclobber or confirm == 'yes': + try: + if verbosity >= 1: + print "Destroying old test user..." + self._destroy_test_user(cursor, parameters, verbosity) + if verbosity >= 1: + print "Creating test user..." + self._create_test_user(cursor, parameters, verbosity) + except Exception, e: + sys.stderr.write("Got an error recreating the test user: %s\n" % e) + sys.exit(2) + else: + print "Tests cancelled." + sys.exit(1) + + settings.DATABASE_USER = TEST_DATABASE_USER + settings.DATABASE_PASSWORD = TEST_DATABASE_PASSWD + + return settings.DATABASE_NAME + + def _destroy_test_db(self, test_database_name, verbosity=1): + """ + Destroy a test database, prompting the user for confirmation if the + database already exists. Returns the name of the test database created. + """ + TEST_DATABASE_NAME = self._test_database_name(settings) + TEST_DATABASE_USER = self._test_database_user(settings) + TEST_DATABASE_PASSWD = self._test_database_passwd(settings) + TEST_DATABASE_TBLSPACE = self._test_database_tblspace(settings) + TEST_DATABASE_TBLSPACE_TMP = self._test_database_tblspace_tmp(settings) + + settings.DATABASE_USER = self.remember['user'] + settings.DATABASE_PASSWORD = self.remember['passwd'] + + parameters = { + 'dbname': TEST_DATABASE_NAME, + 'user': TEST_DATABASE_USER, + 'password': TEST_DATABASE_PASSWD, + 'tblspace': TEST_DATABASE_TBLSPACE, + 'tblspace_temp': TEST_DATABASE_TBLSPACE_TMP, + } + + self.remember['user'] = settings.DATABASE_USER + self.remember['passwd'] = settings.DATABASE_PASSWORD + + cursor = self.connection.cursor() + time.sleep(1) # To avoid "database is being accessed by other users" errors. + if self._test_user_create(settings): + if verbosity >= 1: + print 'Destroying test user...' + self._destroy_test_user(cursor, parameters, verbosity) + if self._test_database_create(settings): + if verbosity >= 1: + print 'Destroying test database tables...' + self._execute_test_db_destruction(cursor, parameters, verbosity) + self.connection.close() + + def _execute_test_db_creation(self, cursor, parameters, verbosity): + if verbosity >= 2: + print "_create_test_db(): dbname = %s" % parameters['dbname'] + statements = [ + """CREATE TABLESPACE %(tblspace)s + DATAFILE '%(tblspace)s.dbf' SIZE 20M + REUSE AUTOEXTEND ON NEXT 10M MAXSIZE 100M + """, + """CREATE TEMPORARY TABLESPACE %(tblspace_temp)s + TEMPFILE '%(tblspace_temp)s.dbf' SIZE 20M + REUSE AUTOEXTEND ON NEXT 10M MAXSIZE 100M + """, + ] + self._execute_statements(cursor, statements, parameters, verbosity) + + def _create_test_user(self, cursor, parameters, verbosity): + if verbosity >= 2: + print "_create_test_user(): username = %s" % parameters['user'] + statements = [ + """CREATE USER %(user)s + IDENTIFIED BY %(password)s + DEFAULT TABLESPACE %(tblspace)s + TEMPORARY TABLESPACE %(tblspace_temp)s + """, + """GRANT CONNECT, RESOURCE TO %(user)s""", + ] + self._execute_statements(cursor, statements, parameters, verbosity) + + def _execute_test_db_destruction(self, cursor, parameters, verbosity): + if verbosity >= 2: + print "_execute_test_db_destruction(): dbname=%s" % parameters['dbname'] + statements = [ + 'DROP TABLESPACE %(tblspace)s INCLUDING CONTENTS AND DATAFILES CASCADE CONSTRAINTS', + 'DROP TABLESPACE %(tblspace_temp)s INCLUDING CONTENTS AND DATAFILES CASCADE CONSTRAINTS', + ] + self._execute_statements(cursor, statements, parameters, verbosity) + + def _destroy_test_user(self, cursor, parameters, verbosity): + if verbosity >= 2: + print "_destroy_test_user(): user=%s" % parameters['user'] + print "Be patient. This can take some time..." + statements = [ + 'DROP USER %(user)s CASCADE', + ] + self._execute_statements(cursor, statements, parameters, verbosity) + + def _execute_statements(self, cursor, statements, parameters, verbosity): + for template in statements: + stmt = template % parameters + if verbosity >= 2: + print stmt + try: + cursor.execute(stmt) + except Exception, err: + sys.stderr.write("Failed (%s)\n" % (err)) + raise + + def _test_database_name(self, settings): + name = TEST_DATABASE_PREFIX + settings.DATABASE_NAME + try: + if settings.TEST_DATABASE_NAME: + name = settings.TEST_DATABASE_NAME + except AttributeError: + pass + except: + raise + return name + + def _test_database_create(self, settings): + name = True + try: + if settings.TEST_DATABASE_CREATE: + name = True + else: + name = False + except AttributeError: + pass + except: + raise + return name + + def _test_user_create(self, settings): + name = True + try: + if settings.TEST_USER_CREATE: + name = True + else: + name = False + except AttributeError: + pass + except: + raise + return name + + def _test_database_user(self, ettings): + name = TEST_DATABASE_PREFIX + settings.DATABASE_NAME + try: + if settings.TEST_DATABASE_USER: + name = settings.TEST_DATABASE_USER + except AttributeError: + pass + except: + raise + return name + + def _test_database_passwd(self, settings): + name = PASSWORD + try: + if settings.TEST_DATABASE_PASSWD: + name = settings.TEST_DATABASE_PASSWD + except AttributeError: + pass + except: + raise + return name + + def _test_database_tblspace(self, settings): + name = TEST_DATABASE_PREFIX + settings.DATABASE_NAME + try: + if settings.TEST_DATABASE_TBLSPACE: + name = settings.TEST_DATABASE_TBLSPACE + except AttributeError: + pass + except: + raise + return name + + def _test_database_tblspace_tmp(self, settings): + name = TEST_DATABASE_PREFIX + settings.DATABASE_NAME + '_temp' + try: + if settings.TEST_DATABASE_TBLSPACE_TMP: + name = settings.TEST_DATABASE_TBLSPACE_TMP + except AttributeError: + pass + except: + raise + return name diff --git a/webapp/django/db/backends/oracle/introspection.py b/webapp/django/db/backends/oracle/introspection.py new file mode 100644 index 0000000000..890e30a694 --- /dev/null +++ b/webapp/django/db/backends/oracle/introspection.py @@ -0,0 +1,103 @@ +from django.db.backends import BaseDatabaseIntrospection +import cx_Oracle +import re + +foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)") + +class DatabaseIntrospection(BaseDatabaseIntrospection): + # Maps type objects to Django Field types. + data_types_reverse = { + cx_Oracle.CLOB: 'TextField', + cx_Oracle.DATETIME: 'DateTimeField', + cx_Oracle.FIXED_CHAR: 'CharField', + cx_Oracle.NCLOB: 'TextField', + cx_Oracle.NUMBER: 'DecimalField', + cx_Oracle.STRING: 'CharField', + cx_Oracle.TIMESTAMP: 'DateTimeField', + } + + def get_table_list(self, cursor): + "Returns a list of table names in the current database." + cursor.execute("SELECT TABLE_NAME FROM USER_TABLES") + return [row[0].upper() for row in cursor.fetchall()] + + def get_table_description(self, cursor, table_name): + "Returns a description of the table, with the DB-API cursor.description interface." + cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name)) + return cursor.description + + def table_name_converter(self, name): + "Table name comparison is case insensitive under Oracle" + return name.upper() + + def _name_to_index(self, cursor, table_name): + """ + Returns a dictionary of {field_name: field_index} for the given table. + Indexes are 0-based. + """ + return dict([(d[0], i) for i, d in enumerate(self.get_table_description(cursor, table_name))]) + + def get_relations(self, cursor, table_name): + """ + Returns a dictionary of {field_index: (field_index_other_table, other_table)} + representing all relationships to the given table. Indexes are 0-based. + """ + cursor.execute(""" + SELECT ta.column_id - 1, tb.table_name, tb.column_id - 1 + FROM user_constraints, USER_CONS_COLUMNS ca, USER_CONS_COLUMNS cb, + user_tab_cols ta, user_tab_cols tb + WHERE user_constraints.table_name = %s AND + ta.table_name = %s AND + ta.column_name = ca.column_name AND + ca.table_name = %s AND + user_constraints.constraint_name = ca.constraint_name AND + user_constraints.r_constraint_name = cb.constraint_name AND + cb.table_name = tb.table_name AND + cb.column_name = tb.column_name AND + ca.position = cb.position""", [table_name, table_name, table_name]) + + relations = {} + for row in cursor.fetchall(): + relations[row[0]] = (row[2], row[1]) + return relations + + def get_indexes(self, cursor, table_name): + """ + Returns a dictionary of fieldname -> infodict for the given table, + where each infodict is in the format: + {'primary_key': boolean representing whether it's the primary key, + 'unique': boolean representing whether it's a unique index} + """ + # This query retrieves each index on the given table, including the + # first associated field name + # "We were in the nick of time; you were in great peril!" + sql = """ + WITH primarycols AS ( + SELECT user_cons_columns.table_name, user_cons_columns.column_name, 1 AS PRIMARYCOL + FROM user_cons_columns, user_constraints + WHERE user_cons_columns.constraint_name = user_constraints.constraint_name AND + user_constraints.constraint_type = 'P' AND + user_cons_columns.table_name = %s), + uniquecols AS ( + SELECT user_ind_columns.table_name, user_ind_columns.column_name, 1 AS UNIQUECOL + FROM user_indexes, user_ind_columns + WHERE uniqueness = 'UNIQUE' AND + user_indexes.index_name = user_ind_columns.index_name AND + user_ind_columns.table_name = %s) + SELECT allcols.column_name, primarycols.primarycol, uniquecols.UNIQUECOL + FROM (SELECT column_name FROM primarycols UNION SELECT column_name FROM + uniquecols) allcols, + primarycols, uniquecols + WHERE allcols.column_name = primarycols.column_name (+) AND + allcols.column_name = uniquecols.column_name (+) + """ + cursor.execute(sql, [table_name, table_name]) + indexes = {} + for row in cursor.fetchall(): + # row[1] (idx.indkey) is stored in the DB as an array. It comes out as + # a string of space-separated integers. This designates the field + # indexes (1-based) of the fields that have indexes on the table. + # Here, we skip any indexes across multiple fields. + indexes[row[0]] = {'primary_key': row[1], 'unique': row[2]} + return indexes + diff --git a/webapp/django/db/backends/oracle/query.py b/webapp/django/db/backends/oracle/query.py new file mode 100644 index 0000000000..85e2f68822 --- /dev/null +++ b/webapp/django/db/backends/oracle/query.py @@ -0,0 +1,145 @@ +""" +Custom Query class for Oracle. +Derives from: django.db.models.sql.query.Query +""" + +import datetime + +from django.db.backends import util + +# Cache. Maps default query class to new Oracle query class. +_classes = {} + +def query_class(QueryClass, Database): + """ + Returns a custom django.db.models.sql.query.Query subclass that is + appropriate for Oracle. + + The 'Database' module (cx_Oracle) is passed in here so that all the setup + required to import it only needs to be done by the calling module. + """ + global _classes + try: + return _classes[QueryClass] + except KeyError: + pass + + class OracleQuery(QueryClass): + def resolve_columns(self, row, fields=()): + index_start = len(self.extra_select.keys()) + values = [self.convert_values(v, None) for v in row[:index_start]] + for value, field in map(None, row[index_start:], fields): + values.append(self.convert_values(value, field)) + return values + + def convert_values(self, value, field): + from django.db.models.fields import DateField, DateTimeField, \ + TimeField, BooleanField, NullBooleanField, DecimalField, Field + if isinstance(value, Database.LOB): + value = value.read() + # Oracle stores empty strings as null. We need to undo this in + # order to adhere to the Django convention of using the empty + # string instead of null, but only if the field accepts the + # empty string. + if value is None and isinstance(field, Field) and field.empty_strings_allowed: + value = u'' + # Convert 1 or 0 to True or False + elif value in (1, 0) and isinstance(field, (BooleanField, NullBooleanField)): + value = bool(value) + # Convert floats to decimals + elif value is not None and isinstance(field, DecimalField): + value = util.typecast_decimal(field.format_number(value)) + # cx_Oracle always returns datetime.datetime objects for + # DATE and TIMESTAMP columns, but Django wants to see a + # python datetime.date, .time, or .datetime. We use the type + # of the Field to determine which to cast to, but it's not + # always available. + # As a workaround, we cast to date if all the time-related + # values are 0, or to time if the date is 1/1/1900. + # This could be cleaned a bit by adding a method to the Field + # classes to normalize values from the database (the to_python + # method is used for validation and isn't what we want here). + elif isinstance(value, Database.Timestamp): + # In Python 2.3, the cx_Oracle driver returns its own + # Timestamp object that we must convert to a datetime class. + if not isinstance(value, datetime.datetime): + value = datetime.datetime(value.year, value.month, + value.day, value.hour, value.minute, value.second, + value.fsecond) + if isinstance(field, DateTimeField): + # DateTimeField subclasses DateField so must be checked + # first. + pass + elif isinstance(field, DateField): + value = value.date() + elif isinstance(field, TimeField) or (value.year == 1900 and value.month == value.day == 1): + value = value.time() + elif value.hour == value.minute == value.second == value.microsecond == 0: + value = value.date() + return value + + def as_sql(self, with_limits=True, with_col_aliases=False): + """ + Creates the SQL for this query. Returns the SQL string and list + of parameters. This is overriden from the original Query class + to handle the additional SQL Oracle requires to emulate LIMIT + and OFFSET. + + If 'with_limits' is False, any limit/offset information is not + included in the query. + """ + + # The `do_offset` flag indicates whether we need to construct + # the SQL needed to use limit/offset with Oracle. + do_offset = with_limits and (self.high_mark is not None + or self.low_mark) + if not do_offset: + sql, params = super(OracleQuery, self).as_sql(with_limits=False, + with_col_aliases=with_col_aliases) + else: + # `get_columns` needs to be called before `get_ordering` to + # populate `_select_alias`. + self.pre_sql_setup() + self.get_columns() + ordering = self.get_ordering() + + # Oracle's ROW_NUMBER() function requires an ORDER BY clause. + if ordering: + rn_orderby = ', '.join(ordering) + else: + # Create a default ORDER BY since none was specified. + qn = self.quote_name_unless_alias + opts = self.model._meta + rn_orderby = '%s.%s' % (qn(opts.db_table), + qn(opts.fields[0].db_column or opts.fields[0].column)) + + # Ensure the base query SELECTs our special "_RN" column + self.extra_select['_RN'] = ('ROW_NUMBER() OVER (ORDER BY %s)' + % rn_orderby, '') + sql, params = super(OracleQuery, self).as_sql(with_limits=False, + with_col_aliases=True) + + # Wrap the base query in an outer SELECT * with boundaries on + # the "_RN" column. This is the canonical way to emulate LIMIT + # and OFFSET on Oracle. + sql = 'SELECT * FROM (%s) WHERE "_RN" > %d' % (sql, self.low_mark) + if self.high_mark is not None: + sql = '%s AND "_RN" <= %d' % (sql, self.high_mark) + + return sql, params + + def set_limits(self, low=None, high=None): + super(OracleQuery, self).set_limits(low, high) + # We need to select the row number for the LIMIT/OFFSET sql. + # A placeholder is added to extra_select now, because as_sql is + # too late to be modifying extra_select. However, the actual sql + # depends on the ordering, so that is generated in as_sql. + self.extra_select['_RN'] = ('1', '') + + def clear_limits(self): + super(OracleQuery, self).clear_limits() + if '_RN' in self.extra_select: + del self.extra_select['_RN'] + + _classes[QueryClass] = OracleQuery + return OracleQuery diff --git a/webapp/django/db/backends/postgresql/__init__.py b/webapp/django/db/backends/postgresql/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/webapp/django/db/backends/postgresql/__init__.py diff --git a/webapp/django/db/backends/postgresql/base.py b/webapp/django/db/backends/postgresql/base.py new file mode 100644 index 0000000000..376d7ba2c8 --- /dev/null +++ b/webapp/django/db/backends/postgresql/base.py @@ -0,0 +1,148 @@ +""" +PostgreSQL database backend for Django. + +Requires psycopg 1: http://initd.org/projects/psycopg1 +""" + +from django.db.backends import * +from django.db.backends.postgresql.client import DatabaseClient +from django.db.backends.postgresql.creation import DatabaseCreation +from django.db.backends.postgresql.introspection import DatabaseIntrospection +from django.db.backends.postgresql.operations import DatabaseOperations +from django.db.backends.postgresql.version import get_version +from django.utils.encoding import smart_str, smart_unicode + +try: + import psycopg as Database +except ImportError, e: + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("Error loading psycopg module: %s" % e) + +DatabaseError = Database.DatabaseError +IntegrityError = Database.IntegrityError + +class UnicodeCursorWrapper(object): + """ + A thin wrapper around psycopg cursors that allows them to accept Unicode + strings as params. + + This is necessary because psycopg doesn't apply any DB quoting to + parameters that are Unicode strings. If a param is Unicode, this will + convert it to a bytestring using database client's encoding before passing + it to psycopg. + + All results retrieved from the database are converted into Unicode strings + before being returned to the caller. + """ + def __init__(self, cursor, charset): + self.cursor = cursor + self.charset = charset + + def format_params(self, params): + if isinstance(params, dict): + result = {} + charset = self.charset + for key, value in params.items(): + result[smart_str(key, charset)] = smart_str(value, charset) + return result + else: + return tuple([smart_str(p, self.charset, True) for p in params]) + + def execute(self, sql, params=()): + return self.cursor.execute(smart_str(sql, self.charset), self.format_params(params)) + + def executemany(self, sql, param_list): + new_param_list = [self.format_params(params) for params in param_list] + return self.cursor.executemany(sql, new_param_list) + + def __getattr__(self, attr): + if attr in self.__dict__: + return self.__dict__[attr] + else: + return getattr(self.cursor, attr) + + def __iter__(self): + return iter(self.cursor) + +class DatabaseFeatures(BaseDatabaseFeatures): + uses_savepoints = True + +class DatabaseWrapper(BaseDatabaseWrapper): + operators = { + 'exact': '= %s', + 'iexact': '= UPPER(%s)', + 'contains': 'LIKE %s', + 'icontains': 'LIKE UPPER(%s)', + 'regex': '~ %s', + 'iregex': '~* %s', + 'gt': '> %s', + 'gte': '>= %s', + 'lt': '< %s', + 'lte': '<= %s', + 'startswith': 'LIKE %s', + 'endswith': 'LIKE %s', + 'istartswith': 'LIKE UPPER(%s)', + 'iendswith': 'LIKE UPPER(%s)', + } + + def __init__(self, *args, **kwargs): + super(DatabaseWrapper, self).__init__(*args, **kwargs) + + self.features = DatabaseFeatures() + self.ops = DatabaseOperations() + self.client = DatabaseClient() + self.creation = DatabaseCreation(self) + self.introspection = DatabaseIntrospection(self) + self.validation = BaseDatabaseValidation() + + def _cursor(self, settings): + set_tz = False + if self.connection is None: + set_tz = True + if settings.DATABASE_NAME == '': + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("You need to specify DATABASE_NAME in your Django settings file.") + conn_string = "dbname=%s" % settings.DATABASE_NAME + if settings.DATABASE_USER: + conn_string = "user=%s %s" % (settings.DATABASE_USER, conn_string) + if settings.DATABASE_PASSWORD: + conn_string += " password='%s'" % settings.DATABASE_PASSWORD + if settings.DATABASE_HOST: + conn_string += " host=%s" % settings.DATABASE_HOST + if settings.DATABASE_PORT: + conn_string += " port=%s" % settings.DATABASE_PORT + self.connection = Database.connect(conn_string, **self.options) + self.connection.set_isolation_level(1) # make transactions transparent to all cursors + cursor = self.connection.cursor() + if set_tz: + cursor.execute("SET TIME ZONE %s", [settings.TIME_ZONE]) + if not hasattr(self, '_version'): + version = get_version(cursor) + self.__class__._version = version + if version < (8, 0): + # No savepoint support for earlier version of PostgreSQL. + self.features.uses_savepoints = False + cursor.execute("SET client_encoding to 'UNICODE'") + cursor = UnicodeCursorWrapper(cursor, 'utf-8') + return cursor + +def typecast_string(s): + """ + Cast all returned strings to unicode strings. + """ + if not s and not isinstance(s, str): + return s + return smart_unicode(s) + +# Register these custom typecasts, because Django expects dates/times to be +# in Python's native (standard-library) datetime/time format, whereas psycopg +# use mx.DateTime by default. +try: + Database.register_type(Database.new_type((1082,), "DATE", util.typecast_date)) +except AttributeError: + raise Exception("You appear to be using psycopg version 2. Set your DATABASE_ENGINE to 'postgresql_psycopg2' instead of 'postgresql'.") +Database.register_type(Database.new_type((1083,1266), "TIME", util.typecast_time)) +Database.register_type(Database.new_type((1114,1184), "TIMESTAMP", util.typecast_timestamp)) +Database.register_type(Database.new_type((16,), "BOOLEAN", util.typecast_boolean)) +Database.register_type(Database.new_type((1700,), "NUMERIC", util.typecast_decimal)) +Database.register_type(Database.new_type(Database.types[1043].values, 'STRING', typecast_string)) diff --git a/webapp/django/db/backends/postgresql/client.py b/webapp/django/db/backends/postgresql/client.py new file mode 100644 index 0000000000..28daed833a --- /dev/null +++ b/webapp/django/db/backends/postgresql/client.py @@ -0,0 +1,17 @@ +from django.db.backends import BaseDatabaseClient +from django.conf import settings +import os + +class DatabaseClient(BaseDatabaseClient): + def runshell(self): + args = ['psql'] + if settings.DATABASE_USER: + args += ["-U", settings.DATABASE_USER] + if settings.DATABASE_PASSWORD: + args += ["-W"] + if settings.DATABASE_HOST: + args.extend(["-h", settings.DATABASE_HOST]) + if settings.DATABASE_PORT: + args.extend(["-p", str(settings.DATABASE_PORT)]) + args += [settings.DATABASE_NAME] + os.execvp('psql', args) diff --git a/webapp/django/db/backends/postgresql/creation.py b/webapp/django/db/backends/postgresql/creation.py new file mode 100644 index 0000000000..3e537e345e --- /dev/null +++ b/webapp/django/db/backends/postgresql/creation.py @@ -0,0 +1,38 @@ +from django.conf import settings +from django.db.backends.creation import BaseDatabaseCreation + +class DatabaseCreation(BaseDatabaseCreation): + # This dictionary maps Field objects to their associated PostgreSQL column + # types, as strings. Column-type strings can contain format strings; they'll + # be interpolated against the values of Field.__dict__ before being output. + # If a column type is set to None, it won't be included in the output. + data_types = { + 'AutoField': 'serial', + 'BooleanField': 'boolean', + 'CharField': 'varchar(%(max_length)s)', + 'CommaSeparatedIntegerField': 'varchar(%(max_length)s)', + 'DateField': 'date', + 'DateTimeField': 'timestamp with time zone', + 'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)', + 'FileField': 'varchar(%(max_length)s)', + 'FilePathField': 'varchar(%(max_length)s)', + 'FloatField': 'double precision', + 'IntegerField': 'integer', + 'IPAddressField': 'inet', + 'NullBooleanField': 'boolean', + 'OneToOneField': 'integer', + 'PhoneNumberField': 'varchar(20)', + 'PositiveIntegerField': 'integer CHECK ("%(column)s" >= 0)', + 'PositiveSmallIntegerField': 'smallint CHECK ("%(column)s" >= 0)', + 'SlugField': 'varchar(%(max_length)s)', + 'SmallIntegerField': 'smallint', + 'TextField': 'text', + 'TimeField': 'time', + 'USStateField': 'varchar(2)', + } + + def sql_table_creation_suffix(self): + assert settings.TEST_DATABASE_COLLATION is None, "PostgreSQL does not support collation setting at database creation time." + if settings.TEST_DATABASE_CHARSET: + return "WITH ENCODING '%s'" % settings.TEST_DATABASE_CHARSET + return '' diff --git a/webapp/django/db/backends/postgresql/introspection.py b/webapp/django/db/backends/postgresql/introspection.py new file mode 100644 index 0000000000..7b3ab3bb8a --- /dev/null +++ b/webapp/django/db/backends/postgresql/introspection.py @@ -0,0 +1,86 @@ +from django.db.backends import BaseDatabaseIntrospection + +class DatabaseIntrospection(BaseDatabaseIntrospection): + # Maps type codes to Django Field types. + data_types_reverse = { + 16: 'BooleanField', + 21: 'SmallIntegerField', + 23: 'IntegerField', + 25: 'TextField', + 701: 'FloatField', + 869: 'IPAddressField', + 1043: 'CharField', + 1082: 'DateField', + 1083: 'TimeField', + 1114: 'DateTimeField', + 1184: 'DateTimeField', + 1266: 'TimeField', + 1700: 'DecimalField', + } + + def get_table_list(self, cursor): + "Returns a list of table names in the current database." + cursor.execute(""" + SELECT c.relname + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'v', '') + AND n.nspname NOT IN ('pg_catalog', 'pg_toast') + AND pg_catalog.pg_table_is_visible(c.oid)""") + return [row[0] for row in cursor.fetchall()] + + def get_table_description(self, cursor, table_name): + "Returns a description of the table, with the DB-API cursor.description interface." + cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name)) + return cursor.description + + def get_relations(self, cursor, table_name): + """ + Returns a dictionary of {field_index: (field_index_other_table, other_table)} + representing all relationships to the given table. Indexes are 0-based. + """ + cursor.execute(""" + SELECT con.conkey, con.confkey, c2.relname + FROM pg_constraint con, pg_class c1, pg_class c2 + WHERE c1.oid = con.conrelid + AND c2.oid = con.confrelid + AND c1.relname = %s + AND con.contype = 'f'""", [table_name]) + relations = {} + for row in cursor.fetchall(): + try: + # row[0] and row[1] are like "{2}", so strip the curly braces. + relations[int(row[0][1:-1]) - 1] = (int(row[1][1:-1]) - 1, row[2]) + except ValueError: + continue + return relations + + def get_indexes(self, cursor, table_name): + """ + Returns a dictionary of fieldname -> infodict for the given table, + where each infodict is in the format: + {'primary_key': boolean representing whether it's the primary key, + 'unique': boolean representing whether it's a unique index} + """ + # This query retrieves each index on the given table, including the + # first associated field name + cursor.execute(""" + SELECT attr.attname, idx.indkey, idx.indisunique, idx.indisprimary + FROM pg_catalog.pg_class c, pg_catalog.pg_class c2, + pg_catalog.pg_index idx, pg_catalog.pg_attribute attr + WHERE c.oid = idx.indrelid + AND idx.indexrelid = c2.oid + AND attr.attrelid = c.oid + AND attr.attnum = idx.indkey[0] + AND c.relname = %s""", [table_name]) + indexes = {} + for row in cursor.fetchall(): + # row[1] (idx.indkey) is stored in the DB as an array. It comes out as + # a string of space-separated integers. This designates the field + # indexes (1-based) of the fields that have indexes on the table. + # Here, we skip any indexes across multiple fields. + if ' ' in row[1]: + continue + indexes[row[0]] = {'primary_key': row[3], 'unique': row[2]} + return indexes + diff --git a/webapp/django/db/backends/postgresql/operations.py b/webapp/django/db/backends/postgresql/operations.py new file mode 100644 index 0000000000..01cc1fc8b7 --- /dev/null +++ b/webapp/django/db/backends/postgresql/operations.py @@ -0,0 +1,144 @@ +import re + +from django.db.backends import BaseDatabaseOperations + +server_version_re = re.compile(r'PostgreSQL (\d{1,2})\.(\d{1,2})\.?(\d{1,2})?') + +# This DatabaseOperations class lives in here instead of base.py because it's +# used by both the 'postgresql' and 'postgresql_psycopg2' backends. + +class DatabaseOperations(BaseDatabaseOperations): + def __init__(self): + self._postgres_version = None + + def _get_postgres_version(self): + if self._postgres_version is None: + from django.db import connection + cursor = connection.cursor() + cursor.execute("SELECT version()") + version_string = cursor.fetchone()[0] + m = server_version_re.match(version_string) + if not m: + raise Exception('Unable to determine PostgreSQL version from version() function string: %r' % version_string) + self._postgres_version = [int(val) for val in m.groups() if val] + return self._postgres_version + postgres_version = property(_get_postgres_version) + + def date_extract_sql(self, lookup_type, field_name): + # http://www.postgresql.org/docs/8.0/static/functions-datetime.html#FUNCTIONS-DATETIME-EXTRACT + return "EXTRACT('%s' FROM %s)" % (lookup_type, field_name) + + def date_trunc_sql(self, lookup_type, field_name): + # http://www.postgresql.org/docs/8.0/static/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC + return "DATE_TRUNC('%s', %s)" % (lookup_type, field_name) + + def deferrable_sql(self): + return " DEFERRABLE INITIALLY DEFERRED" + + def lookup_cast(self, lookup_type): + lookup = '%s' + + # Cast text lookups to text to allow things like filter(x__contains=4) + if lookup_type in ('iexact', 'contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith'): + lookup = "%s::text" + + # Use UPPER(x) for case-insensitive lookups; it's faster. + if lookup_type in ('iexact', 'icontains', 'istartswith', 'iendswith'): + lookup = 'UPPER(%s)' % lookup + + return lookup + + def field_cast_sql(self, db_type): + if db_type == 'inet': + return 'HOST(%s)' + return '%s' + + def last_insert_id(self, cursor, table_name, pk_name): + cursor.execute("SELECT CURRVAL('\"%s_%s_seq\"')" % (table_name, pk_name)) + return cursor.fetchone()[0] + + def no_limit_value(self): + return None + + def quote_name(self, name): + if name.startswith('"') and name.endswith('"'): + return name # Quoting once is enough. + return '"%s"' % name + + def sql_flush(self, style, tables, sequences): + if tables: + if self.postgres_version[0] >= 8 and self.postgres_version[1] >= 1: + # Postgres 8.1+ can do 'TRUNCATE x, y, z...;'. In fact, it *has to* + # in order to be able to truncate tables referenced by a foreign + # key in any other table. The result is a single SQL TRUNCATE + # statement. + sql = ['%s %s;' % \ + (style.SQL_KEYWORD('TRUNCATE'), + style.SQL_FIELD(', '.join([self.quote_name(table) for table in tables])) + )] + else: + # Older versions of Postgres can't do TRUNCATE in a single call, so + # they must use a simple delete. + sql = ['%s %s %s;' % \ + (style.SQL_KEYWORD('DELETE'), + style.SQL_KEYWORD('FROM'), + style.SQL_FIELD(self.quote_name(table)) + ) for table in tables] + + # 'ALTER SEQUENCE sequence_name RESTART WITH 1;'... style SQL statements + # to reset sequence indices + for sequence_info in sequences: + table_name = sequence_info['table'] + column_name = sequence_info['column'] + if column_name and len(column_name) > 0: + sequence_name = '%s_%s_seq' % (table_name, column_name) + else: + sequence_name = '%s_id_seq' % table_name + sql.append("%s setval('%s', 1, false);" % \ + (style.SQL_KEYWORD('SELECT'), + style.SQL_FIELD(self.quote_name(sequence_name))) + ) + return sql + else: + return [] + + def sequence_reset_sql(self, style, model_list): + from django.db import models + output = [] + qn = self.quote_name + for model in model_list: + # Use `coalesce` to set the sequence for each model to the max pk value if there are records, + # or 1 if there are none. Set the `is_called` property (the third argument to `setval`) to true + # if there are records (as the max pk value is already in use), otherwise set it to false. + for f in model._meta.local_fields: + if isinstance(f, models.AutoField): + output.append("%s setval('%s', coalesce(max(%s), 1), max(%s) %s null) %s %s;" % \ + (style.SQL_KEYWORD('SELECT'), + style.SQL_FIELD(qn('%s_%s_seq' % (model._meta.db_table, f.column))), + style.SQL_FIELD(qn(f.column)), + style.SQL_FIELD(qn(f.column)), + style.SQL_KEYWORD('IS NOT'), + style.SQL_KEYWORD('FROM'), + style.SQL_TABLE(qn(model._meta.db_table)))) + break # Only one AutoField is allowed per model, so don't bother continuing. + for f in model._meta.many_to_many: + output.append("%s setval('%s', coalesce(max(%s), 1), max(%s) %s null) %s %s;" % \ + (style.SQL_KEYWORD('SELECT'), + style.SQL_FIELD(qn('%s_id_seq' % f.m2m_db_table())), + style.SQL_FIELD(qn('id')), + style.SQL_FIELD(qn('id')), + style.SQL_KEYWORD('IS NOT'), + style.SQL_KEYWORD('FROM'), + style.SQL_TABLE(qn(f.m2m_db_table())))) + return output + + def savepoint_create_sql(self, sid): + return "SAVEPOINT %s" % sid + + def savepoint_commit_sql(self, sid): + return "RELEASE SAVEPOINT %s" % sid + + def savepoint_rollback_sql(self, sid): + return "ROLLBACK TO SAVEPOINT %s" % sid + diff --git a/webapp/django/db/backends/postgresql/version.py b/webapp/django/db/backends/postgresql/version.py new file mode 100644 index 0000000000..e14d791b07 --- /dev/null +++ b/webapp/django/db/backends/postgresql/version.py @@ -0,0 +1,18 @@ +""" +Extracts the version of the PostgreSQL server. +""" + +import re + +VERSION_RE = re.compile(r'PostgreSQL (\d+)\.(\d+)\.') + +def get_version(cursor): + """ + Returns a tuple representing the major and minor version number of the + server. For example, (7, 4) or (8, 3). + """ + cursor.execute("SELECT version()") + version = cursor.fetchone()[0] + major, minor = VERSION_RE.search(version).groups() + return int(major), int(minor) + diff --git a/webapp/django/db/backends/postgresql_psycopg2/__init__.py b/webapp/django/db/backends/postgresql_psycopg2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/webapp/django/db/backends/postgresql_psycopg2/__init__.py diff --git a/webapp/django/db/backends/postgresql_psycopg2/base.py b/webapp/django/db/backends/postgresql_psycopg2/base.py new file mode 100644 index 0000000000..6ecf86b705 --- /dev/null +++ b/webapp/django/db/backends/postgresql_psycopg2/base.py @@ -0,0 +1,97 @@ +""" +PostgreSQL database backend for Django. + +Requires psycopg 2: http://initd.org/projects/psycopg2 +""" + +from django.db.backends import * +from django.db.backends.postgresql.operations import DatabaseOperations as PostgresqlDatabaseOperations +from django.db.backends.postgresql.client import DatabaseClient +from django.db.backends.postgresql.creation import DatabaseCreation +from django.db.backends.postgresql.version import get_version +from django.db.backends.postgresql_psycopg2.introspection import DatabaseIntrospection +from django.utils.safestring import SafeUnicode, SafeString + +try: + import psycopg2 as Database + import psycopg2.extensions +except ImportError, e: + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("Error loading psycopg2 module: %s" % e) + +DatabaseError = Database.DatabaseError +IntegrityError = Database.IntegrityError + +psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) +psycopg2.extensions.register_adapter(SafeString, psycopg2.extensions.QuotedString) +psycopg2.extensions.register_adapter(SafeUnicode, psycopg2.extensions.QuotedString) + +class DatabaseFeatures(BaseDatabaseFeatures): + needs_datetime_string_cast = False + uses_savepoints = True + +class DatabaseOperations(PostgresqlDatabaseOperations): + def last_executed_query(self, cursor, sql, params): + # With psycopg2, cursor objects have a "query" attribute that is the + # exact query sent to the database. See docs here: + # http://www.initd.org/tracker/psycopg/wiki/psycopg2_documentation#postgresql-status-message-and-executed-query + return cursor.query + +class DatabaseWrapper(BaseDatabaseWrapper): + operators = { + 'exact': '= %s', + 'iexact': '= UPPER(%s)', + 'contains': 'LIKE %s', + 'icontains': 'LIKE UPPER(%s)', + 'regex': '~ %s', + 'iregex': '~* %s', + 'gt': '> %s', + 'gte': '>= %s', + 'lt': '< %s', + 'lte': '<= %s', + 'startswith': 'LIKE %s', + 'endswith': 'LIKE %s', + 'istartswith': 'LIKE UPPER(%s)', + 'iendswith': 'LIKE UPPER(%s)', + } + + def __init__(self, *args, **kwargs): + super(DatabaseWrapper, self).__init__(*args, **kwargs) + + self.features = DatabaseFeatures() + self.ops = DatabaseOperations() + self.client = DatabaseClient() + self.creation = DatabaseCreation(self) + self.introspection = DatabaseIntrospection(self) + self.validation = BaseDatabaseValidation() + + def _cursor(self, settings): + set_tz = False + if self.connection is None: + set_tz = True + if settings.DATABASE_NAME == '': + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured("You need to specify DATABASE_NAME in your Django settings file.") + conn_string = "dbname=%s" % settings.DATABASE_NAME + if settings.DATABASE_USER: + conn_string = "user=%s %s" % (settings.DATABASE_USER, conn_string) + if settings.DATABASE_PASSWORD: + conn_string += " password='%s'" % settings.DATABASE_PASSWORD + if settings.DATABASE_HOST: + conn_string += " host=%s" % settings.DATABASE_HOST + if settings.DATABASE_PORT: + conn_string += " port=%s" % settings.DATABASE_PORT + self.connection = Database.connect(conn_string, **self.options) + self.connection.set_isolation_level(1) # make transactions transparent to all cursors + self.connection.set_client_encoding('UTF8') + cursor = self.connection.cursor() + cursor.tzinfo_factory = None + if set_tz: + cursor.execute("SET TIME ZONE %s", [settings.TIME_ZONE]) + if not hasattr(self, '_version'): + version = get_version(cursor) + self.__class__._version = version + if version < (8, 0): + # No savepoint support for earlier version of PostgreSQL. + self.features.uses_savepoints = False + return cursor diff --git a/webapp/django/db/backends/postgresql_psycopg2/introspection.py b/webapp/django/db/backends/postgresql_psycopg2/introspection.py new file mode 100644 index 0000000000..83bd9b4c44 --- /dev/null +++ b/webapp/django/db/backends/postgresql_psycopg2/introspection.py @@ -0,0 +1,21 @@ +from django.db.backends.postgresql.introspection import DatabaseIntrospection as PostgresDatabaseIntrospection + +class DatabaseIntrospection(PostgresDatabaseIntrospection): + + def get_relations(self, cursor, table_name): + """ + Returns a dictionary of {field_index: (field_index_other_table, other_table)} + representing all relationships to the given table. Indexes are 0-based. + """ + cursor.execute(""" + SELECT con.conkey, con.confkey, c2.relname + FROM pg_constraint con, pg_class c1, pg_class c2 + WHERE c1.oid = con.conrelid + AND c2.oid = con.confrelid + AND c1.relname = %s + AND con.contype = 'f'""", [table_name]) + relations = {} + for row in cursor.fetchall(): + # row[0] and row[1] are single-item lists, so grab the single item. + relations[row[0][0] - 1] = (row[1][0] - 1, row[2]) + return relations diff --git a/webapp/django/db/backends/sqlite3/__init__.py b/webapp/django/db/backends/sqlite3/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/webapp/django/db/backends/sqlite3/__init__.py diff --git a/webapp/django/db/backends/sqlite3/base.py b/webapp/django/db/backends/sqlite3/base.py new file mode 100644 index 0000000000..64da52e45f --- /dev/null +++ b/webapp/django/db/backends/sqlite3/base.py @@ -0,0 +1,204 @@ +""" +SQLite3 backend for django. + +Python 2.3 and 2.4 require pysqlite2 (http://pysqlite.org/). + +Python 2.5 and later use the sqlite3 module in the standard library. +""" + +from django.db.backends import * +from django.db.backends.sqlite3.client import DatabaseClient +from django.db.backends.sqlite3.creation import DatabaseCreation +from django.db.backends.sqlite3.introspection import DatabaseIntrospection + +try: + try: + from sqlite3 import dbapi2 as Database + except ImportError: + from pysqlite2 import dbapi2 as Database +except ImportError, e: + import sys + from django.core.exceptions import ImproperlyConfigured + if sys.version_info < (2, 5, 0): + module = 'pysqlite2' + else: + module = 'sqlite3' + raise ImproperlyConfigured, "Error loading %s module: %s" % (module, e) + +try: + import decimal +except ImportError: + from django.utils import _decimal as decimal # for Python 2.3 + +DatabaseError = Database.DatabaseError +IntegrityError = Database.IntegrityError + +Database.register_converter("bool", lambda s: str(s) == '1') +Database.register_converter("time", util.typecast_time) +Database.register_converter("date", util.typecast_date) +Database.register_converter("datetime", util.typecast_timestamp) +Database.register_converter("timestamp", util.typecast_timestamp) +Database.register_converter("TIMESTAMP", util.typecast_timestamp) +Database.register_converter("decimal", util.typecast_decimal) +Database.register_adapter(decimal.Decimal, util.rev_typecast_decimal) +if Database.version_info >= (2,4,1): + # Starting in 2.4.1, the str type is not accepted anymore, therefore, + # we convert all str objects to Unicode + # As registering a adapter for a primitive type causes a small + # slow-down, this adapter is only registered for sqlite3 versions + # needing it. + Database.register_adapter(str, lambda s:s.decode('utf-8')) + +class DatabaseFeatures(BaseDatabaseFeatures): + # SQLite cannot handle us only partially reading from a cursor's result set + # and then writing the same rows to the database in another cursor. This + # setting ensures we always read result sets fully into memory all in one + # go. + can_use_chunked_reads = False + +class DatabaseOperations(BaseDatabaseOperations): + def date_extract_sql(self, lookup_type, field_name): + # sqlite doesn't support extract, so we fake it with the user-defined + # function django_extract that's registered in connect(). + return 'django_extract("%s", %s)' % (lookup_type.lower(), field_name) + + def date_trunc_sql(self, lookup_type, field_name): + # sqlite doesn't support DATE_TRUNC, so we fake it with a user-defined + # function django_date_trunc that's registered in connect(). + return 'django_date_trunc("%s", %s)' % (lookup_type.lower(), field_name) + + def drop_foreignkey_sql(self): + return "" + + def pk_default_value(self): + return 'NULL' + + def quote_name(self, name): + if name.startswith('"') and name.endswith('"'): + return name # Quoting once is enough. + return '"%s"' % name + + def no_limit_value(self): + return -1 + + def sql_flush(self, style, tables, sequences): + # NB: The generated SQL below is specific to SQLite + # Note: The DELETE FROM... SQL generated below works for SQLite databases + # because constraints don't exist + sql = ['%s %s %s;' % \ + (style.SQL_KEYWORD('DELETE'), + style.SQL_KEYWORD('FROM'), + style.SQL_FIELD(self.quote_name(table)) + ) for table in tables] + # Note: No requirement for reset of auto-incremented indices (cf. other + # sql_flush() implementations). Just return SQL at this point + return sql + + def year_lookup_bounds(self, value): + first = '%s-01-01' + second = '%s-12-31 23:59:59.999999' + return [first % value, second % value] + +class DatabaseWrapper(BaseDatabaseWrapper): + + # SQLite requires LIKE statements to include an ESCAPE clause if the value + # being escaped has a percent or underscore in it. + # See http://www.sqlite.org/lang_expr.html for an explanation. + operators = { + 'exact': '= %s', + 'iexact': "LIKE %s ESCAPE '\\'", + 'contains': "LIKE %s ESCAPE '\\'", + 'icontains': "LIKE %s ESCAPE '\\'", + 'regex': 'REGEXP %s', + 'iregex': "REGEXP '(?i)' || %s", + 'gt': '> %s', + 'gte': '>= %s', + 'lt': '< %s', + 'lte': '<= %s', + 'startswith': "LIKE %s ESCAPE '\\'", + 'endswith': "LIKE %s ESCAPE '\\'", + 'istartswith': "LIKE %s ESCAPE '\\'", + 'iendswith': "LIKE %s ESCAPE '\\'", + } + + def __init__(self, *args, **kwargs): + super(DatabaseWrapper, self).__init__(*args, **kwargs) + + self.features = DatabaseFeatures() + self.ops = DatabaseOperations() + self.client = DatabaseClient() + self.creation = DatabaseCreation(self) + self.introspection = DatabaseIntrospection(self) + self.validation = BaseDatabaseValidation() + + def _cursor(self, settings): + if self.connection is None: + if not settings.DATABASE_NAME: + from django.core.exceptions import ImproperlyConfigured + raise ImproperlyConfigured, "Please fill out DATABASE_NAME in the settings module before using the database." + kwargs = { + 'database': settings.DATABASE_NAME, + 'detect_types': Database.PARSE_DECLTYPES | Database.PARSE_COLNAMES, + } + kwargs.update(self.options) + self.connection = Database.connect(**kwargs) + # Register extract, date_trunc, and regexp functions. + self.connection.create_function("django_extract", 2, _sqlite_extract) + self.connection.create_function("django_date_trunc", 2, _sqlite_date_trunc) + self.connection.create_function("regexp", 2, _sqlite_regexp) + return self.connection.cursor(factory=SQLiteCursorWrapper) + + def close(self): + from django.conf import settings + # If database is in memory, closing the connection destroys the + # database. To prevent accidental data loss, ignore close requests on + # an in-memory db. + if settings.DATABASE_NAME != ":memory:": + BaseDatabaseWrapper.close(self) + +class SQLiteCursorWrapper(Database.Cursor): + """ + Django uses "format" style placeholders, but pysqlite2 uses "qmark" style. + This fixes it -- but note that if you want to use a literal "%s" in a query, + you'll need to use "%%s". + """ + def execute(self, query, params=()): + query = self.convert_query(query, len(params)) + return Database.Cursor.execute(self, query, params) + + def executemany(self, query, param_list): + try: + query = self.convert_query(query, len(param_list[0])) + return Database.Cursor.executemany(self, query, param_list) + except (IndexError,TypeError): + # No parameter list provided + return None + + def convert_query(self, query, num_params): + return query % tuple("?" * num_params) + +def _sqlite_extract(lookup_type, dt): + try: + dt = util.typecast_timestamp(dt) + except (ValueError, TypeError): + return None + return unicode(getattr(dt, lookup_type)) + +def _sqlite_date_trunc(lookup_type, dt): + try: + dt = util.typecast_timestamp(dt) + except (ValueError, TypeError): + return None + if lookup_type == 'year': + return "%i-01-01 00:00:00" % dt.year + elif lookup_type == 'month': + return "%i-%02i-01 00:00:00" % (dt.year, dt.month) + elif lookup_type == 'day': + return "%i-%02i-%02i 00:00:00" % (dt.year, dt.month, dt.day) + +def _sqlite_regexp(re_pattern, re_string): + import re + try: + return bool(re.search(re_pattern, re_string)) + except: + return False diff --git a/webapp/django/db/backends/sqlite3/client.py b/webapp/django/db/backends/sqlite3/client.py new file mode 100644 index 0000000000..affb1c228c --- /dev/null +++ b/webapp/django/db/backends/sqlite3/client.py @@ -0,0 +1,8 @@ +from django.db.backends import BaseDatabaseClient +from django.conf import settings +import os + +class DatabaseClient(BaseDatabaseClient): + def runshell(self): + args = ['', settings.DATABASE_NAME] + os.execvp('sqlite3', args) diff --git a/webapp/django/db/backends/sqlite3/creation.py b/webapp/django/db/backends/sqlite3/creation.py new file mode 100644 index 0000000000..8943854af4 --- /dev/null +++ b/webapp/django/db/backends/sqlite3/creation.py @@ -0,0 +1,72 @@ +import os +import sys +from django.conf import settings +from django.db.backends.creation import BaseDatabaseCreation + +class DatabaseCreation(BaseDatabaseCreation): + # SQLite doesn't actually support most of these types, but it "does the right + # thing" given more verbose field definitions, so leave them as is so that + # schema inspection is more useful. + data_types = { + 'AutoField': 'integer', + 'BooleanField': 'bool', + 'CharField': 'varchar(%(max_length)s)', + 'CommaSeparatedIntegerField': 'varchar(%(max_length)s)', + 'DateField': 'date', + 'DateTimeField': 'datetime', + 'DecimalField': 'decimal', + 'FileField': 'varchar(%(max_length)s)', + 'FilePathField': 'varchar(%(max_length)s)', + 'FloatField': 'real', + 'IntegerField': 'integer', + 'IPAddressField': 'char(15)', + 'NullBooleanField': 'bool', + 'OneToOneField': 'integer', + 'PhoneNumberField': 'varchar(20)', + 'PositiveIntegerField': 'integer unsigned', + 'PositiveSmallIntegerField': 'smallint unsigned', + 'SlugField': 'varchar(%(max_length)s)', + 'SmallIntegerField': 'smallint', + 'TextField': 'text', + 'TimeField': 'time', + 'USStateField': 'varchar(2)', + } + + def sql_for_pending_references(self, model, style, pending_references): + "SQLite3 doesn't support constraints" + return [] + + def sql_remove_table_constraints(self, model, references_to_delete, style): + "SQLite3 doesn't support constraints" + return [] + + def _create_test_db(self, verbosity, autoclobber): + if settings.TEST_DATABASE_NAME and settings.TEST_DATABASE_NAME != ":memory:": + test_database_name = settings.TEST_DATABASE_NAME + # Erase the old test database + if verbosity >= 1: + print "Destroying old test database..." + if os.access(test_database_name, os.F_OK): + if not autoclobber: + confirm = raw_input("Type 'yes' if you would like to try deleting the test database '%s', or 'no' to cancel: " % test_database_name) + if autoclobber or confirm == 'yes': + try: + if verbosity >= 1: + print "Destroying old test database..." + os.remove(test_database_name) + except Exception, e: + sys.stderr.write("Got an error deleting the old test database: %s\n" % e) + sys.exit(2) + else: + print "Tests cancelled." + sys.exit(1) + if verbosity >= 1: + print "Creating test database..." + else: + test_database_name = ":memory:" + return test_database_name + + def _destroy_test_db(self, test_database_name, verbosity): + if test_database_name and test_database_name != ":memory:": + # Remove the SQLite database file + os.remove(test_database_name) diff --git a/webapp/django/db/backends/sqlite3/introspection.py b/webapp/django/db/backends/sqlite3/introspection.py new file mode 100644 index 0000000000..5e26f33ea6 --- /dev/null +++ b/webapp/django/db/backends/sqlite3/introspection.py @@ -0,0 +1,89 @@ +from django.db.backends import BaseDatabaseIntrospection + +# This light wrapper "fakes" a dictionary interface, because some SQLite data +# types include variables in them -- e.g. "varchar(30)" -- and can't be matched +# as a simple dictionary lookup. +class FlexibleFieldLookupDict: + # Maps SQL types to Django Field types. Some of the SQL types have multiple + # entries here because SQLite allows for anything and doesn't normalize the + # field type; it uses whatever was given. + base_data_types_reverse = { + 'bool': 'BooleanField', + 'boolean': 'BooleanField', + 'smallint': 'SmallIntegerField', + 'smallinteger': 'SmallIntegerField', + 'int': 'IntegerField', + 'integer': 'IntegerField', + 'text': 'TextField', + 'char': 'CharField', + 'date': 'DateField', + 'datetime': 'DateTimeField', + 'time': 'TimeField', + } + + def __getitem__(self, key): + key = key.lower() + try: + return self.base_data_types_reverse[key] + except KeyError: + import re + m = re.search(r'^\s*(?:var)?char\s*\(\s*(\d+)\s*\)\s*$', key) + if m: + return ('CharField', {'max_length': int(m.group(1))}) + raise KeyError + +class DatabaseIntrospection(BaseDatabaseIntrospection): + data_types_reverse = FlexibleFieldLookupDict() + + def get_table_list(self, cursor): + "Returns a list of table names in the current database." + # Skip the sqlite_sequence system table used for autoincrement key + # generation. + cursor.execute(""" + SELECT name FROM sqlite_master + WHERE type='table' AND NOT name='sqlite_sequence' + ORDER BY name""") + return [row[0] for row in cursor.fetchall()] + + def get_table_description(self, cursor, table_name): + "Returns a description of the table, with the DB-API cursor.description interface." + return [(info['name'], info['type'], None, None, None, None, + info['null_ok']) for info in self._table_info(cursor, table_name)] + + def get_relations(self, cursor, table_name): + raise NotImplementedError + + def get_indexes(self, cursor, table_name): + """ + Returns a dictionary of fieldname -> infodict for the given table, + where each infodict is in the format: + {'primary_key': boolean representing whether it's the primary key, + 'unique': boolean representing whether it's a unique index} + """ + indexes = {} + for info in self._table_info(cursor, table_name): + indexes[info['name']] = {'primary_key': info['pk'] != 0, + 'unique': False} + cursor.execute('PRAGMA index_list(%s)' % self.connection.ops.quote_name(table_name)) + # seq, name, unique + for index, unique in [(field[1], field[2]) for field in cursor.fetchall()]: + if not unique: + continue + cursor.execute('PRAGMA index_info(%s)' % self.connection.ops.quote_name(index)) + info = cursor.fetchall() + # Skip indexes across multiple fields + if len(info) != 1: + continue + name = info[0][2] # seqno, cid, name + indexes[name]['unique'] = True + return indexes + + def _table_info(self, cursor, name): + cursor.execute('PRAGMA table_info(%s)' % self.connection.ops.quote_name(name)) + # cid, name, type, notnull, dflt_value, pk + return [{'name': field[1], + 'type': field[2], + 'null_ok': not field[3], + 'pk': field[5] # undocumented + } for field in cursor.fetchall()] + diff --git a/webapp/django/db/backends/util.py b/webapp/django/db/backends/util.py new file mode 100644 index 0000000000..7228b4046b --- /dev/null +++ b/webapp/django/db/backends/util.py @@ -0,0 +1,127 @@ +import datetime +from time import time + +from django.utils.hashcompat import md5_constructor + +try: + import decimal +except ImportError: + from django.utils import _decimal as decimal # for Python 2.3 + +class CursorDebugWrapper(object): + def __init__(self, cursor, db): + self.cursor = cursor + self.db = db # Instance of a BaseDatabaseWrapper subclass + + def execute(self, sql, params=()): + start = time() + try: + return self.cursor.execute(sql, params) + finally: + stop = time() + sql = self.db.ops.last_executed_query(self.cursor, sql, params) + self.db.queries.append({ + 'sql': sql, + 'time': "%.3f" % (stop - start), + }) + + def executemany(self, sql, param_list): + start = time() + try: + return self.cursor.executemany(sql, param_list) + finally: + stop = time() + self.db.queries.append({ + 'sql': '%s times: %s' % (len(param_list), sql), + 'time': "%.3f" % (stop - start), + }) + + def __getattr__(self, attr): + if attr in self.__dict__: + return self.__dict__[attr] + else: + return getattr(self.cursor, attr) + + def __iter__(self): + return iter(self.cursor) + +############################################### +# Converters from database (string) to Python # +############################################### + +def typecast_date(s): + return s and datetime.date(*map(int, s.split('-'))) or None # returns None if s is null + +def typecast_time(s): # does NOT store time zone information + if not s: return None + hour, minutes, seconds = s.split(':') + if '.' in seconds: # check whether seconds have a fractional part + seconds, microseconds = seconds.split('.') + else: + microseconds = '0' + return datetime.time(int(hour), int(minutes), int(seconds), int(float('.'+microseconds) * 1000000)) + +def typecast_timestamp(s): # does NOT store time zone information + # "2005-07-29 15:48:00.590358-05" + # "2005-07-29 09:56:00-05" + if not s: return None + if not ' ' in s: return typecast_date(s) + d, t = s.split() + # Extract timezone information, if it exists. Currently we just throw + # it away, but in the future we may make use of it. + if '-' in t: + t, tz = t.split('-', 1) + tz = '-' + tz + elif '+' in t: + t, tz = t.split('+', 1) + tz = '+' + tz + else: + tz = '' + dates = d.split('-') + times = t.split(':') + seconds = times[2] + if '.' in seconds: # check whether seconds have a fractional part + seconds, microseconds = seconds.split('.') + else: + microseconds = '0' + return datetime.datetime(int(dates[0]), int(dates[1]), int(dates[2]), + int(times[0]), int(times[1]), int(seconds), int(float('.'+microseconds) * 1000000)) + +def typecast_boolean(s): + if s is None: return None + if not s: return False + return str(s)[0].lower() == 't' + +def typecast_decimal(s): + if s is None or s == '': + return None + return decimal.Decimal(s) + +############################################### +# Converters from Python to database (string) # +############################################### + +def rev_typecast_boolean(obj, d): + return obj and '1' or '0' + +def rev_typecast_decimal(d): + if d is None: + return None + return str(d) + +def truncate_name(name, length=None): + """Shortens a string to a repeatable mangled version with the given length. + """ + if length is None or len(name) <= length: + return name + + hash = md5_constructor(name).hexdigest()[:4] + + return '%s%s' % (name[:length-4], hash) + +def format_number(value, max_digits, decimal_places): + """ + Formats a number into a string with the requisite number of digits and + decimal places. + """ + return u"%.*f" % (decimal_places, value) diff --git a/webapp/django/db/models/__init__.py b/webapp/django/db/models/__init__.py new file mode 100644 index 0000000000..cbd685547e --- /dev/null +++ b/webapp/django/db/models/__init__.py @@ -0,0 +1,32 @@ +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist, ImproperlyConfigured +from django.core import validators +from django.db import connection +from django.db.models.loading import get_apps, get_app, get_models, get_model, register_models +from django.db.models.query import Q +from django.db.models.manager import Manager +from django.db.models.base import Model +from django.db.models.fields import * +from django.db.models.fields.subclassing import SubfieldBase +from django.db.models.fields.files import FileField, ImageField +from django.db.models.fields.related import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel, TABULAR, STACKED +from django.db.models import signals + +# Admin stages. +ADD, CHANGE, BOTH = 1, 2, 3 + +def permalink(func): + """ + Decorator that calls urlresolvers.reverse() to return a URL using + parameters returned by the decorated function "func". + + "func" should be a function that returns a tuple in one of the + following formats: + (viewname, viewargs) + (viewname, viewargs, viewkwargs) + """ + from django.core.urlresolvers import reverse + def inner(*args, **kwargs): + bits = func(*args, **kwargs) + return reverse(bits[0], None, *bits[1:3]) + return inner diff --git a/webapp/django/db/models/base.py b/webapp/django/db/models/base.py new file mode 100644 index 0000000000..115d82bd4f --- /dev/null +++ b/webapp/django/db/models/base.py @@ -0,0 +1,504 @@ +import copy +import types +import sys +import os +from itertools import izip +try: + set +except NameError: + from sets import Set as set # Python 2.3 fallback. + +import django.db.models.manipulators # Imported to register signal handler. +import django.db.models.manager # Ditto. +from django.core import validators +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned, FieldError +from django.db.models.fields import AutoField +from django.db.models.fields.related import OneToOneRel, ManyToOneRel, OneToOneField +from django.db.models.query import delete_objects, Q, CollectedObjects +from django.db.models.options import Options +from django.db import connection, transaction, DatabaseError +from django.db.models import signals +from django.db.models.loading import register_models, get_model +from django.utils.functional import curry +from django.utils.encoding import smart_str, force_unicode, smart_unicode +from django.core.files.move import file_move_safe +from django.core.files import locks +from django.conf import settings + + +class ModelBase(type): + """ + Metaclass for all models. + """ + def __new__(cls, name, bases, attrs): + super_new = super(ModelBase, cls).__new__ + parents = [b for b in bases if isinstance(b, ModelBase)] + if not parents: + # If this isn't a subclass of Model, don't do anything special. + return super_new(cls, name, bases, attrs) + + # Create the class. + module = attrs.pop('__module__') + new_class = super_new(cls, name, bases, {'__module__': module}) + attr_meta = attrs.pop('Meta', None) + abstract = getattr(attr_meta, 'abstract', False) + if not attr_meta: + meta = getattr(new_class, 'Meta', None) + else: + meta = attr_meta + base_meta = getattr(new_class, '_meta', None) + + if getattr(meta, 'app_label', None) is None: + # Figure out the app_label by looking one level up. + # For 'django.contrib.sites.models', this would be 'sites'. + model_module = sys.modules[new_class.__module__] + kwargs = {"app_label": model_module.__name__.split('.')[-2]} + else: + kwargs = {} + + new_class.add_to_class('_meta', Options(meta, **kwargs)) + if not abstract: + new_class.add_to_class('DoesNotExist', + subclass_exception('DoesNotExist', ObjectDoesNotExist, module)) + new_class.add_to_class('MultipleObjectsReturned', + subclass_exception('MultipleObjectsReturned', MultipleObjectsReturned, module)) + if base_meta and not base_meta.abstract: + # Non-abstract child classes inherit some attributes from their + # non-abstract parent (unless an ABC comes before it in the + # method resolution order). + if not hasattr(meta, 'ordering'): + new_class._meta.ordering = base_meta.ordering + if not hasattr(meta, 'get_latest_by'): + new_class._meta.get_latest_by = base_meta.get_latest_by + + old_default_mgr = None + if getattr(new_class, '_default_manager', None): + # We have a parent who set the default manager. + if new_class._default_manager.model._meta.abstract: + old_default_mgr = new_class._default_manager + new_class._default_manager = None + + # Bail out early if we have already created this class. + m = get_model(new_class._meta.app_label, name, False) + if m is not None: + return m + + # Add all attributes to the class. + for obj_name, obj in attrs.items(): + new_class.add_to_class(obj_name, obj) + + # Do the appropriate setup for any model parents. + o2o_map = dict([(f.rel.to, f) for f in new_class._meta.local_fields + if isinstance(f, OneToOneField)]) + for base in parents: + if not hasattr(base, '_meta'): + # Things without _meta aren't functional models, so they're + # uninteresting parents. + continue + if not base._meta.abstract: + if base in o2o_map: + field = o2o_map[base] + field.primary_key = True + new_class._meta.setup_pk(field) + else: + attr_name = '%s_ptr' % base._meta.module_name + field = OneToOneField(base, name=attr_name, + auto_created=True, parent_link=True) + new_class.add_to_class(attr_name, field) + new_class._meta.parents[base] = field + else: + # The abstract base class case. + names = set([f.name for f in new_class._meta.local_fields + new_class._meta.many_to_many]) + for field in base._meta.local_fields + base._meta.local_many_to_many: + if field.name in names: + raise FieldError('Local field %r in class %r clashes with field of similar name from abstract base class %r' + % (field.name, name, base.__name__)) + new_class.add_to_class(field.name, copy.deepcopy(field)) + + if abstract: + # Abstract base models can't be instantiated and don't appear in + # the list of models for an app. We do the final setup for them a + # little differently from normal models. + attr_meta.abstract = False + new_class.Meta = attr_meta + return new_class + + if old_default_mgr and not new_class._default_manager: + new_class._default_manager = old_default_mgr._copy_to_model(new_class) + new_class._prepare() + register_models(new_class._meta.app_label, new_class) + + # Because of the way imports happen (recursively), we may or may not be + # the first time this model tries to register with the framework. There + # should only be one class for each model, so we always return the + # registered version. + return get_model(new_class._meta.app_label, name, False) + + def add_to_class(cls, name, value): + if hasattr(value, 'contribute_to_class'): + value.contribute_to_class(cls, name) + else: + setattr(cls, name, value) + + def _prepare(cls): + """ + Creates some methods once self._meta has been populated. + """ + opts = cls._meta + opts._prepare(cls) + + if opts.order_with_respect_to: + cls.get_next_in_order = curry(cls._get_next_or_previous_in_order, is_next=True) + cls.get_previous_in_order = curry(cls._get_next_or_previous_in_order, is_next=False) + setattr(opts.order_with_respect_to.rel.to, 'get_%s_order' % cls.__name__.lower(), curry(method_get_order, cls)) + setattr(opts.order_with_respect_to.rel.to, 'set_%s_order' % cls.__name__.lower(), curry(method_set_order, cls)) + + # Give the class a docstring -- its definition. + if cls.__doc__ is None: + cls.__doc__ = "%s(%s)" % (cls.__name__, ", ".join([f.attname for f in opts.fields])) + + if hasattr(cls, 'get_absolute_url'): + cls.get_absolute_url = curry(get_absolute_url, opts, cls.get_absolute_url) + + signals.class_prepared.send(sender=cls) + + +class Model(object): + __metaclass__ = ModelBase + + def __init__(self, *args, **kwargs): + signals.pre_init.send(sender=self.__class__, args=args, kwargs=kwargs) + + # There is a rather weird disparity here; if kwargs, it's set, then args + # overrides it. It should be one or the other; don't duplicate the work + # The reason for the kwargs check is that standard iterator passes in by + # args, and instantiation for iteration is 33% faster. + args_len = len(args) + if args_len > len(self._meta.fields): + # Daft, but matches old exception sans the err msg. + raise IndexError("Number of args exceeds number of fields") + + fields_iter = iter(self._meta.fields) + if not kwargs: + # The ordering of the izip calls matter - izip throws StopIteration + # when an iter throws it. So if the first iter throws it, the second + # is *not* consumed. We rely on this, so don't change the order + # without changing the logic. + for val, field in izip(args, fields_iter): + setattr(self, field.attname, val) + else: + # Slower, kwargs-ready version. + for val, field in izip(args, fields_iter): + setattr(self, field.attname, val) + kwargs.pop(field.name, None) + # Maintain compatibility with existing calls. + if isinstance(field.rel, ManyToOneRel): + kwargs.pop(field.attname, None) + + # Now we're left with the unprocessed fields that *must* come from + # keywords, or default. + + for field in fields_iter: + rel_obj = None + if kwargs: + if isinstance(field.rel, ManyToOneRel): + try: + # Assume object instance was passed in. + rel_obj = kwargs.pop(field.name) + except KeyError: + try: + # Object instance wasn't passed in -- must be an ID. + val = kwargs.pop(field.attname) + except KeyError: + val = field.get_default() + else: + # Object instance was passed in. Special case: You can + # pass in "None" for related objects if it's allowed. + if rel_obj is None and field.null: + val = None + else: + val = kwargs.pop(field.attname, field.get_default()) + else: + val = field.get_default() + # If we got passed a related instance, set it using the field.name + # instead of field.attname (e.g. "user" instead of "user_id") so + # that the object gets properly cached (and type checked) by the + # RelatedObjectDescriptor. + if rel_obj: + setattr(self, field.name, rel_obj) + else: + setattr(self, field.attname, val) + + if kwargs: + for prop in kwargs.keys(): + try: + if isinstance(getattr(self.__class__, prop), property): + setattr(self, prop, kwargs.pop(prop)) + except AttributeError: + pass + if kwargs: + raise TypeError, "'%s' is an invalid keyword argument for this function" % kwargs.keys()[0] + signals.post_init.send(sender=self.__class__, instance=self) + + def __repr__(self): + return smart_str(u'<%s: %s>' % (self.__class__.__name__, unicode(self))) + + def __str__(self): + if hasattr(self, '__unicode__'): + return force_unicode(self).encode('utf-8') + return '%s object' % self.__class__.__name__ + + def __eq__(self, other): + return isinstance(other, self.__class__) and self._get_pk_val() == other._get_pk_val() + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self._get_pk_val()) + + def _get_pk_val(self, meta=None): + if not meta: + meta = self._meta + return getattr(self, meta.pk.attname) + + def _set_pk_val(self, value): + return setattr(self, self._meta.pk.attname, value) + + pk = property(_get_pk_val, _set_pk_val) + + def save(self, force_insert=False, force_update=False): + """ + Saves the current instance. Override this in a subclass if you want to + control the saving process. + + The 'force_insert' and 'force_update' parameters can be used to insist + that the "save" must be an SQL insert or update (or equivalent for + non-SQL backends), respectively. Normally, they should not be set. + """ + if force_insert and force_update: + raise ValueError("Cannot force both insert and updating in " + "model saving.") + self.save_base(force_insert=force_insert, force_update=force_update) + + save.alters_data = True + + def save_base(self, raw=False, cls=None, force_insert=False, + force_update=False): + """ + Does the heavy-lifting involved in saving. Subclasses shouldn't need to + override this method. It's separate from save() in order to hide the + need for overrides of save() to pass around internal-only parameters + ('raw' and 'cls'). + """ + assert not (force_insert and force_update) + if not cls: + cls = self.__class__ + meta = self._meta + signal = True + signals.pre_save.send(sender=self.__class__, instance=self, raw=raw) + else: + meta = cls._meta + signal = False + + # If we are in a raw save, save the object exactly as presented. + # That means that we don't try to be smart about saving attributes + # that might have come from the parent class - we just save the + # attributes we have been given to the class we have been given. + if not raw: + for parent, field in meta.parents.items(): + # At this point, parent's primary key field may be unknown + # (for example, from administration form which doesn't fill + # this field). If so, fill it. + if getattr(self, parent._meta.pk.attname) is None and getattr(self, field.attname) is not None: + setattr(self, parent._meta.pk.attname, getattr(self, field.attname)) + + self.save_base(raw, parent) + setattr(self, field.attname, self._get_pk_val(parent._meta)) + + non_pks = [f for f in meta.local_fields if not f.primary_key] + + # First, try an UPDATE. If that doesn't update anything, do an INSERT. + pk_val = self._get_pk_val(meta) + # Note: the comparison with '' is required for compatibility with + # oldforms-style model creation. + pk_set = pk_val is not None and smart_unicode(pk_val) != u'' + record_exists = True + manager = cls._default_manager + if pk_set: + # Determine whether a record with the primary key already exists. + if (force_update or (not force_insert and + manager.filter(pk=pk_val).extra(select={'a': 1}).values('a').order_by())): + # It does already exist, so do an UPDATE. + if force_update or non_pks: + values = [(f, None, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, False))) for f in non_pks] + rows = manager.filter(pk=pk_val)._update(values) + if force_update and not rows: + raise DatabaseError("Forced update did not affect any rows.") + else: + record_exists = False + if not pk_set or not record_exists: + if not pk_set: + if force_update: + raise ValueError("Cannot force an update in save() with no primary key.") + values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True))) for f in meta.local_fields if not isinstance(f, AutoField)] + else: + values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True))) for f in meta.local_fields] + + if meta.order_with_respect_to: + field = meta.order_with_respect_to + values.append((meta.get_field_by_name('_order')[0], manager.filter(**{field.name: getattr(self, field.attname)}).count())) + record_exists = False + + update_pk = bool(meta.has_auto_field and not pk_set) + if values: + # Create a new record. + result = manager._insert(values, return_id=update_pk) + else: + # Create a new record with defaults for everything. + result = manager._insert([(meta.pk, connection.ops.pk_default_value())], return_id=update_pk, raw_values=True) + + if update_pk: + setattr(self, meta.pk.attname, result) + transaction.commit_unless_managed() + + if signal: + signals.post_save.send(sender=self.__class__, instance=self, + created=(not record_exists), raw=raw) + + save_base.alters_data = True + + def _collect_sub_objects(self, seen_objs, parent=None, nullable=False): + """ + Recursively populates seen_objs with all objects related to this + object. + + When done, seen_objs.items() will be in the format: + [(model_class, {pk_val: obj, pk_val: obj, ...}), + (model_class, {pk_val: obj, pk_val: obj, ...}), ...] + """ + pk_val = self._get_pk_val() + if seen_objs.add(self.__class__, pk_val, self, parent, nullable): + return + + for related in self._meta.get_all_related_objects(): + rel_opts_name = related.get_accessor_name() + if isinstance(related.field.rel, OneToOneRel): + try: + sub_obj = getattr(self, rel_opts_name) + except ObjectDoesNotExist: + pass + else: + sub_obj._collect_sub_objects(seen_objs, self.__class__, related.field.null) + else: + for sub_obj in getattr(self, rel_opts_name).all(): + sub_obj._collect_sub_objects(seen_objs, self.__class__, related.field.null) + + # Handle any ancestors (for the model-inheritance case). We do this by + # traversing to the most remote parent classes -- those with no parents + # themselves -- and then adding those instances to the collection. That + # will include all the child instances down to "self". + parent_stack = self._meta.parents.values() + while parent_stack: + link = parent_stack.pop() + parent_obj = getattr(self, link.name) + if parent_obj._meta.parents: + parent_stack.extend(parent_obj._meta.parents.values()) + continue + # At this point, parent_obj is base class (no ancestor models). So + # delete it and all its descendents. + parent_obj._collect_sub_objects(seen_objs) + + def delete(self): + assert self._get_pk_val() is not None, "%s object can't be deleted because its %s attribute is set to None." % (self._meta.object_name, self._meta.pk.attname) + + # Find all the objects than need to be deleted. + seen_objs = CollectedObjects() + self._collect_sub_objects(seen_objs) + + # Actually delete the objects. + delete_objects(seen_objs) + + delete.alters_data = True + + def _get_FIELD_display(self, field): + value = getattr(self, field.attname) + return force_unicode(dict(field.flatchoices).get(value, value), strings_only=True) + + def _get_next_or_previous_by_FIELD(self, field, is_next, **kwargs): + op = is_next and 'gt' or 'lt' + order = not is_next and '-' or '' + param = smart_str(getattr(self, field.attname)) + q = Q(**{'%s__%s' % (field.name, op): param}) + q = q|Q(**{field.name: param, 'pk__%s' % op: self.pk}) + qs = self.__class__._default_manager.filter(**kwargs).filter(q).order_by('%s%s' % (order, field.name), '%spk' % order) + try: + return qs[0] + except IndexError: + raise self.DoesNotExist, "%s matching query does not exist." % self.__class__._meta.object_name + + def _get_next_or_previous_in_order(self, is_next): + cachename = "__%s_order_cache" % is_next + if not hasattr(self, cachename): + qn = connection.ops.quote_name + op = is_next and '>' or '<' + order = not is_next and '-_order' or '_order' + order_field = self._meta.order_with_respect_to + # FIXME: When querysets support nested queries, this can be turned + # into a pure queryset operation. + where = ['%s %s (SELECT %s FROM %s WHERE %s=%%s)' % \ + (qn('_order'), op, qn('_order'), + qn(self._meta.db_table), qn(self._meta.pk.column))] + params = [self.pk] + obj = self._default_manager.filter(**{order_field.name: getattr(self, order_field.attname)}).extra(where=where, params=params).order_by(order)[:1].get() + setattr(self, cachename, obj) + return getattr(self, cachename) + + + +############################################ +# HELPER FUNCTIONS (CURRIED MODEL METHODS) # +############################################ + +# ORDERING METHODS ######################### + +def method_set_order(ordered_obj, self, id_list): + rel_val = getattr(self, ordered_obj._meta.order_with_respect_to.rel.field_name) + order_name = ordered_obj._meta.order_with_respect_to.name + # FIXME: It would be nice if there was an "update many" version of update + # for situations like this. + for i, j in enumerate(id_list): + ordered_obj.objects.filter(**{'pk': j, order_name: rel_val}).update(_order=i) + transaction.commit_unless_managed() + + +def method_get_order(ordered_obj, self): + rel_val = getattr(self, ordered_obj._meta.order_with_respect_to.rel.field_name) + order_name = ordered_obj._meta.order_with_respect_to.name + pk_name = ordered_obj._meta.pk.name + return [r[pk_name] for r in + ordered_obj.objects.filter(**{order_name: rel_val}).values(pk_name)] + + +############################################## +# HELPER FUNCTIONS (CURRIED MODEL FUNCTIONS) # +############################################## + +def get_absolute_url(opts, func, self, *args, **kwargs): + return settings.ABSOLUTE_URL_OVERRIDES.get('%s.%s' % (opts.app_label, opts.module_name), func)(self, *args, **kwargs) + + +######## +# MISC # +######## + +class Empty(object): + pass + +if sys.version_info < (2, 5): + # Prior to Python 2.5, Exception was an old-style class + def subclass_exception(name, parent, unused): + return types.ClassType(name, (parent,), {}) +else: + def subclass_exception(name, parent, module): + return type(name, (parent,), {'__module__': module}) diff --git a/webapp/django/db/models/fields/__init__.py b/webapp/django/db/models/fields/__init__.py new file mode 100644 index 0000000000..b8d05671b1 --- /dev/null +++ b/webapp/django/db/models/fields/__init__.py @@ -0,0 +1,1035 @@ +import copy +import datetime +import os +import time +try: + import decimal +except ImportError: + from django.utils import _decimal as decimal # for Python 2.3 + +from django.db import connection +from django.db.models import signals +from django.db.models.query_utils import QueryWrapper +from django.dispatch import dispatcher +from django.conf import settings +from django.core import validators +from django import oldforms +from django import forms +from django.core.exceptions import ObjectDoesNotExist +from django.utils.datastructures import DictWrapper +from django.utils.functional import curry +from django.utils.itercompat import tee +from django.utils.text import capfirst +from django.utils.translation import ugettext_lazy, ugettext as _ +from django.utils.encoding import smart_unicode, force_unicode, smart_str +from django.utils import datetime_safe + +class NOT_PROVIDED: + pass + +# The values to use for "blank" in SelectFields. Will be appended to the start of most "choices" lists. +BLANK_CHOICE_DASH = [("", "---------")] +BLANK_CHOICE_NONE = [("", "None")] + +class FieldDoesNotExist(Exception): + pass + +def manipulator_validator_unique(f, opts, self, field_data, all_data): + "Validates that the value is unique for this field." + lookup_type = f.get_validator_unique_lookup_type() + try: + old_obj = self.manager.get(**{lookup_type: field_data}) + except ObjectDoesNotExist: + return + if getattr(self, 'original_object', None) and self.original_object._get_pk_val() == old_obj._get_pk_val(): + return + raise validators.ValidationError, _("%(optname)s with this %(fieldname)s already exists.") % {'optname': capfirst(opts.verbose_name), 'fieldname': f.verbose_name} + +# A guide to Field parameters: +# +# * name: The name of the field specifed in the model. +# * attname: The attribute to use on the model object. This is the same as +# "name", except in the case of ForeignKeys, where "_id" is +# appended. +# * db_column: The db_column specified in the model (or None). +# * column: The database column for this field. This is the same as +# "attname", except if db_column is specified. +# +# Code that introspects values, or does other dynamic things, should use +# attname. For example, this gets the primary key value of object "obj": +# +# getattr(obj, opts.pk.attname) + +class Field(object): + # Designates whether empty strings fundamentally are allowed at the + # database level. + empty_strings_allowed = True + + # These track each time a Field instance is created. Used to retain order. + # The auto_creation_counter is used for fields that Django implicitly + # creates, creation_counter is used for all user-specified fields. + creation_counter = 0 + auto_creation_counter = -1 + + def __init__(self, verbose_name=None, name=None, primary_key=False, + max_length=None, unique=False, blank=False, null=False, + db_index=False, core=False, rel=None, default=NOT_PROVIDED, + editable=True, serialize=True, unique_for_date=None, + unique_for_month=None, unique_for_year=None, validator_list=None, + choices=None, help_text='', db_column=None, db_tablespace=None, + auto_created=False): + self.name = name + self.verbose_name = verbose_name + self.primary_key = primary_key + self.max_length, self._unique = max_length, unique + self.blank, self.null = blank, null + # Oracle treats the empty string ('') as null, so coerce the null + # option whenever '' is a possible value. + if self.empty_strings_allowed and connection.features.interprets_empty_strings_as_nulls: + self.null = True + self.core, self.rel, self.default = core, rel, default + self.editable = editable + self.serialize = serialize + self.validator_list = validator_list or [] + self.unique_for_date, self.unique_for_month = unique_for_date, unique_for_month + self.unique_for_year = unique_for_year + self._choices = choices or [] + self.help_text = help_text + self.db_column = db_column + self.db_tablespace = db_tablespace or settings.DEFAULT_INDEX_TABLESPACE + self.auto_created = auto_created + + # Set db_index to True if the field has a relationship and doesn't explicitly set db_index. + self.db_index = db_index + + # Adjust the appropriate creation counter, and save our local copy. + if auto_created: + self.creation_counter = Field.auto_creation_counter + Field.auto_creation_counter -= 1 + else: + self.creation_counter = Field.creation_counter + Field.creation_counter += 1 + + def __cmp__(self, other): + # This is needed because bisect does not take a comparison function. + return cmp(self.creation_counter, other.creation_counter) + + def __deepcopy__(self, memodict): + # We don't have to deepcopy very much here, since most things are not + # intended to be altered after initial creation. + obj = copy.copy(self) + if self.rel: + obj.rel = copy.copy(self.rel) + memodict[id(self)] = obj + return obj + + def to_python(self, value): + """ + Converts the input value into the expected Python data type, raising + validators.ValidationError if the data can't be converted. Returns the + converted value. Subclasses should override this. + """ + return value + + def db_type(self): + """ + Returns the database column data type for this field, taking into + account the DATABASE_ENGINE setting. + """ + # The default implementation of this method looks at the + # backend-specific DATA_TYPES dictionary, looking up the field by its + # "internal type". + # + # A Field class can implement the get_internal_type() method to specify + # which *preexisting* Django Field class it's most similar to -- i.e., + # an XMLField is represented by a TEXT column type, which is the same + # as the TextField Django field type, which means XMLField's + # get_internal_type() returns 'TextField'. + # + # But the limitation of the get_internal_type() / data_types approach + # is that it cannot handle database column types that aren't already + # mapped to one of the built-in Django field types. In this case, you + # can implement db_type() instead of get_internal_type() to specify + # exactly which wacky database column type you want to use. + data = DictWrapper(self.__dict__, connection.ops.quote_name, "qn_") + try: + return connection.creation.data_types[self.get_internal_type()] % data + except KeyError: + return None + + def unique(self): + return self._unique or self.primary_key + unique = property(unique) + + def set_attributes_from_name(self, name): + self.name = name + self.attname, self.column = self.get_attname_column() + if self.verbose_name is None and name: + self.verbose_name = name.replace('_', ' ') + + def contribute_to_class(self, cls, name): + self.set_attributes_from_name(name) + cls._meta.add_field(self) + if self.choices: + setattr(cls, 'get_%s_display' % self.name, curry(cls._get_FIELD_display, field=self)) + + def get_attname(self): + return self.name + + def get_attname_column(self): + attname = self.get_attname() + column = self.db_column or attname + return attname, column + + def get_cache_name(self): + return '_%s_cache' % self.name + + def get_internal_type(self): + return self.__class__.__name__ + + def pre_save(self, model_instance, add): + "Returns field's value just before saving." + return getattr(model_instance, self.attname) + + def get_db_prep_value(self, value): + """Returns field's value prepared for interacting with the database + backend. + + Used by the default implementations of ``get_db_prep_save``and + `get_db_prep_lookup``` + """ + return value + + def get_db_prep_save(self, value): + "Returns field's value prepared for saving into a database." + return self.get_db_prep_value(value) + + def get_db_prep_lookup(self, lookup_type, value): + "Returns field's value prepared for database lookup." + if hasattr(value, 'as_sql'): + sql, params = value.as_sql() + return QueryWrapper(('(%s)' % sql), params) + if lookup_type in ('regex', 'iregex', 'month', 'day', 'search'): + return [value] + elif lookup_type in ('exact', 'gt', 'gte', 'lt', 'lte'): + return [self.get_db_prep_value(value)] + elif lookup_type in ('range', 'in'): + return [self.get_db_prep_value(v) for v in value] + elif lookup_type in ('contains', 'icontains'): + return ["%%%s%%" % connection.ops.prep_for_like_query(value)] + elif lookup_type == 'iexact': + return [connection.ops.prep_for_like_query(value)] + elif lookup_type in ('startswith', 'istartswith'): + return ["%s%%" % connection.ops.prep_for_like_query(value)] + elif lookup_type in ('endswith', 'iendswith'): + return ["%%%s" % connection.ops.prep_for_like_query(value)] + elif lookup_type == 'isnull': + return [] + elif lookup_type == 'year': + try: + value = int(value) + except ValueError: + raise ValueError("The __year lookup type requires an integer argument") + + if self.get_internal_type() == 'DateField': + return connection.ops.year_lookup_bounds_for_date_field(value) + else: + return connection.ops.year_lookup_bounds(value) + + raise TypeError("Field has invalid lookup: %s" % lookup_type) + + def has_default(self): + "Returns a boolean of whether this field has a default value." + return self.default is not NOT_PROVIDED + + def get_default(self): + "Returns the default value for this field." + if self.default is not NOT_PROVIDED: + if callable(self.default): + return self.default() + return force_unicode(self.default, strings_only=True) + if not self.empty_strings_allowed or (self.null and not connection.features.interprets_empty_strings_as_nulls): + return None + return "" + + def get_manipulator_field_names(self, name_prefix): + """ + Returns a list of field names that this object adds to the manipulator. + """ + return [name_prefix + self.name] + + def prepare_field_objs_and_params(self, manipulator, name_prefix): + params = {'validator_list': self.validator_list[:]} + if self.max_length and not self.choices: # Don't give SelectFields a max_length parameter. + params['max_length'] = self.max_length + + if self.choices: + field_objs = [oldforms.SelectField] + + params['choices'] = self.get_flatchoices() + else: + field_objs = self.get_manipulator_field_objs() + return (field_objs, params) + + def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True): + """ + Returns a list of oldforms.FormField instances for this field. It + calculates the choices at runtime, not at compile time. + + name_prefix is a prefix to prepend to the "field_name" argument. + rel is a boolean specifying whether this field is in a related context. + """ + field_objs, params = self.prepare_field_objs_and_params(manipulator, name_prefix) + + # Add the "unique" validator(s). + for field_name_list in opts.unique_together: + if field_name_list[0] == self.name: + params['validator_list'].append(getattr(manipulator, 'isUnique%s' % '_'.join(field_name_list))) + + # Add the "unique for..." validator(s). + if self.unique_for_date: + params['validator_list'].append(getattr(manipulator, 'isUnique%sFor%s' % (self.name, self.unique_for_date))) + if self.unique_for_month: + params['validator_list'].append(getattr(manipulator, 'isUnique%sFor%s' % (self.name, self.unique_for_month))) + if self.unique_for_year: + params['validator_list'].append(getattr(manipulator, 'isUnique%sFor%s' % (self.name, self.unique_for_year))) + if self.unique and not rel: + params['validator_list'].append(curry(manipulator_validator_unique, self, opts, manipulator)) + + # Only add is_required=True if the field cannot be blank. Primary keys + # are a special case, and fields in a related context should set this + # as False, because they'll be caught by a separate validator -- + # RequiredIfOtherFieldGiven. + params['is_required'] = not self.blank and not self.primary_key and not rel + + # BooleanFields (CheckboxFields) are a special case. They don't take + # is_required. + if isinstance(self, BooleanField): + del params['is_required'] + + # If this field is in a related context, check whether any other fields + # in the related object have core=True. If so, add a validator -- + # RequiredIfOtherFieldsGiven -- to this FormField. + if rel and not self.blank and not isinstance(self, AutoField) and not isinstance(self, FileField): + # First, get the core fields, if any. + core_field_names = [] + for f in opts.fields: + if f.core and f != self: + core_field_names.extend(f.get_manipulator_field_names(name_prefix)) + # Now, if there are any, add the validator to this FormField. + if core_field_names: + params['validator_list'].append(validators.RequiredIfOtherFieldsGiven(core_field_names, ugettext_lazy("This field is required."))) + + # Finally, add the field_names. + field_names = self.get_manipulator_field_names(name_prefix) + return [man(field_name=field_names[i], **params) for i, man in enumerate(field_objs)] + + def get_validator_unique_lookup_type(self): + return '%s__exact' % self.name + + def get_manipulator_new_data(self, new_data, rel=False): + """ + Given the full new_data dictionary (from the manipulator), returns this + field's data. + """ + if rel: + return new_data.get(self.name, [self.get_default()])[0] + val = new_data.get(self.name, self.get_default()) + if not self.empty_strings_allowed and val == '' and self.null: + val = None + return val + + def get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH): + """Returns choices with a default blank choices included, for use + as SelectField choices for this field.""" + first_choice = include_blank and blank_choice or [] + if self.choices: + return first_choice + list(self.choices) + rel_model = self.rel.to + if hasattr(self.rel, 'get_related_field'): + lst = [(getattr(x, self.rel.get_related_field().attname), smart_unicode(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)] + else: + lst = [(x._get_pk_val(), smart_unicode(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)] + return first_choice + lst + + def get_choices_default(self): + return self.get_choices() + + def get_flatchoices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH): + "Returns flattened choices with a default blank choice included." + first_choice = include_blank and blank_choice or [] + return first_choice + list(self.flatchoices) + + def _get_val_from_obj(self, obj): + if obj: + return getattr(obj, self.attname) + else: + return self.get_default() + + def flatten_data(self, follow, obj=None): + """ + Returns a dictionary mapping the field's manipulator field names to its + "flattened" string values for the admin view. obj is the instance to + extract the values from. + """ + return {self.attname: self._get_val_from_obj(obj)} + + def get_follow(self, override=None): + if override != None: + return override + else: + return self.editable + + def bind(self, fieldmapping, original, bound_field_class): + return bound_field_class(self, fieldmapping, original) + + def _get_choices(self): + if hasattr(self._choices, 'next'): + choices, self._choices = tee(self._choices) + return choices + else: + return self._choices + choices = property(_get_choices) + + def _get_flatchoices(self): + """Flattened version of choices tuple.""" + flat = [] + for choice, value in self.choices: + if type(value) in (list, tuple): + flat.extend(value) + else: + flat.append((choice,value)) + return flat + flatchoices = property(_get_flatchoices) + + def save_form_data(self, instance, data): + setattr(instance, self.name, data) + + def formfield(self, form_class=forms.CharField, **kwargs): + "Returns a django.forms.Field instance for this database Field." + defaults = {'required': not self.blank, 'label': capfirst(self.verbose_name), 'help_text': self.help_text} + if self.choices: + defaults['widget'] = forms.Select(choices=self.get_choices(include_blank=self.blank or not (self.has_default() or 'initial' in kwargs))) + if self.has_default(): + defaults['initial'] = self.get_default() + defaults.update(kwargs) + return form_class(**defaults) + + def value_from_object(self, obj): + "Returns the value of this field in the given model instance." + return getattr(obj, self.attname) + +class AutoField(Field): + empty_strings_allowed = False + def __init__(self, *args, **kwargs): + assert kwargs.get('primary_key', False) is True, "%ss must have primary_key=True." % self.__class__.__name__ + kwargs['blank'] = True + Field.__init__(self, *args, **kwargs) + + def to_python(self, value): + if value is None: + return value + try: + return int(value) + except (TypeError, ValueError): + raise validators.ValidationError, _("This value must be an integer.") + + def get_db_prep_value(self, value): + if value is None: + return None + return int(value) + + def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True): + if not rel: + return [] # Don't add a FormField unless it's in a related context. + return Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow) + + def get_manipulator_field_objs(self): + return [oldforms.HiddenField] + + def get_manipulator_new_data(self, new_data, rel=False): + # Never going to be called + # Not in main change pages + # ignored in related context + if not rel: + return None + return Field.get_manipulator_new_data(self, new_data, rel) + + def contribute_to_class(self, cls, name): + assert not cls._meta.has_auto_field, "A model can't have more than one AutoField." + super(AutoField, self).contribute_to_class(cls, name) + cls._meta.has_auto_field = True + cls._meta.auto_field = self + + def formfield(self, **kwargs): + return None + +class BooleanField(Field): + def __init__(self, *args, **kwargs): + kwargs['blank'] = True + if 'default' not in kwargs and not kwargs.get('null'): + kwargs['default'] = False + Field.__init__(self, *args, **kwargs) + + def get_internal_type(self): + return "BooleanField" + + def to_python(self, value): + if value in (True, False): return value + if value in ('t', 'True', '1'): return True + if value in ('f', 'False', '0'): return False + raise validators.ValidationError, _("This value must be either True or False.") + + def get_db_prep_value(self, value): + if value is None: + return None + return bool(value) + + def get_manipulator_field_objs(self): + return [oldforms.CheckboxField] + + def formfield(self, **kwargs): + defaults = {'form_class': forms.BooleanField} + defaults.update(kwargs) + return super(BooleanField, self).formfield(**defaults) + +class CharField(Field): + def get_manipulator_field_objs(self): + return [oldforms.TextField] + + def get_internal_type(self): + return "CharField" + + def to_python(self, value): + if isinstance(value, basestring): + return value + if value is None: + if self.null: + return value + else: + raise validators.ValidationError, ugettext_lazy("This field cannot be null.") + return smart_unicode(value) + + def formfield(self, **kwargs): + defaults = {'max_length': self.max_length} + defaults.update(kwargs) + return super(CharField, self).formfield(**defaults) + +# TODO: Maybe move this into contrib, because it's specialized. +class CommaSeparatedIntegerField(CharField): + def get_manipulator_field_objs(self): + return [oldforms.CommaSeparatedIntegerField] + +class DateField(Field): + empty_strings_allowed = False + def __init__(self, verbose_name=None, name=None, auto_now=False, auto_now_add=False, **kwargs): + self.auto_now, self.auto_now_add = auto_now, auto_now_add + #HACKs : auto_now_add/auto_now should be done as a default or a pre_save. + if auto_now or auto_now_add: + kwargs['editable'] = False + kwargs['blank'] = True + Field.__init__(self, verbose_name, name, **kwargs) + + def get_internal_type(self): + return "DateField" + + def to_python(self, value): + if value is None: + return value + if isinstance(value, datetime.datetime): + return value.date() + if isinstance(value, datetime.date): + return value + validators.isValidANSIDate(value, None) + try: + return datetime.date(*time.strptime(value, '%Y-%m-%d')[:3]) + except ValueError: + raise validators.ValidationError, _('Enter a valid date in YYYY-MM-DD format.') + + def pre_save(self, model_instance, add): + if self.auto_now or (self.auto_now_add and add): + value = datetime.datetime.now() + setattr(model_instance, self.attname, value) + return value + else: + return super(DateField, self).pre_save(model_instance, add) + + def contribute_to_class(self, cls, name): + super(DateField,self).contribute_to_class(cls, name) + if not self.null: + setattr(cls, 'get_next_by_%s' % self.name, + curry(cls._get_next_or_previous_by_FIELD, field=self, is_next=True)) + setattr(cls, 'get_previous_by_%s' % self.name, + curry(cls._get_next_or_previous_by_FIELD, field=self, is_next=False)) + + # Needed because of horrible auto_now[_add] behaviour wrt. editable + def get_follow(self, override=None): + if override != None: + return override + else: + return self.editable or self.auto_now or self.auto_now_add + + def get_db_prep_lookup(self, lookup_type, value): + # For "__month" and "__day" lookups, convert the value to a string so + # the database backend always sees a consistent type. + if lookup_type in ('month', 'day'): + return [force_unicode(value)] + return super(DateField, self).get_db_prep_lookup(lookup_type, value) + + def get_db_prep_value(self, value): + # Casts dates into the format expected by the backend + return connection.ops.value_to_db_date(self.to_python(value)) + + def get_manipulator_field_objs(self): + return [oldforms.DateField] + + def flatten_data(self, follow, obj=None): + val = self._get_val_from_obj(obj) + if val is None: + data = '' + else: + data = datetime_safe.new_date(val).strftime("%Y-%m-%d") + return {self.attname: data} + + def formfield(self, **kwargs): + defaults = {'form_class': forms.DateField} + defaults.update(kwargs) + return super(DateField, self).formfield(**defaults) + +class DateTimeField(DateField): + def get_internal_type(self): + return "DateTimeField" + + def to_python(self, value): + if value is None: + return value + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime(value.year, value.month, value.day) + + # Attempt to parse a datetime: + value = smart_str(value) + # split usecs, because they are not recognized by strptime. + if '.' in value: + try: + value, usecs = value.split('.') + usecs = int(usecs) + except ValueError: + raise validators.ValidationError, _('Enter a valid date/time in YYYY-MM-DD HH:MM[:ss[.uuuuuu]] format.') + else: + usecs = 0 + kwargs = {'microsecond': usecs} + try: # Seconds are optional, so try converting seconds first. + return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6], + **kwargs) + + except ValueError: + try: # Try without seconds. + return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5], + **kwargs) + except ValueError: # Try without hour/minutes/seconds. + try: + return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3], + **kwargs) + except ValueError: + raise validators.ValidationError, _('Enter a valid date/time in YYYY-MM-DD HH:MM[:ss[.uuuuuu]] format.') + + def get_db_prep_value(self, value): + # Casts dates into the format expected by the backend + return connection.ops.value_to_db_datetime(self.to_python(value)) + + def get_manipulator_field_objs(self): + return [oldforms.DateField, oldforms.TimeField] + + def get_manipulator_field_names(self, name_prefix): + return [name_prefix + self.name + '_date', name_prefix + self.name + '_time'] + + def get_manipulator_new_data(self, new_data, rel=False): + date_field, time_field = self.get_manipulator_field_names('') + if rel: + d = new_data.get(date_field, [None])[0] + t = new_data.get(time_field, [None])[0] + else: + d = new_data.get(date_field, None) + t = new_data.get(time_field, None) + if d is not None and t is not None: + return datetime.datetime.combine(d, t) + return self.get_default() + + def flatten_data(self,follow, obj = None): + val = self._get_val_from_obj(obj) + date_field, time_field = self.get_manipulator_field_names('') + if val is None: + date_data = time_data = '' + else: + d = datetime_safe.new_datetime(val) + date_data = d.strftime('%Y-%m-%d') + time_data = d.strftime('%H:%M:%S') + return {date_field: date_data, time_field: time_data} + + def formfield(self, **kwargs): + defaults = {'form_class': forms.DateTimeField} + defaults.update(kwargs) + return super(DateTimeField, self).formfield(**defaults) + +class DecimalField(Field): + empty_strings_allowed = False + def __init__(self, verbose_name=None, name=None, max_digits=None, decimal_places=None, **kwargs): + self.max_digits, self.decimal_places = max_digits, decimal_places + Field.__init__(self, verbose_name, name, **kwargs) + + def get_internal_type(self): + return "DecimalField" + + def to_python(self, value): + if value is None: + return value + try: + return decimal.Decimal(value) + except decimal.InvalidOperation: + raise validators.ValidationError( + _("This value must be a decimal number.")) + + def _format(self, value): + if isinstance(value, basestring) or value is None: + return value + else: + return self.format_number(value) + + def format_number(self, value): + """ + Formats a number into a string with the requisite number of digits and + decimal places. + """ + # Method moved to django.db.backends.util. + # + # It is preserved because it is used by the oracle backend + # (django.db.backends.oracle.query), and also for + # backwards-compatibility with any external code which may have used + # this method. + from django.db.backends import util + return util.format_number(value, self.max_digits, self.decimal_places) + + def get_db_prep_value(self, value): + return connection.ops.value_to_db_decimal(self.to_python(value), + self.max_digits, self.decimal_places) + + def get_manipulator_field_objs(self): + return [curry(oldforms.DecimalField, max_digits=self.max_digits, decimal_places=self.decimal_places)] + + def formfield(self, **kwargs): + defaults = { + 'max_digits': self.max_digits, + 'decimal_places': self.decimal_places, + 'form_class': forms.DecimalField, + } + defaults.update(kwargs) + return super(DecimalField, self).formfield(**defaults) + +class EmailField(CharField): + def __init__(self, *args, **kwargs): + kwargs['max_length'] = kwargs.get('max_length', 75) + CharField.__init__(self, *args, **kwargs) + + def get_manipulator_field_objs(self): + return [oldforms.EmailField] + + def formfield(self, **kwargs): + defaults = {'form_class': forms.EmailField} + defaults.update(kwargs) + return super(EmailField, self).formfield(**defaults) + +class FilePathField(Field): + def __init__(self, verbose_name=None, name=None, path='', match=None, recursive=False, **kwargs): + self.path, self.match, self.recursive = path, match, recursive + kwargs['max_length'] = kwargs.get('max_length', 100) + Field.__init__(self, verbose_name, name, **kwargs) + + def formfield(self, **kwargs): + defaults = { + 'path': self.path, + 'match': self.match, + 'recursive': self.recursive, + 'form_class': forms.FilePathField, + } + defaults.update(kwargs) + return super(FilePathField, self).formfield(**defaults) + + def get_manipulator_field_objs(self): + return [curry(oldforms.FilePathField, path=self.path, match=self.match, recursive=self.recursive)] + + def get_internal_type(self): + return "FilePathField" + +class FloatField(Field): + empty_strings_allowed = False + + def get_db_prep_value(self, value): + if value is None: + return None + return float(value) + + def get_manipulator_field_objs(self): + return [oldforms.FloatField] + + def get_internal_type(self): + return "FloatField" + + def formfield(self, **kwargs): + defaults = {'form_class': forms.FloatField} + defaults.update(kwargs) + return super(FloatField, self).formfield(**defaults) + +class IntegerField(Field): + empty_strings_allowed = False + def get_db_prep_value(self, value): + if value is None: + return None + return int(value) + + def get_manipulator_field_objs(self): + return [oldforms.IntegerField] + + def get_internal_type(self): + return "IntegerField" + + def to_python(self, value): + if value is None: + return value + try: + return int(value) + except (TypeError, ValueError): + raise validators.ValidationError, _("This value must be an integer.") + + def formfield(self, **kwargs): + defaults = {'form_class': forms.IntegerField} + defaults.update(kwargs) + return super(IntegerField, self).formfield(**defaults) + +class IPAddressField(Field): + empty_strings_allowed = False + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 15 + Field.__init__(self, *args, **kwargs) + + def get_manipulator_field_objs(self): + return [oldforms.IPAddressField] + + def get_internal_type(self): + return "IPAddressField" + + def formfield(self, **kwargs): + defaults = {'form_class': forms.IPAddressField} + defaults.update(kwargs) + return super(IPAddressField, self).formfield(**defaults) + +class NullBooleanField(Field): + empty_strings_allowed = False + def __init__(self, *args, **kwargs): + kwargs['null'] = True + Field.__init__(self, *args, **kwargs) + + def get_internal_type(self): + return "NullBooleanField" + + def to_python(self, value): + if value in (None, True, False): return value + if value in ('None'): return None + if value in ('t', 'True', '1'): return True + if value in ('f', 'False', '0'): return False + raise validators.ValidationError, _("This value must be either None, True or False.") + + def get_db_prep_value(self, value): + if value is None: + return None + return bool(value) + + def get_manipulator_field_objs(self): + return [oldforms.NullBooleanField] + + def formfield(self, **kwargs): + defaults = { + 'form_class': forms.NullBooleanField, + 'required': not self.blank, + 'label': capfirst(self.verbose_name), + 'help_text': self.help_text} + defaults.update(kwargs) + return super(NullBooleanField, self).formfield(**defaults) + +class PhoneNumberField(Field): + def get_manipulator_field_objs(self): + return [oldforms.PhoneNumberField] + + def get_internal_type(self): + return "PhoneNumberField" + + def formfield(self, **kwargs): + from django.contrib.localflavor.us.forms import USPhoneNumberField + defaults = {'form_class': USPhoneNumberField} + defaults.update(kwargs) + return super(PhoneNumberField, self).formfield(**defaults) + +class PositiveIntegerField(IntegerField): + def get_manipulator_field_objs(self): + return [oldforms.PositiveIntegerField] + + def get_internal_type(self): + return "PositiveIntegerField" + + def formfield(self, **kwargs): + defaults = {'min_value': 0} + defaults.update(kwargs) + return super(PositiveIntegerField, self).formfield(**defaults) + +class PositiveSmallIntegerField(IntegerField): + def get_manipulator_field_objs(self): + return [oldforms.PositiveSmallIntegerField] + + def get_internal_type(self): + return "PositiveSmallIntegerField" + + def formfield(self, **kwargs): + defaults = {'min_value': 0} + defaults.update(kwargs) + return super(PositiveSmallIntegerField, self).formfield(**defaults) + +class SlugField(CharField): + def __init__(self, *args, **kwargs): + kwargs['max_length'] = kwargs.get('max_length', 50) + kwargs.setdefault('validator_list', []).append(validators.isSlug) + # Set db_index=True unless it's been set manually. + if 'db_index' not in kwargs: + kwargs['db_index'] = True + super(SlugField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "SlugField" + + def formfield(self, **kwargs): + defaults = {'form_class': forms.RegexField, 'regex': r'^[a-zA-Z0-9_-]+$', + 'error_messages': {'invalid': _(u"Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens.")}, + } + defaults.update(kwargs) + return super(SlugField, self).formfield(**defaults) + +class SmallIntegerField(IntegerField): + def get_manipulator_field_objs(self): + return [oldforms.SmallIntegerField] + + def get_internal_type(self): + return "SmallIntegerField" + +class TextField(Field): + def get_manipulator_field_objs(self): + return [oldforms.LargeTextField] + + def get_internal_type(self): + return "TextField" + + def formfield(self, **kwargs): + defaults = {'widget': forms.Textarea} + defaults.update(kwargs) + return super(TextField, self).formfield(**defaults) + +class TimeField(Field): + empty_strings_allowed = False + def __init__(self, verbose_name=None, name=None, auto_now=False, auto_now_add=False, **kwargs): + self.auto_now, self.auto_now_add = auto_now, auto_now_add + if auto_now or auto_now_add: + kwargs['editable'] = False + Field.__init__(self, verbose_name, name, **kwargs) + + def get_internal_type(self): + return "TimeField" + + def to_python(self, value): + if value is None: + return None + if isinstance(value, datetime.time): + return value + + # Attempt to parse a datetime: + value = smart_str(value) + # split usecs, because they are not recognized by strptime. + if '.' in value: + try: + value, usecs = value.split('.') + usecs = int(usecs) + except ValueError: + raise validators.ValidationError, _('Enter a valid time in HH:MM[:ss[.uuuuuu]] format.') + else: + usecs = 0 + kwargs = {'microsecond': usecs} + + try: # Seconds are optional, so try converting seconds first. + return datetime.time(*time.strptime(value, '%H:%M:%S')[3:6], + **kwargs) + except ValueError: + try: # Try without seconds. + return datetime.time(*time.strptime(value, '%H:%M')[3:5], + **kwargs) + except ValueError: + raise validators.ValidationError, _('Enter a valid time in HH:MM[:ss[.uuuuuu]] format.') + + def pre_save(self, model_instance, add): + if self.auto_now or (self.auto_now_add and add): + value = datetime.datetime.now().time() + setattr(model_instance, self.attname, value) + return value + else: + return super(TimeField, self).pre_save(model_instance, add) + + def get_db_prep_value(self, value): + # Casts times into the format expected by the backend + return connection.ops.value_to_db_time(self.to_python(value)) + + def get_manipulator_field_objs(self): + return [oldforms.TimeField] + + def flatten_data(self,follow, obj = None): + val = self._get_val_from_obj(obj) + return {self.attname: (val is not None and val.strftime("%H:%M:%S") or '')} + + def formfield(self, **kwargs): + defaults = {'form_class': forms.TimeField} + defaults.update(kwargs) + return super(TimeField, self).formfield(**defaults) + +class URLField(CharField): + def __init__(self, verbose_name=None, name=None, verify_exists=True, **kwargs): + kwargs['max_length'] = kwargs.get('max_length', 200) + if verify_exists: + kwargs.setdefault('validator_list', []).append(validators.isExistingURL) + self.verify_exists = verify_exists + CharField.__init__(self, verbose_name, name, **kwargs) + + def get_manipulator_field_objs(self): + return [oldforms.URLField] + + def formfield(self, **kwargs): + defaults = {'form_class': forms.URLField, 'verify_exists': self.verify_exists} + defaults.update(kwargs) + return super(URLField, self).formfield(**defaults) + +class USStateField(Field): + def get_manipulator_field_objs(self): + return [oldforms.USStateField] + + def get_internal_type(self): + return "USStateField" + + def formfield(self, **kwargs): + from django.contrib.localflavor.us.forms import USStateSelect + defaults = {'widget': USStateSelect} + defaults.update(kwargs) + return super(USStateField, self).formfield(**defaults) + +class XMLField(TextField): + def __init__(self, verbose_name=None, name=None, schema_path=None, **kwargs): + self.schema_path = schema_path + Field.__init__(self, verbose_name, name, **kwargs) + + def get_manipulator_field_objs(self): + return [curry(oldforms.XMLLargeTextField, schema_path=self.schema_path)] + diff --git a/webapp/django/db/models/fields/files.py b/webapp/django/db/models/fields/files.py new file mode 100644 index 0000000000..935dee6162 --- /dev/null +++ b/webapp/django/db/models/fields/files.py @@ -0,0 +1,291 @@ +import datetime +import os + +from django.conf import settings +from django.db.models.fields import Field +from django.core.files.base import File, ContentFile +from django.core.files.storage import default_storage +from django.core.files.images import ImageFile, get_image_dimensions +from django.core.files.uploadedfile import UploadedFile +from django.utils.functional import curry +from django.db.models import signals +from django.utils.encoding import force_unicode, smart_str +from django.utils.translation import ugettext_lazy, ugettext as _ +from django import oldforms +from django import forms +from django.core import validators +from django.db.models.loading import cache + +class FieldFile(File): + def __init__(self, instance, field, name): + self.instance = instance + self.field = field + self.storage = field.storage + self._name = name or u'' + self._closed = False + + def __eq__(self, other): + # Older code may be expecting FileField values to be simple strings. + # By overriding the == operator, it can remain backwards compatibility. + if hasattr(other, 'name'): + return self.name == other.name + return self.name == other + + # The standard File contains most of the necessary properties, but + # FieldFiles can be instantiated without a name, so that needs to + # be checked for here. + + def _require_file(self): + if not self: + raise ValueError("The '%s' attribute has no file associated with it." % self.field.name) + + def _get_file(self): + self._require_file() + if not hasattr(self, '_file'): + self._file = self.storage.open(self.name, 'rb') + return self._file + file = property(_get_file) + + def _get_path(self): + self._require_file() + return self.storage.path(self.name) + path = property(_get_path) + + def _get_url(self): + self._require_file() + return self.storage.url(self.name) + url = property(_get_url) + + def open(self, mode='rb'): + self._require_file() + return super(FieldFile, self).open(mode) + # open() doesn't alter the file's contents, but it does reset the pointer + open.alters_data = True + + # In addition to the standard File API, FieldFiles have extra methods + # to further manipulate the underlying file, as well as update the + # associated model instance. + + def save(self, name, content, save=True): + name = self.field.generate_filename(self.instance, name) + self._name = self.storage.save(name, content) + setattr(self.instance, self.field.name, self.name) + + # Update the filesize cache + self._size = len(content) + + # Save the object because it has changed, unless save is False + if save: + self.instance.save() + save.alters_data = True + + def delete(self, save=True): + self.close() + self.storage.delete(self.name) + + self._name = None + setattr(self.instance, self.field.name, self.name) + + # Delete the filesize cache + if hasattr(self, '_size'): + del self._size + + if save: + self.instance.save() + delete.alters_data = True + + def __getstate__(self): + # FieldFile needs access to its associated model field and an instance + # it's attached to in order to work properly, but the only necessary + # data to be pickled is the file's name itself. Everything else will + # be restored later, by FileDescriptor below. + return {'_name': self.name, '_closed': False} + +class FileDescriptor(object): + def __init__(self, field): + self.field = field + + def __get__(self, instance=None, owner=None): + if instance is None: + raise AttributeError, "%s can only be accessed from %s instances." % (self.field.name(self.owner.__name__)) + file = instance.__dict__[self.field.name] + if not isinstance(file, FieldFile): + # Create a new instance of FieldFile, based on a given file name + instance.__dict__[self.field.name] = self.field.attr_class(instance, self.field, file) + elif not hasattr(file, 'field'): + # The FieldFile was pickled, so some attributes need to be reset. + file.instance = instance + file.field = self.field + file.storage = self.field.storage + return instance.__dict__[self.field.name] + + def __set__(self, instance, value): + instance.__dict__[self.field.name] = value + +class FileField(Field): + attr_class = FieldFile + + def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, **kwargs): + for arg in ('core', 'primary_key', 'unique'): + if arg in kwargs: + raise TypeError("'%s' is not a valid argument for %s." % (arg, self.__class__)) + + self.storage = storage or default_storage + self.upload_to = upload_to + if callable(upload_to): + self.generate_filename = upload_to + + kwargs['max_length'] = kwargs.get('max_length', 100) + super(FileField, self).__init__(verbose_name, name, **kwargs) + + def get_internal_type(self): + return "FileField" + + def get_db_prep_lookup(self, lookup_type, value): + if hasattr(value, 'name'): + value = value.name + return super(FileField, self).get_db_prep_lookup(lookup_type, value) + + def get_db_prep_value(self, value): + "Returns field's value prepared for saving into a database." + # Need to convert File objects provided via a form to unicode for database insertion + if value is None: + return None + return unicode(value) + + def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True): + field_list = Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow) + if not self.blank: + if rel: + # This validator makes sure FileFields work in a related context. + class RequiredFileField(object): + def __init__(self, other_field_names, other_file_field_name): + self.other_field_names = other_field_names + self.other_file_field_name = other_file_field_name + self.always_test = True + def __call__(self, field_data, all_data): + if not all_data.get(self.other_file_field_name, False): + c = validators.RequiredIfOtherFieldsGiven(self.other_field_names, ugettext_lazy("This field is required.")) + c(field_data, all_data) + # First, get the core fields, if any. + core_field_names = [] + for f in opts.fields: + if f.core and f != self: + core_field_names.extend(f.get_manipulator_field_names(name_prefix)) + # Now, if there are any, add the validator to this FormField. + if core_field_names: + field_list[0].validator_list.append(RequiredFileField(core_field_names, field_list[1].field_name)) + else: + v = validators.RequiredIfOtherFieldNotGiven(field_list[1].field_name, ugettext_lazy("This field is required.")) + v.always_test = True + field_list[0].validator_list.append(v) + field_list[0].is_required = field_list[1].is_required = False + + # If the raw path is passed in, validate it's under the MEDIA_ROOT. + def isWithinMediaRoot(field_data, all_data): + f = os.path.abspath(os.path.join(settings.MEDIA_ROOT, field_data)) + if not f.startswith(os.path.abspath(os.path.normpath(settings.MEDIA_ROOT))): + raise validators.ValidationError(_("Enter a valid filename.")) + field_list[1].validator_list.append(isWithinMediaRoot) + return field_list + + def contribute_to_class(self, cls, name): + super(FileField, self).contribute_to_class(cls, name) + setattr(cls, self.name, FileDescriptor(self)) + signals.post_delete.connect(self.delete_file, sender=cls) + + def delete_file(self, instance, sender, **kwargs): + file = getattr(instance, self.attname) + # If no other object of this type references the file, + # and it's not the default value for future objects, + # delete it from the backend. + if file and file.name != self.default and \ + not sender._default_manager.filter(**{self.name: file.name}): + file.delete(save=False) + elif file: + # Otherwise, just close the file, so it doesn't tie up resources. + file.close() + + def get_manipulator_field_objs(self): + return [oldforms.FileUploadField, oldforms.HiddenField] + + def get_manipulator_field_names(self, name_prefix): + return [name_prefix + self.name + '_file', name_prefix + self.name] + + def save_file(self, new_data, new_object, original_object, change, rel, save=True): + upload_field_name = self.get_manipulator_field_names('')[0] + if new_data.get(upload_field_name, False): + if rel: + file = new_data[upload_field_name][0] + else: + file = new_data[upload_field_name] + + # Backwards-compatible support for files-as-dictionaries. + # We don't need to raise a warning because the storage backend will + # do so for us. + try: + filename = file.name + except AttributeError: + filename = file['filename'] + filename = self.get_filename(filename) + + getattr(new_object, self.attname).save(filename, file, save) + + def get_directory_name(self): + return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to)))) + + def get_filename(self, filename): + return os.path.normpath(self.storage.get_valid_name(os.path.basename(filename))) + + def generate_filename(self, instance, filename): + return os.path.join(self.get_directory_name(), self.get_filename(filename)) + + def save_form_data(self, instance, data): + if data and isinstance(data, UploadedFile): + getattr(instance, self.name).save(data.name, data, save=False) + + def formfield(self, **kwargs): + defaults = {'form_class': forms.FileField} + # If a file has been provided previously, then the form doesn't require + # that a new file is provided this time. + # The code to mark the form field as not required is used by + # form_for_instance, but can probably be removed once form_for_instance + # is gone. ModelForm uses a different method to check for an existing file. + if 'initial' in kwargs: + defaults['required'] = False + defaults.update(kwargs) + return super(FileField, self).formfield(**defaults) + +class ImageFieldFile(ImageFile, FieldFile): + def save(self, name, content, save=True): + # Repopulate the image dimension cache. + self._dimensions_cache = get_image_dimensions(content) + + # Update width/height fields, if needed + if self.field.width_field: + setattr(self.instance, self.field.width_field, self.width) + if self.field.height_field: + setattr(self.instance, self.field.height_field, self.height) + + super(ImageFieldFile, self).save(name, content, save) + + def delete(self, save=True): + # Clear the image dimensions cache + if hasattr(self, '_dimensions_cache'): + del self._dimensions_cache + super(ImageFieldFile, self).delete(save) + +class ImageField(FileField): + attr_class = ImageFieldFile + + def __init__(self, verbose_name=None, name=None, width_field=None, height_field=None, **kwargs): + self.width_field, self.height_field = width_field, height_field + FileField.__init__(self, verbose_name, name, **kwargs) + + def get_manipulator_field_objs(self): + return [oldforms.ImageUploadField, oldforms.HiddenField] + + def formfield(self, **kwargs): + defaults = {'form_class': forms.ImageField} + defaults.update(kwargs) + return super(ImageField, self).formfield(**defaults) diff --git a/webapp/django/db/models/fields/proxy.py b/webapp/django/db/models/fields/proxy.py new file mode 100644 index 0000000000..31a31e3c3c --- /dev/null +++ b/webapp/django/db/models/fields/proxy.py @@ -0,0 +1,16 @@ +""" +Field-like classes that aren't really fields. It's easier to use objects that +have the same attributes as fields sometimes (avoids a lot of special casing). +""" + +from django.db.models import fields + +class OrderWrt(fields.IntegerField): + """ + A proxy for the _order database field that is used when + Meta.order_with_respect_to is specified. + """ + name = '_order' + attname = '_order' + column = '_order' + diff --git a/webapp/django/db/models/fields/related.py b/webapp/django/db/models/fields/related.py new file mode 100644 index 0000000000..b09412aa29 --- /dev/null +++ b/webapp/django/db/models/fields/related.py @@ -0,0 +1,944 @@ +from django.db import connection, transaction +from django.db.models import signals, get_model +from django.db.models.fields import AutoField, Field, IntegerField, PositiveIntegerField, PositiveSmallIntegerField, FieldDoesNotExist +from django.db.models.related import RelatedObject +from django.db.models.query import QuerySet +from django.db.models.query_utils import QueryWrapper +from django.utils.translation import ugettext_lazy, string_concat, ungettext, ugettext as _ +from django.utils.functional import curry +from django.core import validators +from django import oldforms +from django import forms + +try: + set +except NameError: + from sets import Set as set # Python 2.3 fallback + +# Values for Relation.edit_inline. +TABULAR, STACKED = 1, 2 + +RECURSIVE_RELATIONSHIP_CONSTANT = 'self' + +pending_lookups = {} + +def add_lazy_relation(cls, field, relation, operation): + """ + Adds a lookup on ``cls`` when a related field is defined using a string, + i.e.:: + + class MyModel(Model): + fk = ForeignKey("AnotherModel") + + This string can be: + + * RECURSIVE_RELATIONSHIP_CONSTANT (i.e. "self") to indicate a recursive + relation. + + * The name of a model (i.e "AnotherModel") to indicate another model in + the same app. + + * An app-label and model name (i.e. "someapp.AnotherModel") to indicate + another model in a different app. + + If the other model hasn't yet been loaded -- almost a given if you're using + lazy relationships -- then the relation won't be set up until the + class_prepared signal fires at the end of model initialization. + + operation is the work that must be performed once the relation can be resolved. + """ + # Check for recursive relations + if relation == RECURSIVE_RELATIONSHIP_CONSTANT: + app_label = cls._meta.app_label + model_name = cls.__name__ + + else: + # Look for an "app.Model" relation + try: + app_label, model_name = relation.split(".") + except ValueError: + # If we can't split, assume a model in current app + app_label = cls._meta.app_label + model_name = relation + + # Try to look up the related model, and if it's already loaded resolve the + # string right away. If get_model returns None, it means that the related + # model isn't loaded yet, so we need to pend the relation until the class + # is prepared. + model = get_model(app_label, model_name, False) + if model: + operation(field, model, cls) + else: + key = (app_label, model_name) + value = (cls, field, operation) + pending_lookups.setdefault(key, []).append(value) + +def do_pending_lookups(sender, **kwargs): + """ + Handle any pending relations to the sending model. Sent from class_prepared. + """ + key = (sender._meta.app_label, sender.__name__) + for cls, field, operation in pending_lookups.pop(key, []): + operation(field, sender, cls) + +signals.class_prepared.connect(do_pending_lookups) + +def manipulator_valid_rel_key(f, self, field_data, all_data): + "Validates that the value is a valid foreign key" + klass = f.rel.to + try: + klass._default_manager.get(**{f.rel.field_name: field_data}) + except klass.DoesNotExist: + raise validators.ValidationError, _("Please enter a valid %s.") % f.verbose_name + +#HACK +class RelatedField(object): + def contribute_to_class(self, cls, name): + sup = super(RelatedField, self) + + # Add an accessor to allow easy determination of the related query path for this field + self.related_query_name = curry(self._get_related_query_name, cls._meta) + + if hasattr(sup, 'contribute_to_class'): + sup.contribute_to_class(cls, name) + + if not cls._meta.abstract and self.rel.related_name: + self.rel.related_name = self.rel.related_name % {'class': cls.__name__.lower()} + + other = self.rel.to + if isinstance(other, basestring): + def resolve_related_class(field, model, cls): + field.rel.to = model + field.do_related_class(model, cls) + add_lazy_relation(cls, self, other, resolve_related_class) + else: + self.do_related_class(other, cls) + + def set_attributes_from_rel(self): + self.name = self.name or (self.rel.to._meta.object_name.lower() + '_' + self.rel.to._meta.pk.name) + if self.verbose_name is None: + self.verbose_name = self.rel.to._meta.verbose_name + self.rel.field_name = self.rel.field_name or self.rel.to._meta.pk.name + + def do_related_class(self, other, cls): + self.set_attributes_from_rel() + related = RelatedObject(other, cls, self) + if not cls._meta.abstract: + self.contribute_to_related_class(other, related) + + def get_db_prep_lookup(self, lookup_type, value): + # If we are doing a lookup on a Related Field, we must be + # comparing object instances. The value should be the PK of value, + # not value itself. + def pk_trace(value): + # Value may be a primary key, or an object held in a relation. + # If it is an object, then we need to get the primary key value for + # that object. In certain conditions (especially one-to-one relations), + # the primary key may itself be an object - so we need to keep drilling + # down until we hit a value that can be used for a comparison. + v = value + try: + while True: + v = getattr(v, v._meta.pk.name) + except AttributeError: + pass + return v + + if hasattr(value, 'as_sql'): + sql, params = value.as_sql() + return QueryWrapper(('(%s)' % sql), params) + if lookup_type == 'exact': + return [pk_trace(value)] + if lookup_type == 'in': + return [pk_trace(v) for v in value] + elif lookup_type == 'isnull': + return [] + raise TypeError, "Related Field has invalid lookup: %s" % lookup_type + + def _get_related_query_name(self, opts): + # This method defines the name that can be used to identify this + # related object in a table-spanning query. It uses the lower-cased + # object_name by default, but this can be overridden with the + # "related_name" option. + return self.rel.related_name or opts.object_name.lower() + +class SingleRelatedObjectDescriptor(object): + # This class provides the functionality that makes the related-object + # managers available as attributes on a model class, for fields that have + # a single "remote" value, on the class pointed to by a related field. + # In the example "place.restaurant", the restaurant attribute is a + # SingleRelatedObjectDescriptor instance. + def __init__(self, related): + self.related = related + self.cache_name = '_%s_cache' % related.get_accessor_name() + + def __get__(self, instance, instance_type=None): + if instance is None: + raise AttributeError, "%s must be accessed via instance" % self.related.opts.object_name + + try: + return getattr(instance, self.cache_name) + except AttributeError: + params = {'%s__pk' % self.related.field.name: instance._get_pk_val()} + rel_obj = self.related.model._default_manager.get(**params) + setattr(instance, self.cache_name, rel_obj) + return rel_obj + + def __set__(self, instance, value): + if instance is None: + raise AttributeError, "%s must be accessed via instance" % self.related.opts.object_name + + # The similarity of the code below to the code in + # ReverseSingleRelatedObjectDescriptor is annoying, but there's a bunch + # of small differences that would make a common base class convoluted. + + # If null=True, we can assign null here, but otherwise the value needs + # to be an instance of the related class. + if value is None and self.related.field.null == False: + raise ValueError('Cannot assign None: "%s.%s" does not allow null values.' % + (instance._meta.object_name, self.related.get_accessor_name())) + elif value is not None and not isinstance(value, self.related.model): + raise ValueError('Cannot assign "%r": "%s.%s" must be a "%s" instance.' % + (value, instance._meta.object_name, + self.related.get_accessor_name(), self.related.opts.object_name)) + + # Set the value of the related field + setattr(value, self.related.field.rel.get_related_field().attname, instance) + + # Since we already know what the related object is, seed the related + # object caches now, too. This avoids another db hit if you get the + # object you just set. + setattr(instance, self.cache_name, value) + setattr(value, self.related.field.get_cache_name(), instance) + +class ReverseSingleRelatedObjectDescriptor(object): + # This class provides the functionality that makes the related-object + # managers available as attributes on a model class, for fields that have + # a single "remote" value, on the class that defines the related field. + # In the example "choice.poll", the poll attribute is a + # ReverseSingleRelatedObjectDescriptor instance. + def __init__(self, field_with_rel): + self.field = field_with_rel + + def __get__(self, instance, instance_type=None): + if instance is None: + raise AttributeError, "%s must be accessed via instance" % self.field.name + cache_name = self.field.get_cache_name() + try: + return getattr(instance, cache_name) + except AttributeError: + val = getattr(instance, self.field.attname) + if val is None: + # If NULL is an allowed value, return it. + if self.field.null: + return None + raise self.field.rel.to.DoesNotExist + other_field = self.field.rel.get_related_field() + if other_field.rel: + params = {'%s__pk' % self.field.rel.field_name: val} + else: + params = {'%s__exact' % self.field.rel.field_name: val} + + # If the related manager indicates that it should be used for + # related fields, respect that. + rel_mgr = self.field.rel.to._default_manager + if getattr(rel_mgr, 'use_for_related_fields', False): + rel_obj = rel_mgr.get(**params) + else: + rel_obj = QuerySet(self.field.rel.to).get(**params) + setattr(instance, cache_name, rel_obj) + return rel_obj + + def __set__(self, instance, value): + if instance is None: + raise AttributeError, "%s must be accessed via instance" % self._field.name + + # If null=True, we can assign null here, but otherwise the value needs + # to be an instance of the related class. + if value is None and self.field.null == False: + raise ValueError('Cannot assign None: "%s.%s" does not allow null values.' % + (instance._meta.object_name, self.field.name)) + elif value is not None and not isinstance(value, self.field.rel.to): + raise ValueError('Cannot assign "%r": "%s.%s" must be a "%s" instance.' % + (value, instance._meta.object_name, + self.field.name, self.field.rel.to._meta.object_name)) + + # Set the value of the related field + try: + val = getattr(value, self.field.rel.get_related_field().attname) + except AttributeError: + val = None + setattr(instance, self.field.attname, val) + + # Since we already know what the related object is, seed the related + # object cache now, too. This avoids another db hit if you get the + # object you just set. + setattr(instance, self.field.get_cache_name(), value) + +class ForeignRelatedObjectsDescriptor(object): + # This class provides the functionality that makes the related-object + # managers available as attributes on a model class, for fields that have + # multiple "remote" values and have a ForeignKey pointed at them by + # some other model. In the example "poll.choice_set", the choice_set + # attribute is a ForeignRelatedObjectsDescriptor instance. + def __init__(self, related): + self.related = related # RelatedObject instance + + def __get__(self, instance, instance_type=None): + if instance is None: + raise AttributeError, "Manager must be accessed via instance" + + rel_field = self.related.field + rel_model = self.related.model + + # Dynamically create a class that subclasses the related + # model's default manager. + superclass = self.related.model._default_manager.__class__ + + class RelatedManager(superclass): + def get_query_set(self): + return superclass.get_query_set(self).filter(**(self.core_filters)) + + def add(self, *objs): + for obj in objs: + setattr(obj, rel_field.name, instance) + obj.save() + add.alters_data = True + + def create(self, **kwargs): + new_obj = self.model(**kwargs) + self.add(new_obj) + return new_obj + create.alters_data = True + + def get_or_create(self, **kwargs): + # Update kwargs with the related object that this + # ForeignRelatedObjectsDescriptor knows about. + kwargs.update({rel_field.name: instance}) + return super(RelatedManager, self).get_or_create(**kwargs) + get_or_create.alters_data = True + + # remove() and clear() are only provided if the ForeignKey can have a value of null. + if rel_field.null: + def remove(self, *objs): + val = getattr(instance, rel_field.rel.get_related_field().attname) + for obj in objs: + # Is obj actually part of this descriptor set? + if getattr(obj, rel_field.attname) == val: + setattr(obj, rel_field.name, None) + obj.save() + else: + raise rel_field.rel.to.DoesNotExist, "%r is not related to %r." % (obj, instance) + remove.alters_data = True + + def clear(self): + for obj in self.all(): + setattr(obj, rel_field.name, None) + obj.save() + clear.alters_data = True + + manager = RelatedManager() + attname = rel_field.rel.get_related_field().name + manager.core_filters = {'%s__%s' % (rel_field.name, attname): + getattr(instance, attname)} + manager.model = self.related.model + + return manager + + def __set__(self, instance, value): + if instance is None: + raise AttributeError, "Manager must be accessed via instance" + + manager = self.__get__(instance) + # If the foreign key can support nulls, then completely clear the related set. + # Otherwise, just move the named objects into the set. + if self.related.field.null: + manager.clear() + manager.add(*value) + +def create_many_related_manager(superclass, through=False): + """Creates a manager that subclasses 'superclass' (which is a Manager) + and adds behavior for many-to-many related objects.""" + class ManyRelatedManager(superclass): + def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None, + join_table=None, source_col_name=None, target_col_name=None): + super(ManyRelatedManager, self).__init__() + self.core_filters = core_filters + self.model = model + self.symmetrical = symmetrical + self.instance = instance + self.join_table = join_table + self.source_col_name = source_col_name + self.target_col_name = target_col_name + self.through = through + self._pk_val = self.instance._get_pk_val() + if self._pk_val is None: + raise ValueError("%r instance needs to have a primary key value before a many-to-many relationship can be used." % instance.__class__.__name__) + + def get_query_set(self): + return superclass.get_query_set(self)._next_is_sticky().filter(**(self.core_filters)) + + # If the ManyToMany relation has an intermediary model, + # the add and remove methods do not exist. + if through is None: + def add(self, *objs): + self._add_items(self.source_col_name, self.target_col_name, *objs) + + # If this is a symmetrical m2m relation to self, add the mirror entry in the m2m table + if self.symmetrical: + self._add_items(self.target_col_name, self.source_col_name, *objs) + add.alters_data = True + + def remove(self, *objs): + self._remove_items(self.source_col_name, self.target_col_name, *objs) + + # If this is a symmetrical m2m relation to self, remove the mirror entry in the m2m table + if self.symmetrical: + self._remove_items(self.target_col_name, self.source_col_name, *objs) + remove.alters_data = True + + def clear(self): + self._clear_items(self.source_col_name) + + # If this is a symmetrical m2m relation to self, clear the mirror entry in the m2m table + if self.symmetrical: + self._clear_items(self.target_col_name) + clear.alters_data = True + + def create(self, **kwargs): + # This check needs to be done here, since we can't later remove this + # from the method lookup table, as we do with add and remove. + if through is not None: + raise AttributeError, "Cannot use create() on a ManyToManyField which specifies an intermediary model. Use %s's Manager instead." % through + new_obj = self.model(**kwargs) + new_obj.save() + self.add(new_obj) + return new_obj + create.alters_data = True + + def get_or_create(self, **kwargs): + obj, created = \ + super(ManyRelatedManager, self).get_or_create(**kwargs) + # We only need to add() if created because if we got an object back + # from get() then the relationship already exists. + if created: + self.add(obj) + return obj, created + get_or_create.alters_data = True + + def _add_items(self, source_col_name, target_col_name, *objs): + # join_table: name of the m2m link table + # source_col_name: the PK colname in join_table for the source object + # target_col_name: the PK colname in join_table for the target object + # *objs - objects to add. Either object instances, or primary keys of object instances. + + # If there aren't any objects, there is nothing to do. + if objs: + # Check that all the objects are of the right type + new_ids = set() + for obj in objs: + if isinstance(obj, self.model): + new_ids.add(obj._get_pk_val()) + else: + new_ids.add(obj) + # Add the newly created or already existing objects to the join table. + # First find out which items are already added, to avoid adding them twice + cursor = connection.cursor() + cursor.execute("SELECT %s FROM %s WHERE %s = %%s AND %s IN (%s)" % \ + (target_col_name, self.join_table, source_col_name, + target_col_name, ",".join(['%s'] * len(new_ids))), + [self._pk_val] + list(new_ids)) + existing_ids = set([row[0] for row in cursor.fetchall()]) + + # Add the ones that aren't there already + for obj_id in (new_ids - existing_ids): + cursor.execute("INSERT INTO %s (%s, %s) VALUES (%%s, %%s)" % \ + (self.join_table, source_col_name, target_col_name), + [self._pk_val, obj_id]) + transaction.commit_unless_managed() + + def _remove_items(self, source_col_name, target_col_name, *objs): + # source_col_name: the PK colname in join_table for the source object + # target_col_name: the PK colname in join_table for the target object + # *objs - objects to remove + + # If there aren't any objects, there is nothing to do. + if objs: + # Check that all the objects are of the right type + old_ids = set() + for obj in objs: + if isinstance(obj, self.model): + old_ids.add(obj._get_pk_val()) + else: + old_ids.add(obj) + # Remove the specified objects from the join table + cursor = connection.cursor() + cursor.execute("DELETE FROM %s WHERE %s = %%s AND %s IN (%s)" % \ + (self.join_table, source_col_name, + target_col_name, ",".join(['%s'] * len(old_ids))), + [self._pk_val] + list(old_ids)) + transaction.commit_unless_managed() + + def _clear_items(self, source_col_name): + # source_col_name: the PK colname in join_table for the source object + cursor = connection.cursor() + cursor.execute("DELETE FROM %s WHERE %s = %%s" % \ + (self.join_table, source_col_name), + [self._pk_val]) + transaction.commit_unless_managed() + + return ManyRelatedManager + +class ManyRelatedObjectsDescriptor(object): + # This class provides the functionality that makes the related-object + # managers available as attributes on a model class, for fields that have + # multiple "remote" values and have a ManyToManyField pointed at them by + # some other model (rather than having a ManyToManyField themselves). + # In the example "publication.article_set", the article_set attribute is a + # ManyRelatedObjectsDescriptor instance. + def __init__(self, related): + self.related = related # RelatedObject instance + + def __get__(self, instance, instance_type=None): + if instance is None: + raise AttributeError, "Manager must be accessed via instance" + + # Dynamically create a class that subclasses the related + # model's default manager. + rel_model = self.related.model + superclass = rel_model._default_manager.__class__ + RelatedManager = create_many_related_manager(superclass, self.related.field.rel.through) + + qn = connection.ops.quote_name + manager = RelatedManager( + model=rel_model, + core_filters={'%s__pk' % self.related.field.name: instance._get_pk_val()}, + instance=instance, + symmetrical=False, + join_table=qn(self.related.field.m2m_db_table()), + source_col_name=qn(self.related.field.m2m_reverse_name()), + target_col_name=qn(self.related.field.m2m_column_name()) + ) + + return manager + + def __set__(self, instance, value): + if instance is None: + raise AttributeError, "Manager must be accessed via instance" + + through = getattr(self.related.field.rel, 'through', None) + if through is not None: + raise AttributeError, "Cannot set values on a ManyToManyField which specifies an intermediary model. Use %s's Manager instead." % through + + manager = self.__get__(instance) + manager.clear() + manager.add(*value) + +class ReverseManyRelatedObjectsDescriptor(object): + # This class provides the functionality that makes the related-object + # managers available as attributes on a model class, for fields that have + # multiple "remote" values and have a ManyToManyField defined in their + # model (rather than having another model pointed *at* them). + # In the example "article.publications", the publications attribute is a + # ReverseManyRelatedObjectsDescriptor instance. + def __init__(self, m2m_field): + self.field = m2m_field + + def __get__(self, instance, instance_type=None): + if instance is None: + raise AttributeError, "Manager must be accessed via instance" + + # Dynamically create a class that subclasses the related + # model's default manager. + rel_model=self.field.rel.to + superclass = rel_model._default_manager.__class__ + RelatedManager = create_many_related_manager(superclass, self.field.rel.through) + + qn = connection.ops.quote_name + manager = RelatedManager( + model=rel_model, + core_filters={'%s__pk' % self.field.related_query_name(): instance._get_pk_val()}, + instance=instance, + symmetrical=(self.field.rel.symmetrical and instance.__class__ == rel_model), + join_table=qn(self.field.m2m_db_table()), + source_col_name=qn(self.field.m2m_column_name()), + target_col_name=qn(self.field.m2m_reverse_name()) + ) + + return manager + + def __set__(self, instance, value): + if instance is None: + raise AttributeError, "Manager must be accessed via instance" + + through = getattr(self.field.rel, 'through', None) + if through is not None: + raise AttributeError, "Cannot set values on a ManyToManyField which specifies an intermediary model. Use %s's Manager instead." % through + + manager = self.__get__(instance) + manager.clear() + manager.add(*value) + +class ManyToOneRel(object): + def __init__(self, to, field_name, num_in_admin=3, min_num_in_admin=None, + max_num_in_admin=None, num_extra_on_change=1, edit_inline=False, + related_name=None, limit_choices_to=None, lookup_overrides=None, + parent_link=False): + try: + to._meta + except AttributeError: # to._meta doesn't exist, so it must be RECURSIVE_RELATIONSHIP_CONSTANT + assert isinstance(to, basestring), "'to' must be either a model, a model name or the string %r" % RECURSIVE_RELATIONSHIP_CONSTANT + self.to, self.field_name = to, field_name + self.num_in_admin, self.edit_inline = num_in_admin, edit_inline + self.min_num_in_admin, self.max_num_in_admin = min_num_in_admin, max_num_in_admin + self.num_extra_on_change, self.related_name = num_extra_on_change, related_name + if limit_choices_to is None: + limit_choices_to = {} + self.limit_choices_to = limit_choices_to + self.lookup_overrides = lookup_overrides or {} + self.multiple = True + self.parent_link = parent_link + + def get_related_field(self): + """ + Returns the Field in the 'to' object to which this relationship is + tied. + """ + data = self.to._meta.get_field_by_name(self.field_name) + if not data[2]: + raise FieldDoesNotExist("No related field named '%s'" % + self.field_name) + return data[0] + +class OneToOneRel(ManyToOneRel): + def __init__(self, to, field_name, num_in_admin=0, min_num_in_admin=None, + max_num_in_admin=None, num_extra_on_change=None, edit_inline=False, + related_name=None, limit_choices_to=None, lookup_overrides=None, + parent_link=False): + # NOTE: *_num_in_admin and num_extra_on_change are intentionally + # ignored here. We accept them as parameters only to match the calling + # signature of ManyToOneRel.__init__(). + super(OneToOneRel, self).__init__(to, field_name, num_in_admin, + edit_inline=edit_inline, related_name=related_name, + limit_choices_to=limit_choices_to, + lookup_overrides=lookup_overrides, parent_link=parent_link) + self.multiple = False + +class ManyToManyRel(object): + def __init__(self, to, num_in_admin=0, related_name=None, + limit_choices_to=None, symmetrical=True, through=None): + self.to = to + self.num_in_admin = num_in_admin + self.related_name = related_name + if limit_choices_to is None: + limit_choices_to = {} + self.limit_choices_to = limit_choices_to + self.edit_inline = False + self.symmetrical = symmetrical + self.multiple = True + self.through = through + +class ForeignKey(RelatedField, Field): + empty_strings_allowed = False + def __init__(self, to, to_field=None, rel_class=ManyToOneRel, **kwargs): + try: + to_name = to._meta.object_name.lower() + except AttributeError: # to._meta doesn't exist, so it must be RECURSIVE_RELATIONSHIP_CONSTANT + assert isinstance(to, basestring), "%s(%r) is invalid. First parameter to ForeignKey must be either a model, a model name, or the string %r" % (self.__class__.__name__, to, RECURSIVE_RELATIONSHIP_CONSTANT) + else: + assert not to._meta.abstract, "%s cannot define a relation with abstract class %s" % (self.__class__.__name__, to._meta.object_name) + to_field = to_field or to._meta.pk.name + kwargs['verbose_name'] = kwargs.get('verbose_name', None) + + kwargs['rel'] = rel_class(to, to_field, + num_in_admin=kwargs.pop('num_in_admin', 3), + min_num_in_admin=kwargs.pop('min_num_in_admin', None), + max_num_in_admin=kwargs.pop('max_num_in_admin', None), + num_extra_on_change=kwargs.pop('num_extra_on_change', 1), + edit_inline=kwargs.pop('edit_inline', False), + related_name=kwargs.pop('related_name', None), + limit_choices_to=kwargs.pop('limit_choices_to', None), + lookup_overrides=kwargs.pop('lookup_overrides', None), + parent_link=kwargs.pop('parent_link', False)) + Field.__init__(self, **kwargs) + + self.db_index = True + + def get_attname(self): + return '%s_id' % self.name + + def get_validator_unique_lookup_type(self): + return '%s__%s__exact' % (self.name, self.rel.get_related_field().name) + + def prepare_field_objs_and_params(self, manipulator, name_prefix): + params = {'validator_list': self.validator_list[:], 'member_name': name_prefix + self.attname} + if self.null: + field_objs = [oldforms.NullSelectField] + else: + field_objs = [oldforms.SelectField] + params['choices'] = self.get_choices_default() + return field_objs, params + + def get_default(self): + "Here we check if the default value is an object and return the to_field if so." + field_default = super(ForeignKey, self).get_default() + if isinstance(field_default, self.rel.to): + return getattr(field_default, self.rel.get_related_field().attname) + return field_default + + def get_manipulator_field_objs(self): + rel_field = self.rel.get_related_field() + return [oldforms.IntegerField] + + def get_db_prep_save(self, value): + if value == '' or value == None: + return None + else: + return self.rel.get_related_field().get_db_prep_save(value) + + def flatten_data(self, follow, obj=None): + if not obj: + # In required many-to-one fields with only one available choice, + # select that one available choice. Note: For SelectFields + # we have to check that the length of choices is *2*, not 1, + # because SelectFields always have an initial "blank" value. + if not self.blank and self.choices: + choice_list = self.get_choices_default() + if len(choice_list) == 2: + return {self.attname: choice_list[1][0]} + return Field.flatten_data(self, follow, obj) + + def contribute_to_class(self, cls, name): + super(ForeignKey, self).contribute_to_class(cls, name) + setattr(cls, self.name, ReverseSingleRelatedObjectDescriptor(self)) + if isinstance(self.rel.to, basestring): + target = self.rel.to + else: + target = self.rel.to._meta.db_table + cls._meta.duplicate_targets[self.column] = (target, "o2m") + + def contribute_to_related_class(self, cls, related): + setattr(cls, related.get_accessor_name(), ForeignRelatedObjectsDescriptor(related)) + + def formfield(self, **kwargs): + defaults = {'form_class': forms.ModelChoiceField, 'queryset': self.rel.to._default_manager.complex_filter(self.rel.limit_choices_to)} + defaults.update(kwargs) + return super(ForeignKey, self).formfield(**defaults) + + def db_type(self): + # The database column type of a ForeignKey is the column type + # of the field to which it points. An exception is if the ForeignKey + # points to an AutoField/PositiveIntegerField/PositiveSmallIntegerField, + # in which case the column type is simply that of an IntegerField. + rel_field = self.rel.get_related_field() + if isinstance(rel_field, (AutoField, PositiveIntegerField, PositiveSmallIntegerField)): + return IntegerField().db_type() + return rel_field.db_type() + +class OneToOneField(ForeignKey): + """ + A OneToOneField is essentially the same as a ForeignKey, with the exception + that always carries a "unique" constraint with it and the reverse relation + always returns the object pointed to (since there will only ever be one), + rather than returning a list. + """ + def __init__(self, to, to_field=None, **kwargs): + kwargs['unique'] = True + if 'num_in_admin' not in kwargs: + kwargs['num_in_admin'] = 0 + super(OneToOneField, self).__init__(to, to_field, OneToOneRel, **kwargs) + + def contribute_to_related_class(self, cls, related): + setattr(cls, related.get_accessor_name(), + SingleRelatedObjectDescriptor(related)) + if not cls._meta.one_to_one_field: + cls._meta.one_to_one_field = self + + def formfield(self, **kwargs): + if self.rel.parent_link: + return None + return super(OneToOneField, self).formfield(**kwargs) + +class ManyToManyField(RelatedField, Field): + def __init__(self, to, **kwargs): + try: + assert not to._meta.abstract, "%s cannot define a relation with abstract class %s" % (self.__class__.__name__, to._meta.object_name) + except AttributeError: # to._meta doesn't exist, so it must be RECURSIVE_RELATIONSHIP_CONSTANT + assert isinstance(to, basestring), "%s(%r) is invalid. First parameter to ManyToManyField must be either a model, a model name, or the string %r" % (self.__class__.__name__, to, RECURSIVE_RELATIONSHIP_CONSTANT) + + kwargs['verbose_name'] = kwargs.get('verbose_name', None) + kwargs['rel'] = ManyToManyRel(to, + num_in_admin=kwargs.pop('num_in_admin', 0), + related_name=kwargs.pop('related_name', None), + limit_choices_to=kwargs.pop('limit_choices_to', None), + symmetrical=kwargs.pop('symmetrical', True), + through=kwargs.pop('through', None)) + + self.db_table = kwargs.pop('db_table', None) + if kwargs['rel'].through is not None: + self.creates_table = False + assert self.db_table is None, "Cannot specify a db_table if an intermediary model is used." + else: + self.creates_table = True + + Field.__init__(self, **kwargs) + + msg = ugettext_lazy('Hold down "Control", or "Command" on a Mac, to select more than one.') + self.help_text = string_concat(self.help_text, ' ', msg) + + def get_manipulator_field_objs(self): + choices = self.get_choices_default() + return [curry(oldforms.SelectMultipleField, size=min(max(len(choices), 5), 15), choices=choices)] + + def get_choices_default(self): + return Field.get_choices(self, include_blank=False) + + def _get_m2m_db_table(self, opts): + "Function that can be curried to provide the m2m table name for this relation" + if self.rel.through is not None: + return self.rel.through_model._meta.db_table + elif self.db_table: + return self.db_table + else: + return '%s_%s' % (opts.db_table, self.name) + + def _get_m2m_column_name(self, related): + "Function that can be curried to provide the source column name for the m2m table" + try: + return self._m2m_column_name_cache + except: + if self.rel.through is not None: + for f in self.rel.through_model._meta.fields: + if hasattr(f,'rel') and f.rel and f.rel.to == related.model: + self._m2m_column_name_cache = f.column + break + # If this is an m2m relation to self, avoid the inevitable name clash + elif related.model == related.parent_model: + self._m2m_column_name_cache = 'from_' + related.model._meta.object_name.lower() + '_id' + else: + self._m2m_column_name_cache = related.model._meta.object_name.lower() + '_id' + + # Return the newly cached value + return self._m2m_column_name_cache + + def _get_m2m_reverse_name(self, related): + "Function that can be curried to provide the related column name for the m2m table" + try: + return self._m2m_reverse_name_cache + except: + if self.rel.through is not None: + found = False + for f in self.rel.through_model._meta.fields: + if hasattr(f,'rel') and f.rel and f.rel.to == related.parent_model: + if related.model == related.parent_model: + # If this is an m2m-intermediate to self, + # the first foreign key you find will be + # the source column. Keep searching for + # the second foreign key. + if found: + self._m2m_reverse_name_cache = f.column + break + else: + found = True + else: + self._m2m_reverse_name_cache = f.column + break + # If this is an m2m relation to self, avoid the inevitable name clash + elif related.model == related.parent_model: + self._m2m_reverse_name_cache = 'to_' + related.parent_model._meta.object_name.lower() + '_id' + else: + self._m2m_reverse_name_cache = related.parent_model._meta.object_name.lower() + '_id' + + # Return the newly cached value + return self._m2m_reverse_name_cache + + def isValidIDList(self, field_data, all_data): + "Validates that the value is a valid list of foreign keys" + mod = self.rel.to + try: + pks = map(int, field_data.split(',')) + except ValueError: + # the CommaSeparatedIntegerField validator will catch this error + return + objects = mod._default_manager.in_bulk(pks) + if len(objects) != len(pks): + badkeys = [k for k in pks if k not in objects] + raise validators.ValidationError, ungettext("Please enter valid %(self)s IDs. The value %(value)r is invalid.", + "Please enter valid %(self)s IDs. The values %(value)r are invalid.", len(badkeys)) % { + 'self': self.verbose_name, + 'value': len(badkeys) == 1 and badkeys[0] or tuple(badkeys), + } + + def flatten_data(self, follow, obj = None): + new_data = {} + if obj: + instance_ids = [instance._get_pk_val() for instance in getattr(obj, self.name).all()] + new_data[self.name] = instance_ids + else: + # In required many-to-many fields with only one available choice, + # select that one available choice. + if not self.blank and not self.rel.edit_inline: + choices_list = self.get_choices_default() + if len(choices_list) == 1: + new_data[self.name] = [choices_list[0][0]] + return new_data + + def contribute_to_class(self, cls, name): + super(ManyToManyField, self).contribute_to_class(cls, name) + # Add the descriptor for the m2m relation + setattr(cls, self.name, ReverseManyRelatedObjectsDescriptor(self)) + + # Set up the accessor for the m2m table name for the relation + self.m2m_db_table = curry(self._get_m2m_db_table, cls._meta) + + # Populate some necessary rel arguments so that cross-app relations + # work correctly. + if isinstance(self.rel.through, basestring): + def resolve_through_model(field, model, cls): + field.rel.through_model = model + add_lazy_relation(cls, self, self.rel.through, resolve_through_model) + elif self.rel.through: + self.rel.through_model = self.rel.through + self.rel.through = self.rel.through._meta.object_name + + if isinstance(self.rel.to, basestring): + target = self.rel.to + else: + target = self.rel.to._meta.db_table + cls._meta.duplicate_targets[self.column] = (target, "m2m") + + def contribute_to_related_class(self, cls, related): + # m2m relations to self do not have a ManyRelatedObjectsDescriptor, + # as it would be redundant - unless the field is non-symmetrical. + if related.model != related.parent_model or not self.rel.symmetrical: + # Add the descriptor for the m2m relation + setattr(cls, related.get_accessor_name(), ManyRelatedObjectsDescriptor(related)) + + # Set up the accessors for the column names on the m2m table + self.m2m_column_name = curry(self._get_m2m_column_name, related) + self.m2m_reverse_name = curry(self._get_m2m_reverse_name, related) + + def set_attributes_from_rel(self): + pass + + def value_from_object(self, obj): + "Returns the value of this field in the given model instance." + return getattr(obj, self.attname).all() + + def save_form_data(self, instance, data): + setattr(instance, self.attname, data) + + def formfield(self, **kwargs): + defaults = {'form_class': forms.ModelMultipleChoiceField, 'queryset': self.rel.to._default_manager.complex_filter(self.rel.limit_choices_to)} + defaults.update(kwargs) + # If initial is passed in, it's a list of related objects, but the + # MultipleChoiceField takes a list of IDs. + if defaults.get('initial') is not None: + defaults['initial'] = [i._get_pk_val() for i in defaults['initial']] + return super(ManyToManyField, self).formfield(**defaults) + + def db_type(self): + # A ManyToManyField is not represented by a single column, + # so return None. + return None + diff --git a/webapp/django/db/models/fields/subclassing.py b/webapp/django/db/models/fields/subclassing.py new file mode 100644 index 0000000000..10add10739 --- /dev/null +++ b/webapp/django/db/models/fields/subclassing.py @@ -0,0 +1,50 @@ +""" +Convenience routines for creating non-trivial Field subclasses. + +Add SubfieldBase as the __metaclass__ for your Field subclass, implement +to_python() and the other necessary methods and everything will work seamlessly. +""" + +class SubfieldBase(type): + """ + A metaclass for custom Field subclasses. This ensures the model's attribute + has the descriptor protocol attached to it. + """ + def __new__(cls, base, name, attrs): + new_class = super(SubfieldBase, cls).__new__(cls, base, name, attrs) + new_class.contribute_to_class = make_contrib( + attrs.get('contribute_to_class')) + return new_class + +class Creator(object): + """ + A placeholder class that provides a way to set the attribute on the model. + """ + def __init__(self, field): + self.field = field + + def __get__(self, obj, type=None): + if obj is None: + raise AttributeError('Can only be accessed via an instance.') + return obj.__dict__[self.field.name] + + def __set__(self, obj, value): + obj.__dict__[self.field.name] = self.field.to_python(value) + +def make_contrib(func=None): + """ + Returns a suitable contribute_to_class() method for the Field subclass. + + If 'func' is passed in, it is the existing contribute_to_class() method on + the subclass and it is called before anything else. It is assumed in this + case that the existing contribute_to_class() calls all the necessary + superclass methods. + """ + def contribute_to_class(self, cls, name): + if func: + func(self, cls, name) + else: + super(self.__class__, self).contribute_to_class(cls, name) + setattr(cls, self.name, Creator(self)) + + return contribute_to_class diff --git a/webapp/django/db/models/loading.py b/webapp/django/db/models/loading.py new file mode 100644 index 0000000000..6837e070ac --- /dev/null +++ b/webapp/django/db/models/loading.py @@ -0,0 +1,189 @@ +"Utilities for loading models and the modules that contain them." + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.utils.datastructures import SortedDict + +import sys +import os +import threading + +__all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models', + 'load_app', 'app_cache_ready') + +class AppCache(object): + """ + A cache that stores installed applications and their models. Used to + provide reverse-relations and for app introspection (e.g. admin). + """ + # Use the Borg pattern to share state between all instances. Details at + # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66531. + __shared_state = dict( + # Keys of app_store are the model modules for each application. + app_store = SortedDict(), + + # Mapping of app_labels to a dictionary of model names to model code. + app_models = SortedDict(), + + # Mapping of app_labels to errors raised when trying to import the app. + app_errors = {}, + + # -- Everything below here is only used when populating the cache -- + loaded = False, + handled = {}, + postponed = [], + nesting_level = 0, + write_lock = threading.RLock(), + ) + + def __init__(self): + self.__dict__ = self.__shared_state + + def _populate(self): + """ + Fill in all the cache information. This method is threadsafe, in the + sense that every caller will see the same state upon return, and if the + cache is already initialised, it does no work. + """ + if self.loaded: + return + self.write_lock.acquire() + try: + if self.loaded: + return + for app_name in settings.INSTALLED_APPS: + if app_name in self.handled: + continue + self.load_app(app_name, True) + if not self.nesting_level: + for app_name in self.postponed: + self.load_app(app_name) + self.loaded = True + finally: + self.write_lock.release() + + def load_app(self, app_name, can_postpone=False): + """ + Loads the app with the provided fully qualified name, and returns the + model module. + """ + self.handled[app_name] = None + self.nesting_level += 1 + mod = __import__(app_name, {}, {}, ['models']) + self.nesting_level -= 1 + if not hasattr(mod, 'models'): + if can_postpone: + # Either the app has no models, or the package is still being + # imported by Python and the model module isn't available yet. + # We will check again once all the recursion has finished (in + # populate). + self.postponed.append(app_name) + return None + if mod.models not in self.app_store: + self.app_store[mod.models] = len(self.app_store) + return mod.models + + def app_cache_ready(self): + """ + Returns true if the model cache is fully populated. + + Useful for code that wants to cache the results of get_models() for + themselves once it is safe to do so. + """ + return self.loaded + + def get_apps(self): + "Returns a list of all installed modules that contain models." + self._populate() + + # Ensure the returned list is always in the same order (with new apps + # added at the end). This avoids unstable ordering on the admin app + # list page, for example. + apps = [(v, k) for k, v in self.app_store.items()] + apps.sort() + return [elt[1] for elt in apps] + + def get_app(self, app_label, emptyOK=False): + """ + Returns the module containing the models for the given app_label. If + the app has no models in it and 'emptyOK' is True, returns None. + """ + self._populate() + self.write_lock.acquire() + try: + for app_name in settings.INSTALLED_APPS: + if app_label == app_name.split('.')[-1]: + mod = self.load_app(app_name, False) + if mod is None: + if emptyOK: + return None + else: + return mod + raise ImproperlyConfigured, "App with label %s could not be found" % app_label + finally: + self.write_lock.release() + + def get_app_errors(self): + "Returns the map of known problems with the INSTALLED_APPS." + self._populate() + return self.app_errors + + def get_models(self, app_mod=None): + """ + Given a module containing models, returns a list of the models. + Otherwise returns a list of all installed models. + """ + self._populate() + if app_mod: + return self.app_models.get(app_mod.__name__.split('.')[-2], SortedDict()).values() + else: + model_list = [] + for app_entry in self.app_models.itervalues(): + model_list.extend(app_entry.values()) + return model_list + + def get_model(self, app_label, model_name, seed_cache=True): + """ + Returns the model matching the given app_label and case-insensitive + model_name. + + Returns None if no model is found. + """ + if seed_cache: + self._populate() + return self.app_models.get(app_label, SortedDict()).get(model_name.lower()) + + def register_models(self, app_label, *models): + """ + Register a set of models as belonging to an app. + """ + for model in models: + # Store as 'name: model' pair in a dictionary + # in the app_models dictionary + model_name = model._meta.object_name.lower() + model_dict = self.app_models.setdefault(app_label, SortedDict()) + if model_name in model_dict: + # The same model may be imported via different paths (e.g. + # appname.models and project.appname.models). We use the source + # filename as a means to detect identity. + fname1 = os.path.abspath(sys.modules[model.__module__].__file__) + fname2 = os.path.abspath(sys.modules[model_dict[model_name].__module__].__file__) + # Since the filename extension could be .py the first time and + # .pyc or .pyo the second time, ignore the extension when + # comparing. + if os.path.splitext(fname1)[0] == os.path.splitext(fname2)[0]: + continue + model_dict[model_name] = model + +cache = AppCache() + +# These methods were always module level, so are kept that way for backwards +# compatibility. +get_apps = cache.get_apps +get_app = cache.get_app +get_app_errors = cache.get_app_errors +get_models = cache.get_models +get_model = cache.get_model +register_models = cache.register_models +load_app = cache.load_app +app_cache_ready = cache.app_cache_ready diff --git a/webapp/django/db/models/manager.py b/webapp/django/db/models/manager.py new file mode 100644 index 0000000000..96bd53f240 --- /dev/null +++ b/webapp/django/db/models/manager.py @@ -0,0 +1,144 @@ +import copy + +from django.db.models.query import QuerySet, EmptyQuerySet, insert_query +from django.db.models import signals +from django.db.models.fields import FieldDoesNotExist + +def ensure_default_manager(sender, **kwargs): + cls = sender + if not getattr(cls, '_default_manager', None) and not cls._meta.abstract: + # Create the default manager, if needed. + try: + cls._meta.get_field('objects') + raise ValueError, "Model %s must specify a custom Manager, because it has a field named 'objects'" % cls.__name__ + except FieldDoesNotExist: + pass + cls.add_to_class('objects', Manager()) + +signals.class_prepared.connect(ensure_default_manager) + +class Manager(object): + # Tracks each time a Manager instance is created. Used to retain order. + creation_counter = 0 + + def __init__(self): + super(Manager, self).__init__() + # Increase the creation counter, and save our local copy. + self.creation_counter = Manager.creation_counter + Manager.creation_counter += 1 + self.model = None + + def contribute_to_class(self, model, name): + # TODO: Use weakref because of possible memory leak / circular reference. + self.model = model + setattr(model, name, ManagerDescriptor(self)) + if not getattr(model, '_default_manager', None) or self.creation_counter < model._default_manager.creation_counter: + model._default_manager = self + + def _copy_to_model(self, model): + """ + Makes a copy of the manager and assigns it to 'model', which should be + a child of the existing model (used when inheriting a manager from an + abstract base class). + """ + assert issubclass(model, self.model) + mgr = copy.copy(self) + mgr.model = model + return mgr + + ####################### + # PROXIES TO QUERYSET # + ####################### + + def get_empty_query_set(self): + return EmptyQuerySet(self.model) + + def get_query_set(self): + """Returns a new QuerySet object. Subclasses can override this method + to easily customize the behavior of the Manager. + """ + return QuerySet(self.model) + + def none(self): + return self.get_empty_query_set() + + def all(self): + return self.get_query_set() + + def count(self): + return self.get_query_set().count() + + def dates(self, *args, **kwargs): + return self.get_query_set().dates(*args, **kwargs) + + def distinct(self, *args, **kwargs): + return self.get_query_set().distinct(*args, **kwargs) + + def extra(self, *args, **kwargs): + return self.get_query_set().extra(*args, **kwargs) + + def get(self, *args, **kwargs): + return self.get_query_set().get(*args, **kwargs) + + def get_or_create(self, **kwargs): + return self.get_query_set().get_or_create(**kwargs) + + def create(self, **kwargs): + return self.get_query_set().create(**kwargs) + + def filter(self, *args, **kwargs): + return self.get_query_set().filter(*args, **kwargs) + + def complex_filter(self, *args, **kwargs): + return self.get_query_set().complex_filter(*args, **kwargs) + + def exclude(self, *args, **kwargs): + return self.get_query_set().exclude(*args, **kwargs) + + def in_bulk(self, *args, **kwargs): + return self.get_query_set().in_bulk(*args, **kwargs) + + def iterator(self, *args, **kwargs): + return self.get_query_set().iterator(*args, **kwargs) + + def latest(self, *args, **kwargs): + return self.get_query_set().latest(*args, **kwargs) + + def order_by(self, *args, **kwargs): + return self.get_query_set().order_by(*args, **kwargs) + + def select_related(self, *args, **kwargs): + return self.get_query_set().select_related(*args, **kwargs) + + def values(self, *args, **kwargs): + return self.get_query_set().values(*args, **kwargs) + + def values_list(self, *args, **kwargs): + return self.get_query_set().values_list(*args, **kwargs) + + def update(self, *args, **kwargs): + return self.get_query_set().update(*args, **kwargs) + + def reverse(self, *args, **kwargs): + return self.get_query_set().reverse(*args, **kwargs) + + def _insert(self, values, **kwargs): + return insert_query(self.model, values, **kwargs) + + def _update(self, values, **kwargs): + return self.get_query_set()._update(values, **kwargs) + +class ManagerDescriptor(object): + # This class ensures managers aren't accessible via model instances. + # For example, Poll.objects works, but poll_obj.objects raises AttributeError. + def __init__(self, manager): + self.manager = manager + + def __get__(self, instance, type=None): + if instance != None: + raise AttributeError, "Manager isn't accessible via %s instances" % type.__name__ + return self.manager + +class EmptyManager(Manager): + def get_query_set(self): + return self.get_empty_query_set() diff --git a/webapp/django/db/models/manipulators.py b/webapp/django/db/models/manipulators.py new file mode 100644 index 0000000000..c657d0158b --- /dev/null +++ b/webapp/django/db/models/manipulators.py @@ -0,0 +1,333 @@ +from django.core.exceptions import ObjectDoesNotExist +from django import oldforms +from django.core import validators +from django.db.models.fields import AutoField +from django.db.models.fields.files import FileField +from django.db.models import signals +from django.utils.functional import curry +from django.utils.datastructures import DotExpandedDict +from django.utils.text import capfirst +from django.utils.encoding import smart_str +from django.utils.translation import ugettext as _ +from django.utils import datetime_safe + +def add_manipulators(sender, **kwargs): + cls = sender + cls.add_to_class('AddManipulator', AutomaticAddManipulator) + cls.add_to_class('ChangeManipulator', AutomaticChangeManipulator) + +signals.class_prepared.connect(add_manipulators) + +class ManipulatorDescriptor(object): + # This class provides the functionality that makes the default model + # manipulators (AddManipulator and ChangeManipulator) available via the + # model class. + def __init__(self, name, base): + self.man = None # Cache of the manipulator class. + self.name = name + self.base = base + + def __get__(self, instance, model=None): + if instance != None: + raise AttributeError, "Manipulator cannot be accessed via instance" + else: + if not self.man: + # Create a class that inherits from the "Manipulator" class + # given in the model class (if specified) and the automatic + # manipulator. + bases = [self.base] + if hasattr(model, 'Manipulator'): + bases = [model.Manipulator] + bases + self.man = type(self.name, tuple(bases), {}) + self.man._prepare(model) + return self.man + +class AutomaticManipulator(oldforms.Manipulator): + def _prepare(cls, model): + cls.model = model + cls.manager = model._default_manager + cls.opts = model._meta + for field_name_list in cls.opts.unique_together: + setattr(cls, 'isUnique%s' % '_'.join(field_name_list), curry(manipulator_validator_unique_together, field_name_list, cls.opts)) + for f in cls.opts.fields: + if f.unique_for_date: + setattr(cls, 'isUnique%sFor%s' % (f.name, f.unique_for_date), curry(manipulator_validator_unique_for_date, f, cls.opts.get_field(f.unique_for_date), cls.opts, 'date')) + if f.unique_for_month: + setattr(cls, 'isUnique%sFor%s' % (f.name, f.unique_for_month), curry(manipulator_validator_unique_for_date, f, cls.opts.get_field(f.unique_for_month), cls.opts, 'month')) + if f.unique_for_year: + setattr(cls, 'isUnique%sFor%s' % (f.name, f.unique_for_year), curry(manipulator_validator_unique_for_date, f, cls.opts.get_field(f.unique_for_year), cls.opts, 'year')) + _prepare = classmethod(_prepare) + + def contribute_to_class(cls, other_cls, name): + setattr(other_cls, name, ManipulatorDescriptor(name, cls)) + contribute_to_class = classmethod(contribute_to_class) + + def __init__(self, follow=None): + self.follow = self.opts.get_follow(follow) + self.fields = [] + + for f in self.opts.fields + self.opts.many_to_many: + if self.follow.get(f.name, False): + self.fields.extend(f.get_manipulator_fields(self.opts, self, self.change)) + + # Add fields for related objects. + for f in self.opts.get_all_related_objects(): + if self.follow.get(f.name, False): + fol = self.follow[f.name] + self.fields.extend(f.get_manipulator_fields(self.opts, self, self.change, fol)) + + # Add field for ordering. + if self.change and self.opts.get_ordered_objects(): + self.fields.append(oldforms.CommaSeparatedIntegerField(field_name="order_")) + + def save(self, new_data): + # TODO: big cleanup when core fields go -> use recursive manipulators. + params = {} + for f in self.opts.fields: + # Fields with auto_now_add should keep their original value in the change stage. + auto_now_add = self.change and getattr(f, 'auto_now_add', False) + if self.follow.get(f.name, None) and not auto_now_add: + param = f.get_manipulator_new_data(new_data) + else: + if self.change: + param = getattr(self.original_object, f.attname) + else: + param = f.get_default() + params[f.attname] = param + + if self.change: + params[self.opts.pk.attname] = self.obj_key + + # First, create the basic object itself. + new_object = self.model(**params) + + # Now that the object's been created, save any uploaded files. + for f in self.opts.fields: + if isinstance(f, FileField): + f.save_file(new_data, new_object, self.change and self.original_object or None, self.change, rel=False, save=False) + + # Now save the object + new_object.save() + + # Calculate which primary fields have changed. + if self.change: + self.fields_added, self.fields_changed, self.fields_deleted = [], [], [] + for f in self.opts.fields: + if not f.primary_key and smart_str(getattr(self.original_object, f.attname)) != smart_str(getattr(new_object, f.attname)): + self.fields_changed.append(f.verbose_name) + + # Save many-to-many objects. Example: Set sites for a poll. + for f in self.opts.many_to_many: + if self.follow.get(f.name, None): + if not f.rel.edit_inline: + new_vals = new_data.getlist(f.name) + # First, clear the existing values. + rel_manager = getattr(new_object, f.name) + rel_manager.clear() + # Then, set the new values. + for n in new_vals: + rel_manager.add(f.rel.to._default_manager.get(pk=n)) + # TODO: Add to 'fields_changed' + + expanded_data = DotExpandedDict(dict(new_data)) + # Save many-to-one objects. Example: Add the Choice objects for a Poll. + for related in self.opts.get_all_related_objects(): + # Create obj_list, which is a DotExpandedDict such as this: + # [('0', {'id': ['940'], 'choice': ['This is the first choice']}), + # ('1', {'id': ['941'], 'choice': ['This is the second choice']}), + # ('2', {'id': [''], 'choice': ['']})] + child_follow = self.follow.get(related.name, None) + + if child_follow: + obj_list = expanded_data.get(related.var_name, {}).items() + if not obj_list: + continue + + obj_list.sort(lambda x, y: cmp(int(x[0]), int(y[0]))) + + # For each related item... + for _, rel_new_data in obj_list: + + params = {} + + # Keep track of which core=True fields were provided. + # If all core fields were given, the related object will be saved. + # If none of the core fields were given, the object will be deleted. + # If some, but not all, of the fields were given, the validator would + # have caught that. + all_cores_given, all_cores_blank = True, True + + # Get a reference to the old object. We'll use it to compare the + # old to the new, to see which fields have changed. + old_rel_obj = None + if self.change: + if rel_new_data[related.opts.pk.name][0]: + try: + old_rel_obj = getattr(self.original_object, related.get_accessor_name()).get(**{'%s__exact' % related.opts.pk.name: rel_new_data[related.opts.pk.attname][0]}) + except ObjectDoesNotExist: + pass + + for f in related.opts.fields: + if f.core and not isinstance(f, FileField) and f.get_manipulator_new_data(rel_new_data, rel=True) in (None, ''): + all_cores_given = False + elif f.core and not isinstance(f, FileField) and f.get_manipulator_new_data(rel_new_data, rel=True) not in (None, ''): + all_cores_blank = False + # If this field isn't editable, give it the same value it had + # previously, according to the given ID. If the ID wasn't + # given, use a default value. FileFields are also a special + # case, because they'll be dealt with later. + + if f == related.field: + param = getattr(new_object, related.field.rel.get_related_field().attname) + elif (not self.change) and isinstance(f, AutoField): + param = None + elif self.change and (isinstance(f, FileField) or not child_follow.get(f.name, None)): + if old_rel_obj: + param = getattr(old_rel_obj, f.column) + else: + param = f.get_default() + else: + param = f.get_manipulator_new_data(rel_new_data, rel=True) + if param != None: + params[f.attname] = param + + # Create the related item. + new_rel_obj = related.model(**params) + + # If all the core fields were provided (non-empty), save the item. + if all_cores_given: + new_rel_obj.save() + + # Save any uploaded files. + for f in related.opts.fields: + if child_follow.get(f.name, None): + if isinstance(f, FileField) and rel_new_data.get(f.name, False): + f.save_file(rel_new_data, new_rel_obj, self.change and old_rel_obj or None, old_rel_obj is not None, rel=True) + + # Calculate whether any fields have changed. + if self.change: + if not old_rel_obj: # This object didn't exist before. + self.fields_added.append('%s "%s"' % (related.opts.verbose_name, new_rel_obj)) + else: + for f in related.opts.fields: + if not f.primary_key and f != related.field and smart_str(getattr(old_rel_obj, f.attname)) != smart_str(getattr(new_rel_obj, f.attname)): + self.fields_changed.append('%s for %s "%s"' % (f.verbose_name, related.opts.verbose_name, new_rel_obj)) + + # Save many-to-many objects. + for f in related.opts.many_to_many: + if child_follow.get(f.name, None) and not f.rel.edit_inline: + new_value = rel_new_data[f.attname] + setattr(new_rel_obj, f.name, f.rel.to.objects.filter(pk__in=new_value)) + if self.change: + self.fields_changed.append('%s for %s "%s"' % (f.verbose_name, related.opts.verbose_name, new_rel_obj)) + + # If, in the change stage, all of the core fields were blank and + # the primary key (ID) was provided, delete the item. + if self.change and all_cores_blank and old_rel_obj: + new_rel_obj.delete() + self.fields_deleted.append('%s "%s"' % (related.opts.verbose_name, old_rel_obj)) + + # Save the order, if applicable. + if self.change and self.opts.get_ordered_objects(): + order = new_data['order_'] and map(int, new_data['order_'].split(',')) or [] + for rel_opts in self.opts.get_ordered_objects(): + getattr(new_object, 'set_%s_order' % rel_opts.object_name.lower())(order) + return new_object + + def get_related_objects(self): + return self.opts.get_followed_related_objects(self.follow) + + def flatten_data(self): + new_data = {} + obj = self.change and self.original_object or None + for f in self.opts.get_data_holders(self.follow): + fol = self.follow.get(f.name) + new_data.update(f.flatten_data(fol, obj)) + return new_data + +class AutomaticAddManipulator(AutomaticManipulator): + change = False + +class AutomaticChangeManipulator(AutomaticManipulator): + change = True + def __init__(self, obj_key, follow=None): + self.obj_key = obj_key + try: + self.original_object = self.manager.get(pk=obj_key) + except ObjectDoesNotExist: + # If the object doesn't exist, this might be a manipulator for a + # one-to-one related object that hasn't created its subobject yet. + # For example, this might be a Restaurant for a Place that doesn't + # yet have restaurant information. + if self.opts.one_to_one_field: + # Sanity check -- Make sure the "parent" object exists. + # For example, make sure the Place exists for the Restaurant. + # Let the ObjectDoesNotExist exception propagate up. + limit_choices_to = self.opts.one_to_one_field.rel.limit_choices_to + lookup_kwargs = {'%s__exact' % self.opts.one_to_one_field.rel.field_name: obj_key} + self.opts.one_to_one_field.rel.to.get_model_module().complex_filter(limit_choices_to).get(**lookup_kwargs) + params = dict([(f.attname, f.get_default()) for f in self.opts.fields]) + params[self.opts.pk.attname] = obj_key + self.original_object = self.opts.get_model_module().Klass(**params) + else: + raise + super(AutomaticChangeManipulator, self).__init__(follow=follow) + +def manipulator_validator_unique_together(field_name_list, opts, self, field_data, all_data): + from django.db.models.fields.related import ManyToOneRel + from django.utils.text import get_text_list + field_list = [opts.get_field(field_name) for field_name in field_name_list] + if isinstance(field_list[0].rel, ManyToOneRel): + kwargs = {'%s__%s__iexact' % (field_name_list[0], field_list[0].rel.field_name): field_data} + else: + kwargs = {'%s__iexact' % field_name_list[0]: field_data} + for f in field_list[1:]: + # This is really not going to work for fields that have different + # form fields, e.g. DateTime. + # This validation needs to occur after html2python to be effective. + field_val = all_data.get(f.name, None) + if field_val is None: + # This will be caught by another validator, assuming the field + # doesn't have blank=True. + return + if isinstance(f.rel, ManyToOneRel): + kwargs['%s__pk' % f.name] = field_val + else: + kwargs['%s__iexact' % f.name] = field_val + try: + old_obj = self.manager.get(**kwargs) + except ObjectDoesNotExist: + return + if hasattr(self, 'original_object') and self.original_object._get_pk_val() == old_obj._get_pk_val(): + pass + else: + raise validators.ValidationError, _("%(object)s with this %(type)s already exists for the given %(field)s.") % \ + {'object': capfirst(opts.verbose_name), 'type': field_list[0].verbose_name, 'field': get_text_list([f.verbose_name for f in field_list[1:]], _('and'))} + +def manipulator_validator_unique_for_date(from_field, date_field, opts, lookup_type, self, field_data, all_data): + from django.db.models.fields.related import ManyToOneRel + date_str = all_data.get(date_field.get_manipulator_field_names('')[0], None) + date_val = oldforms.DateField.html2python(date_str) + if date_val is None: + return # Date was invalid. This will be caught by another validator. + lookup_kwargs = {'%s__year' % date_field.name: date_val.year} + if isinstance(from_field.rel, ManyToOneRel): + lookup_kwargs['%s__pk' % from_field.name] = field_data + else: + lookup_kwargs['%s__iexact' % from_field.name] = field_data + if lookup_type in ('month', 'date'): + lookup_kwargs['%s__month' % date_field.name] = date_val.month + if lookup_type == 'date': + lookup_kwargs['%s__day' % date_field.name] = date_val.day + try: + old_obj = self.manager.get(**lookup_kwargs) + except ObjectDoesNotExist: + return + else: + if hasattr(self, 'original_object') and self.original_object._get_pk_val() == old_obj._get_pk_val(): + pass + else: + format_string = (lookup_type == 'date') and '%B %d, %Y' or '%B %Y' + date_val = datetime_safe.new_datetime(date_val) + raise validators.ValidationError, "Please enter a different %s. The one you entered is already being used for %s." % \ + (from_field.verbose_name, date_val.strftime(format_string)) diff --git a/webapp/django/db/models/options.py b/webapp/django/db/models/options.py new file mode 100644 index 0000000000..ffea6d5082 --- /dev/null +++ b/webapp/django/db/models/options.py @@ -0,0 +1,486 @@ +import re +from bisect import bisect +try: + set +except NameError: + from sets import Set as set # Python 2.3 fallback + +from django.conf import settings +from django.db.models.related import RelatedObject +from django.db.models.fields.related import ManyToManyRel +from django.db.models.fields import AutoField, FieldDoesNotExist +from django.db.models.fields.proxy import OrderWrt +from django.db.models.loading import get_models, app_cache_ready +from django.utils.translation import activate, deactivate_all, get_language, string_concat +from django.utils.encoding import force_unicode, smart_str +from django.utils.datastructures import SortedDict + +# Calculate the verbose_name by converting from InitialCaps to "lowercase with spaces". +get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).lower().strip() + +DEFAULT_NAMES = ('verbose_name', 'db_table', 'ordering', + 'unique_together', 'permissions', 'get_latest_by', + 'order_with_respect_to', 'app_label', 'db_tablespace', + 'abstract') + +class Options(object): + def __init__(self, meta, app_label=None): + self.local_fields, self.local_many_to_many = [], [] + self.module_name, self.verbose_name = None, None + self.verbose_name_plural = None + self.db_table = '' + self.ordering = [] + self.unique_together = [] + self.permissions = [] + self.object_name, self.app_label = None, app_label + self.get_latest_by = None + self.order_with_respect_to = None + self.db_tablespace = settings.DEFAULT_TABLESPACE + self.admin = None + self.meta = meta + self.pk = None + self.has_auto_field, self.auto_field = False, None + self.one_to_one_field = None + self.abstract = False + self.parents = SortedDict() + self.duplicate_targets = {} + + def contribute_to_class(self, cls, name): + from django.db import connection + from django.db.backends.util import truncate_name + + cls._meta = self + self.installed = re.sub('\.models$', '', cls.__module__) in settings.INSTALLED_APPS + # First, construct the default values for these options. + self.object_name = cls.__name__ + self.module_name = self.object_name.lower() + self.verbose_name = get_verbose_name(self.object_name) + + # Next, apply any overridden values from 'class Meta'. + if self.meta: + meta_attrs = self.meta.__dict__.copy() + for name in self.meta.__dict__: + # Ignore any private attributes that Django doesn't care about. + # NOTE: We can't modify a dictionary's contents while looping + # over it, so we loop over the *original* dictionary instead. + if name.startswith('_'): + del meta_attrs[name] + for attr_name in DEFAULT_NAMES: + if attr_name in meta_attrs: + setattr(self, attr_name, meta_attrs.pop(attr_name)) + elif hasattr(self.meta, attr_name): + setattr(self, attr_name, getattr(self.meta, attr_name)) + + # unique_together can be either a tuple of tuples, or a single + # tuple of two strings. Normalize it to a tuple of tuples, so that + # calling code can uniformly expect that. + ut = meta_attrs.pop('unique_together', getattr(self, 'unique_together')) + if ut and not isinstance(ut[0], (tuple, list)): + ut = (ut,) + setattr(self, 'unique_together', ut) + + # verbose_name_plural is a special case because it uses a 's' + # by default. + setattr(self, 'verbose_name_plural', meta_attrs.pop('verbose_name_plural', string_concat(self.verbose_name, 's'))) + + # Any leftover attributes must be invalid. + if meta_attrs != {}: + raise TypeError, "'class Meta' got invalid attribute(s): %s" % ','.join(meta_attrs.keys()) + else: + self.verbose_name_plural = string_concat(self.verbose_name, 's') + del self.meta + + # If the db_table wasn't provided, use the app_label + module_name. + if not self.db_table: + self.db_table = "%s_%s" % (self.app_label, self.module_name) + self.db_table = truncate_name(self.db_table, connection.ops.max_name_length()) + + + def _prepare(self, model): + if self.order_with_respect_to: + self.order_with_respect_to = self.get_field(self.order_with_respect_to) + self.ordering = ('_order',) + else: + self.order_with_respect_to = None + + if self.pk is None: + if self.parents: + # Promote the first parent link in lieu of adding yet another + # field. + field = self.parents.value_for_index(0) + field.primary_key = True + self.setup_pk(field) + else: + auto = AutoField(verbose_name='ID', primary_key=True, + auto_created=True) + model.add_to_class('id', auto) + + # Determine any sets of fields that are pointing to the same targets + # (e.g. two ForeignKeys to the same remote model). The query + # construction code needs to know this. At the end of this, + # self.duplicate_targets will map each duplicate field column to the + # columns it duplicates. + collections = {} + for column, target in self.duplicate_targets.iteritems(): + try: + collections[target].add(column) + except KeyError: + collections[target] = set([column]) + self.duplicate_targets = {} + for elt in collections.itervalues(): + if len(elt) == 1: + continue + for column in elt: + self.duplicate_targets[column] = elt.difference(set([column])) + + def add_field(self, field): + # Insert the given field in the order in which it was created, using + # the "creation_counter" attribute of the field. + # Move many-to-many related fields from self.fields into + # self.many_to_many. + if field.rel and isinstance(field.rel, ManyToManyRel): + self.local_many_to_many.insert(bisect(self.local_many_to_many, field), field) + if hasattr(self, '_m2m_cache'): + del self._m2m_cache + else: + self.local_fields.insert(bisect(self.local_fields, field), field) + self.setup_pk(field) + if hasattr(self, '_field_cache'): + del self._field_cache + del self._field_name_cache + + if hasattr(self, '_name_map'): + del self._name_map + + def setup_pk(self, field): + if not self.pk and field.primary_key: + self.pk = field + field.serialize = False + + def __repr__(self): + return '<Options for %s>' % self.object_name + + def __str__(self): + return "%s.%s" % (smart_str(self.app_label), smart_str(self.module_name)) + + def verbose_name_raw(self): + """ + There are a few places where the untranslated verbose name is needed + (so that we get the same value regardless of currently active + locale). + """ + lang = get_language() + deactivate_all() + raw = force_unicode(self.verbose_name) + activate(lang) + return raw + verbose_name_raw = property(verbose_name_raw) + + def _fields(self): + """ + The getter for self.fields. This returns the list of field objects + available to this model (including through parent models). + + Callers are not permitted to modify this list, since it's a reference + to this instance (not a copy). + """ + try: + self._field_name_cache + except AttributeError: + self._fill_fields_cache() + return self._field_name_cache + fields = property(_fields) + + def get_fields_with_model(self): + """ + Returns a sequence of (field, model) pairs for all fields. The "model" + element is None for fields on the current model. Mostly of use when + constructing queries so that we know which model a field belongs to. + """ + try: + self._field_cache + except AttributeError: + self._fill_fields_cache() + return self._field_cache + + def _fill_fields_cache(self): + cache = [] + for parent in self.parents: + for field, model in parent._meta.get_fields_with_model(): + if model: + cache.append((field, model)) + else: + cache.append((field, parent)) + cache.extend([(f, None) for f in self.local_fields]) + self._field_cache = tuple(cache) + self._field_name_cache = [x for x, _ in cache] + + def _many_to_many(self): + try: + self._m2m_cache + except AttributeError: + self._fill_m2m_cache() + return self._m2m_cache.keys() + many_to_many = property(_many_to_many) + + def get_m2m_with_model(self): + """ + The many-to-many version of get_fields_with_model(). + """ + try: + self._m2m_cache + except AttributeError: + self._fill_m2m_cache() + return self._m2m_cache.items() + + def _fill_m2m_cache(self): + cache = SortedDict() + for parent in self.parents: + for field, model in parent._meta.get_m2m_with_model(): + if model: + cache[field] = model + else: + cache[field] = parent + for field in self.local_many_to_many: + cache[field] = None + self._m2m_cache = cache + + def get_field(self, name, many_to_many=True): + """ + Returns the requested field by name. Raises FieldDoesNotExist on error. + """ + to_search = many_to_many and (self.fields + self.many_to_many) or self.fields + for f in to_search: + if f.name == name: + return f + raise FieldDoesNotExist, '%s has no field named %r' % (self.object_name, name) + + def get_field_by_name(self, name): + """ + Returns the (field_object, model, direct, m2m), where field_object is + the Field instance for the given name, model is the model containing + this field (None for local fields), direct is True if the field exists + on this model, and m2m is True for many-to-many relations. When + 'direct' is False, 'field_object' is the corresponding RelatedObject + for this field (since the field doesn't have an instance associated + with it). + + Uses a cache internally, so after the first access, this is very fast. + """ + try: + try: + return self._name_map[name] + except AttributeError: + cache = self.init_name_map() + return cache[name] + except KeyError: + raise FieldDoesNotExist('%s has no field named %r' + % (self.object_name, name)) + + def get_all_field_names(self): + """ + Returns a list of all field names that are possible for this model + (including reverse relation names). + """ + try: + cache = self._name_map + except AttributeError: + cache = self.init_name_map() + names = cache.keys() + names.sort() + return names + + def init_name_map(self): + """ + Initialises the field name -> field object mapping. + """ + cache = {} + # We intentionally handle related m2m objects first so that symmetrical + # m2m accessor names can be overridden, if necessary. + for f, model in self.get_all_related_m2m_objects_with_model(): + cache[f.field.related_query_name()] = (f, model, False, True) + for f, model in self.get_all_related_objects_with_model(): + cache[f.field.related_query_name()] = (f, model, False, False) + for f, model in self.get_m2m_with_model(): + cache[f.name] = (f, model, True, True) + for f, model in self.get_fields_with_model(): + cache[f.name] = (f, model, True, False) + if self.order_with_respect_to: + cache['_order'] = OrderWrt(), None, True, False + if app_cache_ready(): + self._name_map = cache + return cache + + def get_add_permission(self): + return 'add_%s' % self.object_name.lower() + + def get_change_permission(self): + return 'change_%s' % self.object_name.lower() + + def get_delete_permission(self): + return 'delete_%s' % self.object_name.lower() + + def get_all_related_objects(self, local_only=False): + try: + self._related_objects_cache + except AttributeError: + self._fill_related_objects_cache() + if local_only: + return [k for k, v in self._related_objects_cache.items() if not v] + return self._related_objects_cache.keys() + + def get_all_related_objects_with_model(self): + """ + Returns a list of (related-object, model) pairs. Similar to + get_fields_with_model(). + """ + try: + self._related_objects_cache + except AttributeError: + self._fill_related_objects_cache() + return self._related_objects_cache.items() + + def _fill_related_objects_cache(self): + cache = SortedDict() + parent_list = self.get_parent_list() + for parent in self.parents: + for obj, model in parent._meta.get_all_related_objects_with_model(): + if (obj.field.creation_counter < 0 or obj.field.rel.parent_link) and obj.model not in parent_list: + continue + if not model: + cache[obj] = parent + else: + cache[obj] = model + for klass in get_models(): + for f in klass._meta.local_fields: + if f.rel and not isinstance(f.rel.to, str) and self == f.rel.to._meta: + cache[RelatedObject(f.rel.to, klass, f)] = None + self._related_objects_cache = cache + + def get_all_related_many_to_many_objects(self, local_only=False): + try: + cache = self._related_many_to_many_cache + except AttributeError: + cache = self._fill_related_many_to_many_cache() + if local_only: + return [k for k, v in cache.items() if not v] + return cache.keys() + + def get_all_related_m2m_objects_with_model(self): + """ + Returns a list of (related-m2m-object, model) pairs. Similar to + get_fields_with_model(). + """ + try: + cache = self._related_many_to_many_cache + except AttributeError: + cache = self._fill_related_many_to_many_cache() + return cache.items() + + def _fill_related_many_to_many_cache(self): + cache = SortedDict() + parent_list = self.get_parent_list() + for parent in self.parents: + for obj, model in parent._meta.get_all_related_m2m_objects_with_model(): + if obj.field.creation_counter < 0 and obj.model not in parent_list: + continue + if not model: + cache[obj] = parent + else: + cache[obj] = model + for klass in get_models(): + for f in klass._meta.local_many_to_many: + if f.rel and not isinstance(f.rel.to, str) and self == f.rel.to._meta: + cache[RelatedObject(f.rel.to, klass, f)] = None + if app_cache_ready(): + self._related_many_to_many_cache = cache + return cache + + def get_followed_related_objects(self, follow=None): + if follow == None: + follow = self.get_follow() + return [f for f in self.get_all_related_objects() if follow.get(f.name, None)] + + def get_data_holders(self, follow=None): + if follow == None: + follow = self.get_follow() + return [f for f in self.fields + self.many_to_many + self.get_all_related_objects() if follow.get(f.name, None)] + + def get_follow(self, override=None): + follow = {} + for f in self.fields + self.many_to_many + self.get_all_related_objects(): + if override and f.name in override: + child_override = override[f.name] + else: + child_override = None + fol = f.get_follow(child_override) + if fol != None: + follow[f.name] = fol + return follow + + def get_base_chain(self, model): + """ + Returns a list of parent classes leading to 'model' (order from closet + to most distant ancestor). This has to handle the case were 'model' is + a granparent or even more distant relation. + """ + if not self.parents: + return + if model in self.parents: + return [model] + for parent in self.parents: + res = parent._meta.get_base_chain(model) + if res: + res.insert(0, parent) + return res + raise TypeError('%r is not an ancestor of this model' + % model._meta.module_name) + + def get_parent_list(self): + """ + Returns a list of all the ancestor of this model as a list. Useful for + determining if something is an ancestor, regardless of lineage. + """ + result = set() + for parent in self.parents: + result.add(parent) + result.update(parent._meta.get_parent_list()) + return result + + def get_ordered_objects(self): + "Returns a list of Options objects that are ordered with respect to this object." + if not hasattr(self, '_ordered_objects'): + objects = [] + # TODO + #for klass in get_models(get_app(self.app_label)): + # opts = klass._meta + # if opts.order_with_respect_to and opts.order_with_respect_to.rel \ + # and self == opts.order_with_respect_to.rel.to._meta: + # objects.append(opts) + self._ordered_objects = objects + return self._ordered_objects + + def has_field_type(self, field_type, follow=None): + """ + Returns True if this object's admin form has at least one of the given + field_type (e.g. FileField). + """ + # TODO: follow + if not hasattr(self, '_field_types'): + self._field_types = {} + if field_type not in self._field_types: + try: + # First check self.fields. + for f in self.fields: + if isinstance(f, field_type): + raise StopIteration + # Failing that, check related fields. + for related in self.get_followed_related_objects(follow): + for f in related.opts.fields: + if isinstance(f, field_type): + raise StopIteration + except StopIteration: + self._field_types[field_type] = True + else: + self._field_types[field_type] = False + return self._field_types[field_type] diff --git a/webapp/django/db/models/query.py b/webapp/django/db/models/query.py new file mode 100644 index 0000000000..2ff1c26344 --- /dev/null +++ b/webapp/django/db/models/query.py @@ -0,0 +1,884 @@ +try: + set +except NameError: + from sets import Set as set # Python 2.3 fallback + +from django.db import connection, transaction, IntegrityError +from django.db.models.fields import DateField +from django.db.models.query_utils import Q, select_related_descend +from django.db.models import signals, sql +from django.utils.datastructures import SortedDict + + +# Used to control how many objects are worked with at once in some cases (e.g. +# when deleting objects). +CHUNK_SIZE = 100 +ITER_CHUNK_SIZE = CHUNK_SIZE + +# Pull into this namespace for backwards compatibility. +EmptyResultSet = sql.EmptyResultSet + + +class CyclicDependency(Exception): + """ + An error when dealing with a collection of objects that have a cyclic + dependency, i.e. when deleting multiple objects. + """ + pass + + +class CollectedObjects(object): + """ + A container that stores keys and lists of values along with remembering the + parent objects for all the keys. + + This is used for the database object deletion routines so that we can + calculate the 'leaf' objects which should be deleted first. + """ + + def __init__(self): + self.data = {} + self.children = {} + + def add(self, model, pk, obj, parent_model, nullable=False): + """ + Adds an item to the container. + + Arguments: + * model - the class of the object being added. + * pk - the primary key. + * obj - the object itself. + * parent_model - the model of the parent object that this object was + reached through. + * nullable - should be True if this relation is nullable. + + Returns True if the item already existed in the structure and + False otherwise. + """ + d = self.data.setdefault(model, SortedDict()) + retval = pk in d + d[pk] = obj + # Nullable relationships can be ignored -- they are nulled out before + # deleting, and therefore do not affect the order in which objects + # have to be deleted. + if parent_model is not None and not nullable: + self.children.setdefault(parent_model, []).append(model) + return retval + + def __contains__(self, key): + return self.data.__contains__(key) + + def __getitem__(self, key): + return self.data[key] + + def __nonzero__(self): + return bool(self.data) + + def iteritems(self): + for k in self.ordered_keys(): + yield k, self[k] + + def items(self): + return list(self.iteritems()) + + def keys(self): + return self.ordered_keys() + + def ordered_keys(self): + """ + Returns the models in the order that they should be dealt with (i.e. + models with no dependencies first). + """ + dealt_with = SortedDict() + # Start with items that have no children + models = self.data.keys() + while len(dealt_with) < len(models): + found = False + for model in models: + children = self.children.setdefault(model, []) + if len([c for c in children if c not in dealt_with]) == 0: + dealt_with[model] = None + found = True + if not found: + raise CyclicDependency( + "There is a cyclic dependency of items to be processed.") + + return dealt_with.keys() + + def unordered_keys(self): + """ + Fallback for the case where is a cyclic dependency but we don't care. + """ + return self.data.keys() + + +class QuerySet(object): + """ + Represents a lazy database lookup for a set of objects. + """ + def __init__(self, model=None, query=None): + self.model = model + self.query = query or sql.Query(self.model, connection) + self._result_cache = None + self._iter = None + self._sticky_filter = False + + ######################## + # PYTHON MAGIC METHODS # + ######################## + + def __getstate__(self): + """ + Allows the QuerySet to be pickled. + """ + # Force the cache to be fully populated. + len(self) + + obj_dict = self.__dict__.copy() + obj_dict['_iter'] = None + return obj_dict + + def __repr__(self): + return repr(list(self)) + + def __len__(self): + # Since __len__ is called quite frequently (for example, as part of + # list(qs), we make some effort here to be as efficient as possible + # whilst not messing up any existing iterators against the QuerySet. + if self._result_cache is None: + if self._iter: + self._result_cache = list(self._iter) + else: + self._result_cache = list(self.iterator()) + elif self._iter: + self._result_cache.extend(list(self._iter)) + return len(self._result_cache) + + def __iter__(self): + if self._result_cache is None: + self._iter = self.iterator() + self._result_cache = [] + if self._iter: + return self._result_iter() + # Python's list iterator is better than our version when we're just + # iterating over the cache. + return iter(self._result_cache) + + def _result_iter(self): + pos = 0 + while 1: + upper = len(self._result_cache) + while pos < upper: + yield self._result_cache[pos] + pos = pos + 1 + if not self._iter: + raise StopIteration + if len(self._result_cache) <= pos: + self._fill_cache() + + def __nonzero__(self): + if self._result_cache is not None: + return bool(self._result_cache) + try: + iter(self).next() + except StopIteration: + return False + return True + + def __getitem__(self, k): + """ + Retrieves an item or slice from the set of results. + """ + if not isinstance(k, (slice, int, long)): + raise TypeError + assert ((not isinstance(k, slice) and (k >= 0)) + or (isinstance(k, slice) and (k.start is None or k.start >= 0) + and (k.stop is None or k.stop >= 0))), \ + "Negative indexing is not supported." + + if self._result_cache is not None: + if self._iter is not None: + # The result cache has only been partially populated, so we may + # need to fill it out a bit more. + if isinstance(k, slice): + if k.stop is not None: + # Some people insist on passing in strings here. + bound = int(k.stop) + else: + bound = None + else: + bound = k + 1 + if len(self._result_cache) < bound: + self._fill_cache(bound - len(self._result_cache)) + return self._result_cache[k] + + if isinstance(k, slice): + qs = self._clone() + if k.start is not None: + start = int(k.start) + else: + start = None + if k.stop is not None: + stop = int(k.stop) + else: + stop = None + qs.query.set_limits(start, stop) + return k.step and list(qs)[::k.step] or qs + try: + qs = self._clone() + qs.query.set_limits(k, k + 1) + return list(qs)[0] + except self.model.DoesNotExist, e: + raise IndexError, e.args + + def __and__(self, other): + self._merge_sanity_check(other) + if isinstance(other, EmptyQuerySet): + return other._clone() + combined = self._clone() + combined.query.combine(other.query, sql.AND) + return combined + + def __or__(self, other): + self._merge_sanity_check(other) + combined = self._clone() + if isinstance(other, EmptyQuerySet): + return combined + combined.query.combine(other.query, sql.OR) + return combined + + #################################### + # METHODS THAT DO DATABASE QUERIES # + #################################### + + def iterator(self): + """ + An iterator over the results from applying this QuerySet to the + database. + """ + fill_cache = self.query.select_related + if isinstance(fill_cache, dict): + requested = fill_cache + else: + requested = None + max_depth = self.query.max_depth + extra_select = self.query.extra_select.keys() + index_start = len(extra_select) + for row in self.query.results_iter(): + if fill_cache: + obj, _ = get_cached_row(self.model, row, index_start, + max_depth, requested=requested) + else: + obj = self.model(*row[index_start:]) + for i, k in enumerate(extra_select): + setattr(obj, k, row[i]) + yield obj + + def count(self): + """ + Performs a SELECT COUNT() and returns the number of records as an + integer. + + If the QuerySet is already fully cached this simply returns the length + of the cached results set to avoid multiple SELECT COUNT(*) calls. + """ + if self._result_cache is not None and not self._iter: + return len(self._result_cache) + + return self.query.get_count() + + def get(self, *args, **kwargs): + """ + Performs the query and returns a single object matching the given + keyword arguments. + """ + clone = self.filter(*args, **kwargs) + num = len(clone) + if num == 1: + return clone._result_cache[0] + if not num: + raise self.model.DoesNotExist("%s matching query does not exist." + % self.model._meta.object_name) + raise self.model.MultipleObjectsReturned("get() returned more than one %s -- it returned %s! Lookup parameters were %s" + % (self.model._meta.object_name, num, kwargs)) + + def create(self, **kwargs): + """ + Creates a new object with the given kwargs, saving it to the database + and returning the created object. + """ + obj = self.model(**kwargs) + obj.save() + return obj + + def get_or_create(self, **kwargs): + """ + Looks up an object with the given kwargs, creating one if necessary. + Returns a tuple of (object, created), where created is a boolean + specifying whether an object was created. + """ + assert kwargs, \ + 'get_or_create() must be passed at least one keyword argument' + defaults = kwargs.pop('defaults', {}) + try: + return self.get(**kwargs), False + except self.model.DoesNotExist: + try: + params = dict([(k, v) for k, v in kwargs.items() if '__' not in k]) + params.update(defaults) + obj = self.model(**params) + sid = transaction.savepoint() + obj.save() + transaction.savepoint_commit(sid) + return obj, True + except IntegrityError, e: + transaction.savepoint_rollback(sid) + try: + return self.get(**kwargs), False + except self.model.DoesNotExist: + raise e + + def latest(self, field_name=None): + """ + Returns the latest object, according to the model's 'get_latest_by' + option or optional given field_name. + """ + latest_by = field_name or self.model._meta.get_latest_by + assert bool(latest_by), "latest() requires either a field_name parameter or 'get_latest_by' in the model" + assert self.query.can_filter(), \ + "Cannot change a query once a slice has been taken." + obj = self._clone() + obj.query.set_limits(high=1) + obj.query.add_ordering('-%s' % latest_by) + return obj.get() + + def in_bulk(self, id_list): + """ + Returns a dictionary mapping each of the given IDs to the object with + that ID. + """ + assert self.query.can_filter(), \ + "Cannot use 'limit' or 'offset' with in_bulk" + assert isinstance(id_list, (tuple, list)), \ + "in_bulk() must be provided with a list of IDs." + if not id_list: + return {} + qs = self._clone() + qs.query.add_filter(('pk__in', id_list)) + return dict([(obj._get_pk_val(), obj) for obj in qs.iterator()]) + + def delete(self): + """ + Deletes the records in the current QuerySet. + """ + assert self.query.can_filter(), \ + "Cannot use 'limit' or 'offset' with delete." + + del_query = self._clone() + + # Disable non-supported fields. + del_query.query.select_related = False + del_query.query.clear_ordering() + + # Delete objects in chunks to prevent the list of related objects from + # becoming too long. + while 1: + # Collect all the objects to be deleted in this chunk, and all the + # objects that are related to the objects that are to be deleted. + seen_objs = CollectedObjects() + for object in del_query[:CHUNK_SIZE]: + object._collect_sub_objects(seen_objs) + + if not seen_objs: + break + delete_objects(seen_objs) + + # Clear the result cache, in case this QuerySet gets reused. + self._result_cache = None + delete.alters_data = True + + def update(self, **kwargs): + """ + Updates all elements in the current QuerySet, setting all the given + fields to the appropriate values. + """ + assert self.query.can_filter(), \ + "Cannot update a query once a slice has been taken." + query = self.query.clone(sql.UpdateQuery) + query.add_update_values(kwargs) + rows = query.execute_sql(None) + transaction.commit_unless_managed() + self._result_cache = None + return rows + update.alters_data = True + + def _update(self, values): + """ + A version of update that accepts field objects instead of field names. + Used primarily for model saving and not intended for use by general + code (it requires too much poking around at model internals to be + useful at that level). + """ + assert self.query.can_filter(), \ + "Cannot update a query once a slice has been taken." + query = self.query.clone(sql.UpdateQuery) + query.add_update_fields(values) + self._result_cache = None + return query.execute_sql(None) + _update.alters_data = True + + ################################################## + # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS # + ################################################## + + def values(self, *fields): + return self._clone(klass=ValuesQuerySet, setup=True, _fields=fields) + + def values_list(self, *fields, **kwargs): + flat = kwargs.pop('flat', False) + if kwargs: + raise TypeError('Unexpected keyword arguments to values_list: %s' + % (kwargs.keys(),)) + if flat and len(fields) > 1: + raise TypeError("'flat' is not valid when values_list is called with more than one field.") + return self._clone(klass=ValuesListQuerySet, setup=True, flat=flat, + _fields=fields) + + def dates(self, field_name, kind, order='ASC'): + """ + Returns a list of datetime objects representing all available dates for + the given field_name, scoped to 'kind'. + """ + assert kind in ("month", "year", "day"), \ + "'kind' must be one of 'year', 'month' or 'day'." + assert order in ('ASC', 'DESC'), \ + "'order' must be either 'ASC' or 'DESC'." + return self._clone(klass=DateQuerySet, setup=True, + _field_name=field_name, _kind=kind, _order=order) + + def none(self): + """ + Returns an empty QuerySet. + """ + return self._clone(klass=EmptyQuerySet) + + ################################################################## + # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET # + ################################################################## + + def all(self): + """ + Returns a new QuerySet that is a copy of the current one. This allows a + QuerySet to proxy for a model manager in some cases. + """ + return self._clone() + + def filter(self, *args, **kwargs): + """ + Returns a new QuerySet instance with the args ANDed to the existing + set. + """ + return self._filter_or_exclude(False, *args, **kwargs) + + def exclude(self, *args, **kwargs): + """ + Returns a new QuerySet instance with NOT (args) ANDed to the existing + set. + """ + return self._filter_or_exclude(True, *args, **kwargs) + + def _filter_or_exclude(self, negate, *args, **kwargs): + if args or kwargs: + assert self.query.can_filter(), \ + "Cannot filter a query once a slice has been taken." + + clone = self._clone() + if negate: + clone.query.add_q(~Q(*args, **kwargs)) + else: + clone.query.add_q(Q(*args, **kwargs)) + return clone + + def complex_filter(self, filter_obj): + """ + Returns a new QuerySet instance with filter_obj added to the filters. + + filter_obj can be a Q object (or anything with an add_to_query() + method) or a dictionary of keyword lookup arguments. + + This exists to support framework features such as 'limit_choices_to', + and usually it will be more natural to use other methods. + """ + if isinstance(filter_obj, Q) or hasattr(filter_obj, 'add_to_query'): + clone = self._clone() + clone.query.add_q(filter_obj) + return clone + else: + return self._filter_or_exclude(None, **filter_obj) + + def select_related(self, *fields, **kwargs): + """ + Returns a new QuerySet instance that will select related objects. + + If fields are specified, they must be ForeignKey fields and only those + related objects are included in the selection. + """ + depth = kwargs.pop('depth', 0) + if kwargs: + raise TypeError('Unexpected keyword arguments to select_related: %s' + % (kwargs.keys(),)) + obj = self._clone() + if fields: + if depth: + raise TypeError('Cannot pass both "depth" and fields to select_related()') + obj.query.add_select_related(fields) + else: + obj.query.select_related = True + if depth: + obj.query.max_depth = depth + return obj + + def dup_select_related(self, other): + """ + Copies the related selection status from the QuerySet 'other' to the + current QuerySet. + """ + self.query.select_related = other.query.select_related + + def order_by(self, *field_names): + """ + Returns a new QuerySet instance with the ordering changed. + """ + assert self.query.can_filter(), \ + "Cannot reorder a query once a slice has been taken." + obj = self._clone() + obj.query.clear_ordering() + obj.query.add_ordering(*field_names) + return obj + + def distinct(self, true_or_false=True): + """ + Returns a new QuerySet instance that will select only distinct results. + """ + obj = self._clone() + obj.query.distinct = true_or_false + return obj + + def extra(self, select=None, where=None, params=None, tables=None, + order_by=None, select_params=None): + """ + Adds extra SQL fragments to the query. + """ + assert self.query.can_filter(), \ + "Cannot change a query once a slice has been taken" + clone = self._clone() + clone.query.add_extra(select, select_params, where, params, tables, order_by) + return clone + + def reverse(self): + """ + Reverses the ordering of the QuerySet. + """ + clone = self._clone() + clone.query.standard_ordering = not clone.query.standard_ordering + return clone + + ################### + # PRIVATE METHODS # + ################### + + def _clone(self, klass=None, setup=False, **kwargs): + if klass is None: + klass = self.__class__ + query = self.query.clone() + if self._sticky_filter: + query.filter_is_sticky = True + c = klass(model=self.model, query=query) + c.__dict__.update(kwargs) + if setup and hasattr(c, '_setup_query'): + c._setup_query() + return c + + def _fill_cache(self, num=None): + """ + Fills the result cache with 'num' more entries (or until the results + iterator is exhausted). + """ + if self._iter: + try: + for i in range(num or ITER_CHUNK_SIZE): + self._result_cache.append(self._iter.next()) + except StopIteration: + self._iter = None + + def _next_is_sticky(self): + """ + Indicates that the next filter call and the one following that should + be treated as a single filter. This is only important when it comes to + determining when to reuse tables for many-to-many filters. Required so + that we can filter naturally on the results of related managers. + + This doesn't return a clone of the current QuerySet (it returns + "self"). The method is only used internally and should be immediately + followed by a filter() that does create a clone. + """ + self._sticky_filter = True + return self + + def _merge_sanity_check(self, other): + """ + Checks that we are merging two comparable QuerySet classes. By default + this does nothing, but see the ValuesQuerySet for an example of where + it's useful. + """ + pass + + +class ValuesQuerySet(QuerySet): + def __init__(self, *args, **kwargs): + super(ValuesQuerySet, self).__init__(*args, **kwargs) + # select_related isn't supported in values(). (FIXME -#3358) + self.query.select_related = False + + # QuerySet.clone() will also set up the _fields attribute with the + # names of the model fields to select. + + def iterator(self): + if (not self.extra_names and + len(self.field_names) != len(self.model._meta.fields)): + self.query.trim_extra_select(self.extra_names) + names = self.query.extra_select.keys() + self.field_names + for row in self.query.results_iter(): + yield dict(zip(names, row)) + + def _setup_query(self): + """ + Constructs the field_names list that the values query will be + retrieving. + + Called by the _clone() method after initializing the rest of the + instance. + """ + self.extra_names = [] + if self._fields: + if not self.query.extra_select: + field_names = list(self._fields) + else: + field_names = [] + for f in self._fields: + if self.query.extra_select.has_key(f): + self.extra_names.append(f) + else: + field_names.append(f) + else: + # Default to all fields. + field_names = [f.attname for f in self.model._meta.fields] + + self.query.add_fields(field_names, False) + self.query.default_cols = False + self.field_names = field_names + + def _clone(self, klass=None, setup=False, **kwargs): + """ + Cloning a ValuesQuerySet preserves the current fields. + """ + c = super(ValuesQuerySet, self)._clone(klass, **kwargs) + c._fields = self._fields[:] + c.field_names = self.field_names + c.extra_names = self.extra_names + if setup and hasattr(c, '_setup_query'): + c._setup_query() + return c + + def _merge_sanity_check(self, other): + super(ValuesQuerySet, self)._merge_sanity_check(other) + if (set(self.extra_names) != set(other.extra_names) or + set(self.field_names) != set(other.field_names)): + raise TypeError("Merging '%s' classes must involve the same values in each case." + % self.__class__.__name__) + + +class ValuesListQuerySet(ValuesQuerySet): + def iterator(self): + self.query.trim_extra_select(self.extra_names) + if self.flat and len(self._fields) == 1: + for row in self.query.results_iter(): + yield row[0] + elif not self.query.extra_select: + for row in self.query.results_iter(): + yield tuple(row) + else: + # When extra(select=...) is involved, the extra cols come are + # always at the start of the row, so we need to reorder the fields + # to match the order in self._fields. + names = self.query.extra_select.keys() + self.field_names + for row in self.query.results_iter(): + data = dict(zip(names, row)) + yield tuple([data[f] for f in self._fields]) + + def _clone(self, *args, **kwargs): + clone = super(ValuesListQuerySet, self)._clone(*args, **kwargs) + clone.flat = self.flat + return clone + + +class DateQuerySet(QuerySet): + def iterator(self): + return self.query.results_iter() + + def _setup_query(self): + """ + Sets up any special features of the query attribute. + + Called by the _clone() method after initializing the rest of the + instance. + """ + self.query = self.query.clone(klass=sql.DateQuery, setup=True) + self.query.select = [] + field = self.model._meta.get_field(self._field_name, many_to_many=False) + assert isinstance(field, DateField), "%r isn't a DateField." \ + % field_name + self.query.add_date_select(field, self._kind, self._order) + if field.null: + self.query.add_filter(('%s__isnull' % field.name, False)) + + def _clone(self, klass=None, setup=False, **kwargs): + c = super(DateQuerySet, self)._clone(klass, False, **kwargs) + c._field_name = self._field_name + c._kind = self._kind + if setup and hasattr(c, '_setup_query'): + c._setup_query() + return c + + +class EmptyQuerySet(QuerySet): + def __init__(self, model=None, query=None): + super(EmptyQuerySet, self).__init__(model, query) + self._result_cache = [] + + def __and__(self, other): + return self._clone() + + def __or__(self, other): + return other._clone() + + def count(self): + return 0 + + def delete(self): + pass + + def _clone(self, klass=None, setup=False, **kwargs): + c = super(EmptyQuerySet, self)._clone(klass, **kwargs) + c._result_cache = [] + return c + + def iterator(self): + # This slightly odd construction is because we need an empty generator + # (it raises StopIteration immediately). + yield iter([]).next() + + +def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0, + requested=None): + """ + Helper function that recursively returns an object with the specified + related attributes already populated. + """ + if max_depth and requested is None and cur_depth > max_depth: + # We've recursed deeply enough; stop now. + return None + + restricted = requested is not None + index_end = index_start + len(klass._meta.fields) + fields = row[index_start:index_end] + if not [x for x in fields if x is not None]: + # If we only have a list of Nones, there was not related object. + return None, index_end + obj = klass(*fields) + for f in klass._meta.fields: + if not select_related_descend(f, restricted, requested): + continue + if restricted: + next = requested[f.name] + else: + next = None + cached_row = get_cached_row(f.rel.to, row, index_end, max_depth, + cur_depth+1, next) + if cached_row: + rel_obj, index_end = cached_row + setattr(obj, f.get_cache_name(), rel_obj) + return obj, index_end + + +def delete_objects(seen_objs): + """ + Iterate through a list of seen classes, and remove any instances that are + referred to. + """ + try: + ordered_classes = seen_objs.keys() + except CyclicDependency: + # If there is a cyclic dependency, we cannot in general delete the + # objects. However, if an appropriate transaction is set up, or if the + # database is lax enough, it will succeed. So for now, we go ahead and + # try anyway. + ordered_classes = seen_objs.unordered_keys() + + obj_pairs = {} + for cls in ordered_classes: + items = seen_objs[cls].items() + items.sort() + obj_pairs[cls] = items + + # Pre-notify all instances to be deleted. + for pk_val, instance in items: + signals.pre_delete.send(sender=cls, instance=instance) + + pk_list = [pk for pk,instance in items] + del_query = sql.DeleteQuery(cls, connection) + del_query.delete_batch_related(pk_list) + + update_query = sql.UpdateQuery(cls, connection) + for field, model in cls._meta.get_fields_with_model(): + if (field.rel and field.null and field.rel.to in seen_objs and + filter(lambda f: f.column == field.column, + field.rel.to._meta.fields)): + if model: + sql.UpdateQuery(model, connection).clear_related(field, + pk_list) + else: + update_query.clear_related(field, pk_list) + + # Now delete the actual data. + for cls in ordered_classes: + items = obj_pairs[cls] + items.reverse() + + pk_list = [pk for pk,instance in items] + del_query = sql.DeleteQuery(cls, connection) + del_query.delete_batch(pk_list) + + # Last cleanup; set NULLs where there once was a reference to the + # object, NULL the primary key of the found objects, and perform + # post-notification. + for pk_val, instance in items: + for field in cls._meta.fields: + if field.rel and field.null and field.rel.to in seen_objs: + setattr(instance, field.attname, None) + + signals.post_delete.send(sender=cls, instance=instance) + setattr(instance, cls._meta.pk.attname, None) + + transaction.commit_unless_managed() + + +def insert_query(model, values, return_id=False, raw_values=False): + """ + Inserts a new record for the given model. This provides an interface to + the InsertQuery class and is how Model.save() is implemented. It is not + part of the public API. + """ + query = sql.InsertQuery(model, connection) + query.insert_values(values, raw_values) + return query.execute_sql(return_id) diff --git a/webapp/django/db/models/query_utils.py b/webapp/django/db/models/query_utils.py new file mode 100644 index 0000000000..8dbb1ec667 --- /dev/null +++ b/webapp/django/db/models/query_utils.py @@ -0,0 +1,67 @@ +""" +Various data structures used in query construction. + +Factored out from django.db.models.query so that they can also be used by other +modules without getting into circular import difficulties. +""" + +from copy import deepcopy + +from django.utils import tree + +class QueryWrapper(object): + """ + A type that indicates the contents are an SQL fragment and the associate + parameters. Can be used to pass opaque data to a where-clause, for example. + """ + def __init__(self, sql, params): + self.data = sql, params + +class Q(tree.Node): + """ + Encapsulates filters as objects that can then be combined logically (using + & and |). + """ + # Connection types + AND = 'AND' + OR = 'OR' + default = AND + + def __init__(self, *args, **kwargs): + super(Q, self).__init__(children=list(args) + kwargs.items()) + + def _combine(self, other, conn): + if not isinstance(other, Q): + raise TypeError(other) + obj = deepcopy(self) + obj.add(other, conn) + return obj + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + def __invert__(self): + obj = deepcopy(self) + obj.negate() + return obj + +def select_related_descend(field, restricted, requested): + """ + Returns True if this field should be used to descend deeper for + select_related() purposes. Used by both the query construction code + (sql.query.fill_related_selections()) and the model instance creation code + (query.get_cached_row()). + """ + if not field.rel: + return False + if field.rel.parent_link: + return False + if restricted and field.name not in requested: + return False + if not restricted and field.null: + return False + return True + diff --git a/webapp/django/db/models/related.py b/webapp/django/db/models/related.py new file mode 100644 index 0000000000..2c1dc5c516 --- /dev/null +++ b/webapp/django/db/models/related.py @@ -0,0 +1,142 @@ +class BoundRelatedObject(object): + def __init__(self, related_object, field_mapping, original): + self.relation = related_object + self.field_mappings = field_mapping[related_object.name] + + def template_name(self): + raise NotImplementedError + + def __repr__(self): + return repr(self.__dict__) + +class RelatedObject(object): + def __init__(self, parent_model, model, field): + self.parent_model = parent_model + self.model = model + self.opts = model._meta + self.field = field + self.edit_inline = field.rel.edit_inline + self.name = '%s:%s' % (self.opts.app_label, self.opts.module_name) + self.var_name = self.opts.object_name.lower() + + def flatten_data(self, follow, obj=None): + new_data = {} + rel_instances = self.get_list(obj) + for i, rel_instance in enumerate(rel_instances): + instance_data = {} + for f in self.opts.fields + self.opts.many_to_many: + # TODO: Fix for recursive manipulators. + fol = follow.get(f.name, None) + if fol: + field_data = f.flatten_data(fol, rel_instance) + for name, value in field_data.items(): + instance_data['%s.%d.%s' % (self.var_name, i, name)] = value + new_data.update(instance_data) + return new_data + + def extract_data(self, data): + """ + Pull out the data meant for inline objects of this class, + i.e. anything starting with our module name. + """ + return data # TODO + + def get_list(self, parent_instance=None): + "Get the list of this type of object from an instance of the parent class." + if parent_instance is not None: + attr = getattr(parent_instance, self.get_accessor_name()) + if self.field.rel.multiple: + # For many-to-many relationships, return a list of objects + # corresponding to the xxx_num_in_admin options of the field + objects = list(attr.all()) + + count = len(objects) + self.field.rel.num_extra_on_change + if self.field.rel.min_num_in_admin: + count = max(count, self.field.rel.min_num_in_admin) + if self.field.rel.max_num_in_admin: + count = min(count, self.field.rel.max_num_in_admin) + + change = count - len(objects) + if change > 0: + return objects + [None] * change + if change < 0: + return objects[:change] + else: # Just right + return objects + else: + # A one-to-one relationship, so just return the single related + # object + return [attr] + else: + if self.field.rel.min_num_in_admin: + return [None] * max(self.field.rel.num_in_admin, self.field.rel.min_num_in_admin) + else: + return [None] * self.field.rel.num_in_admin + + def get_db_prep_lookup(self, lookup_type, value): + # Defer to the actual field definition for db prep + return self.field.get_db_prep_lookup(lookup_type, value) + + def editable_fields(self): + "Get the fields in this class that should be edited inline." + return [f for f in self.opts.fields + self.opts.many_to_many if f.editable and f != self.field] + + def get_follow(self, override=None): + if isinstance(override, bool): + if override: + over = {} + else: + return None + else: + if override: + over = override.copy() + elif self.edit_inline: + over = {} + else: + return None + + over[self.field.name] = False + return self.opts.get_follow(over) + + def get_manipulator_fields(self, opts, manipulator, change, follow): + if self.field.rel.multiple: + if change: + attr = getattr(manipulator.original_object, self.get_accessor_name()) + count = attr.count() + count += self.field.rel.num_extra_on_change + else: + count = self.field.rel.num_in_admin + if self.field.rel.min_num_in_admin: + count = max(count, self.field.rel.min_num_in_admin) + if self.field.rel.max_num_in_admin: + count = min(count, self.field.rel.max_num_in_admin) + else: + count = 1 + + fields = [] + for i in range(count): + for f in self.opts.fields + self.opts.many_to_many: + if follow.get(f.name, False): + prefix = '%s.%d.' % (self.var_name, i) + fields.extend(f.get_manipulator_fields(self.opts, manipulator, change, + name_prefix=prefix, rel=True)) + return fields + + def __repr__(self): + return "<RelatedObject: %s related to %s>" % (self.name, self.field.name) + + def bind(self, field_mapping, original, bound_related_object_class=BoundRelatedObject): + return bound_related_object_class(self, field_mapping, original) + + def get_accessor_name(self): + # This method encapsulates the logic that decides what name to give an + # accessor descriptor that retrieves related many-to-one or + # many-to-many objects. It uses the lower-cased object_name + "_set", + # but this can be overridden with the "related_name" option. + if self.field.rel.multiple: + # If this is a symmetrical m2m relation on self, there is no reverse accessor. + if getattr(self.field.rel, 'symmetrical', False) and self.model == self.parent_model: + return None + return self.field.rel.related_name or (self.opts.object_name.lower() + '_set') + else: + return self.field.rel.related_name or (self.opts.object_name.lower()) diff --git a/webapp/django/db/models/signals.py b/webapp/django/db/models/signals.py new file mode 100644 index 0000000000..045533612d --- /dev/null +++ b/webapp/django/db/models/signals.py @@ -0,0 +1,14 @@ +from django.dispatch import Signal + +class_prepared = Signal(providing_args=["class"]) + +pre_init = Signal(providing_args=["instance", "args", "kwargs"]) +post_init = Signal(providing_args=["instance"]) + +pre_save = Signal(providing_args=["instance", "raw"]) +post_save = Signal(providing_args=["instance", "raw", "created"]) + +pre_delete = Signal(providing_args=["instance"]) +post_delete = Signal(providing_args=["instance"]) + +post_syncdb = Signal(providing_args=["class", "app", "created_models", "verbosity", "interactive"]) diff --git a/webapp/django/db/models/sql/__init__.py b/webapp/django/db/models/sql/__init__.py new file mode 100644 index 0000000000..7310982690 --- /dev/null +++ b/webapp/django/db/models/sql/__init__.py @@ -0,0 +1,7 @@ +from query import * +from subqueries import * +from where import AND, OR +from datastructures import EmptyResultSet + +__all__ = ['Query', 'AND', 'OR', 'EmptyResultSet'] + diff --git a/webapp/django/db/models/sql/constants.py b/webapp/django/db/models/sql/constants.py new file mode 100644 index 0000000000..129a592b31 --- /dev/null +++ b/webapp/django/db/models/sql/constants.py @@ -0,0 +1,36 @@ +import re + +# Valid query types (a dictionary is used for speedy lookups). +QUERY_TERMS = dict([(x, None) for x in ( + 'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in', + 'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year', + 'month', 'day', 'isnull', 'search', 'regex', 'iregex', + )]) + +# Size of each "chunk" for get_iterator calls. +# Larger values are slightly faster at the expense of more storage space. +GET_ITERATOR_CHUNK_SIZE = 100 + +# Separator used to split filter strings apart. +LOOKUP_SEP = '__' + +# Constants to make looking up tuple values clearer. +# Join lists +TABLE_NAME = 0 +RHS_ALIAS = 1 +JOIN_TYPE = 2 +LHS_ALIAS = 3 +LHS_JOIN_COL = 4 +RHS_JOIN_COL = 5 +NULLABLE = 6 + +# How many results to expect from a cursor.execute call +MULTI = 'multi' +SINGLE = 'single' + +ORDER_PATTERN = re.compile(r'\?|[-+]?[.\w]+$') +ORDER_DIR = { + 'ASC': ('ASC', 'DESC'), + 'DESC': ('DESC', 'ASC')} + + diff --git a/webapp/django/db/models/sql/datastructures.py b/webapp/django/db/models/sql/datastructures.py new file mode 100644 index 0000000000..913d8fde25 --- /dev/null +++ b/webapp/django/db/models/sql/datastructures.py @@ -0,0 +1,103 @@ +""" +Useful auxilliary data structures for query construction. Not useful outside +the SQL domain. +""" + +class EmptyResultSet(Exception): + pass + +class FullResultSet(Exception): + pass + +class MultiJoin(Exception): + """ + Used by join construction code to indicate the point at which a + multi-valued join was attempted (if the caller wants to treat that + exceptionally). + """ + def __init__(self, level): + self.level = level + +class Empty(object): + pass + +class RawValue(object): + def __init__(self, value): + self.value = value + +class Aggregate(object): + """ + Base class for all aggregate-related classes (min, max, avg, count, sum). + """ + def relabel_aliases(self, change_map): + """ + Relabel the column alias, if necessary. Must be implemented by + subclasses. + """ + raise NotImplementedError + + def as_sql(self, quote_func=None): + """ + Returns the SQL string fragment for this object. + + The quote_func function is used to quote the column components. If + None, it defaults to doing nothing. + + Must be implemented by subclasses. + """ + raise NotImplementedError + +class Count(Aggregate): + """ + Perform a count on the given column. + """ + def __init__(self, col='*', distinct=False): + """ + Set the column to count on (defaults to '*') and set whether the count + should be distinct or not. + """ + self.col = col + self.distinct = distinct + + def relabel_aliases(self, change_map): + c = self.col + if isinstance(c, (list, tuple)): + self.col = (change_map.get(c[0], c[0]), c[1]) + + def as_sql(self, quote_func=None): + if not quote_func: + quote_func = lambda x: x + if isinstance(self.col, (list, tuple)): + col = ('%s.%s' % tuple([quote_func(c) for c in self.col])) + elif hasattr(self.col, 'as_sql'): + col = self.col.as_sql(quote_func) + else: + col = self.col + if self.distinct: + return 'COUNT(DISTINCT %s)' % col + else: + return 'COUNT(%s)' % col + +class Date(object): + """ + Add a date selection column. + """ + def __init__(self, col, lookup_type, date_sql_func): + self.col = col + self.lookup_type = lookup_type + self.date_sql_func = date_sql_func + + def relabel_aliases(self, change_map): + c = self.col + if isinstance(c, (list, tuple)): + self.col = (change_map.get(c[0], c[0]), c[1]) + + def as_sql(self, quote_func=None): + if not quote_func: + quote_func = lambda x: x + if isinstance(self.col, (list, tuple)): + col = '%s.%s' % tuple([quote_func(c) for c in self.col]) + else: + col = self.col + return self.date_sql_func(self.lookup_type, col) + diff --git a/webapp/django/db/models/sql/query.py b/webapp/django/db/models/sql/query.py new file mode 100644 index 0000000000..287890a63e --- /dev/null +++ b/webapp/django/db/models/sql/query.py @@ -0,0 +1,1705 @@ +""" +Create SQL statements for QuerySets. + +The code in here encapsulates all of the SQL construction so that QuerySets +themselves do not have to (and could be backed by things other than SQL +databases). The abstraction barrier only works one way: this module has to know +all about the internals of models in order to get the information it needs. +""" + +from copy import deepcopy + +from django.utils.tree import Node +from django.utils.datastructures import SortedDict +from django.utils.encoding import force_unicode +from django.db import connection +from django.db.models import signals +from django.db.models.fields import FieldDoesNotExist +from django.db.models.query_utils import select_related_descend +from django.db.models.sql.where import WhereNode, EverythingNode, AND, OR +from django.db.models.sql.datastructures import Count +from django.core.exceptions import FieldError +from datastructures import EmptyResultSet, Empty, MultiJoin +from constants import * + +try: + set +except NameError: + from sets import Set as set # Python 2.3 fallback + +__all__ = ['Query'] + +class Query(object): + """ + A single SQL query. + """ + # SQL join types. These are part of the class because their string forms + # vary from database to database and can be customised by a subclass. + INNER = 'INNER JOIN' + LOUTER = 'LEFT OUTER JOIN' + + alias_prefix = 'T' + query_terms = QUERY_TERMS + + def __init__(self, model, connection, where=WhereNode): + self.model = model + self.connection = connection + self.alias_refcount = {} + self.alias_map = {} # Maps alias to join information + self.table_map = {} # Maps table names to list of aliases. + self.join_map = {} + self.rev_join_map = {} # Reverse of join_map. + self.quote_cache = {} + self.default_cols = True + self.default_ordering = True + self.standard_ordering = True + self.ordering_aliases = [] + self.start_meta = None + self.select_fields = [] + self.related_select_fields = [] + self.dupe_avoidance = {} + self.used_aliases = set() + self.filter_is_sticky = False + + # SQL-related attributes + self.select = [] + self.tables = [] # Aliases in the order they are created. + self.where = where() + self.where_class = where + self.group_by = [] + self.having = [] + self.order_by = [] + self.low_mark, self.high_mark = 0, None # Used for offset/limit + self.distinct = False + self.select_related = False + self.related_select_cols = [] + + # Arbitrary maximum limit for select_related. Prevents infinite + # recursion. Can be changed by the depth parameter to select_related(). + self.max_depth = 5 + + # These are for extensions. The contents are more or less appended + # verbatim to the appropriate clause. + self.extra_select = SortedDict() # Maps col_alias -> (col_sql, params). + self.extra_tables = () + self.extra_where = () + self.extra_params = () + self.extra_order_by = () + + def __str__(self): + """ + Returns the query as a string of SQL with the parameter values + substituted in. + + Parameter values won't necessarily be quoted correctly, since that is + done by the database interface at execution time. + """ + sql, params = self.as_sql() + return sql % params + + def __deepcopy__(self, memo): + result= self.clone() + memo[id(self)] = result + return result + + def __getstate__(self): + """ + Pickling support. + """ + obj_dict = self.__dict__.copy() + obj_dict['related_select_fields'] = [] + obj_dict['related_select_cols'] = [] + del obj_dict['connection'] + return obj_dict + + def __setstate__(self, obj_dict): + """ + Unpickling support. + """ + self.__dict__.update(obj_dict) + # XXX: Need a better solution for this when multi-db stuff is + # supported. It's the only class-reference to the module-level + # connection variable. + self.connection = connection + + def get_meta(self): + """ + Returns the Options instance (the model._meta) from which to start + processing. Normally, this is self.model._meta, but it can change. + """ + if self.start_meta: + return self.start_meta + return self.model._meta + + def quote_name_unless_alias(self, name): + """ + A wrapper around connection.ops.quote_name that doesn't quote aliases + for table names. This avoids problems with some SQL dialects that treat + quoted strings specially (e.g. PostgreSQL). + """ + if name in self.quote_cache: + return self.quote_cache[name] + if ((name in self.alias_map and name not in self.table_map) or + name in self.extra_select): + self.quote_cache[name] = name + return name + r = self.connection.ops.quote_name(name) + self.quote_cache[name] = r + return r + + def clone(self, klass=None, **kwargs): + """ + Creates a copy of the current instance. The 'kwargs' parameter can be + used by clients to update attributes after copying has taken place. + """ + obj = Empty() + obj.__class__ = klass or self.__class__ + obj.model = self.model + obj.connection = self.connection + obj.alias_refcount = self.alias_refcount.copy() + obj.alias_map = self.alias_map.copy() + obj.table_map = self.table_map.copy() + obj.join_map = self.join_map.copy() + obj.rev_join_map = self.rev_join_map.copy() + obj.quote_cache = {} + obj.default_cols = self.default_cols + obj.default_ordering = self.default_ordering + obj.standard_ordering = self.standard_ordering + obj.ordering_aliases = [] + obj.start_meta = self.start_meta + obj.select_fields = self.select_fields[:] + obj.related_select_fields = self.related_select_fields[:] + obj.dupe_avoidance = self.dupe_avoidance.copy() + obj.select = self.select[:] + obj.tables = self.tables[:] + obj.where = deepcopy(self.where) + obj.where_class = self.where_class + obj.group_by = self.group_by[:] + obj.having = self.having[:] + obj.order_by = self.order_by[:] + obj.low_mark, obj.high_mark = self.low_mark, self.high_mark + obj.distinct = self.distinct + obj.select_related = self.select_related + obj.related_select_cols = [] + obj.max_depth = self.max_depth + obj.extra_select = self.extra_select.copy() + obj.extra_tables = self.extra_tables + obj.extra_where = self.extra_where + obj.extra_params = self.extra_params + obj.extra_order_by = self.extra_order_by + if self.filter_is_sticky and self.used_aliases: + obj.used_aliases = self.used_aliases.copy() + else: + obj.used_aliases = set() + obj.filter_is_sticky = False + obj.__dict__.update(kwargs) + if hasattr(obj, '_setup_query'): + obj._setup_query() + return obj + + def results_iter(self): + """ + Returns an iterator over the results from executing this query. + """ + resolve_columns = hasattr(self, 'resolve_columns') + fields = None + for rows in self.execute_sql(MULTI): + for row in rows: + if resolve_columns: + if fields is None: + # We only set this up here because + # related_select_fields isn't populated until + # execute_sql() has been called. + if self.select_fields: + fields = self.select_fields + self.related_select_fields + else: + fields = self.model._meta.fields + row = self.resolve_columns(row, fields) + yield row + + def get_count(self): + """ + Performs a COUNT() query using the current filter constraints. + """ + from subqueries import CountQuery + obj = self.clone() + obj.clear_ordering(True) + obj.clear_limits() + obj.select_related = False + obj.related_select_cols = [] + obj.related_select_fields = [] + if len(obj.select) > 1: + obj = self.clone(CountQuery, _query=obj, where=self.where_class(), + distinct=False) + obj.select = [] + obj.extra_select = SortedDict() + obj.add_count_column() + data = obj.execute_sql(SINGLE) + if not data: + return 0 + number = data[0] + + # Apply offset and limit constraints manually, since using LIMIT/OFFSET + # in SQL (in variants that provide them) doesn't change the COUNT + # output. + number = max(0, number - self.low_mark) + if self.high_mark: + number = min(number, self.high_mark - self.low_mark) + + return number + + def as_sql(self, with_limits=True, with_col_aliases=False): + """ + Creates the SQL for this query. Returns the SQL string and list of + parameters. + + If 'with_limits' is False, any limit/offset information is not included + in the query. + """ + self.pre_sql_setup() + out_cols = self.get_columns(with_col_aliases) + ordering = self.get_ordering() + + # This must come after 'select' and 'ordering' -- see docstring of + # get_from_clause() for details. + from_, f_params = self.get_from_clause() + + where, w_params = self.where.as_sql(qn=self.quote_name_unless_alias) + params = [] + for val in self.extra_select.itervalues(): + params.extend(val[1]) + + result = ['SELECT'] + if self.distinct: + result.append('DISTINCT') + result.append(', '.join(out_cols + self.ordering_aliases)) + + result.append('FROM') + result.extend(from_) + params.extend(f_params) + + if where: + result.append('WHERE %s' % where) + params.extend(w_params) + if self.extra_where: + if not where: + result.append('WHERE') + else: + result.append('AND') + result.append(' AND '.join(self.extra_where)) + + if self.group_by: + grouping = self.get_grouping() + result.append('GROUP BY %s' % ', '.join(grouping)) + + if ordering: + result.append('ORDER BY %s' % ', '.join(ordering)) + + if with_limits: + if self.high_mark is not None: + result.append('LIMIT %d' % (self.high_mark - self.low_mark)) + if self.low_mark: + if self.high_mark is None: + val = self.connection.ops.no_limit_value() + if val: + result.append('LIMIT %d' % val) + result.append('OFFSET %d' % self.low_mark) + + params.extend(self.extra_params) + return ' '.join(result), tuple(params) + + def combine(self, rhs, connector): + """ + Merge the 'rhs' query into the current one (with any 'rhs' effects + being applied *after* (that is, "to the right of") anything in the + current query. 'rhs' is not modified during a call to this function. + + The 'connector' parameter describes how to connect filters from the + 'rhs' query. + """ + assert self.model == rhs.model, \ + "Cannot combine queries on two different base models." + assert self.can_filter(), \ + "Cannot combine queries once a slice has been taken." + assert self.distinct == rhs.distinct, \ + "Cannot combine a unique query with a non-unique query." + + # Work out how to relabel the rhs aliases, if necessary. + change_map = {} + used = set() + conjunction = (connector == AND) + first = True + for alias in rhs.tables: + if not rhs.alias_refcount[alias]: + # An unused alias. + continue + promote = (rhs.alias_map[alias][JOIN_TYPE] == self.LOUTER) + new_alias = self.join(rhs.rev_join_map[alias], + (conjunction and not first), used, promote, not conjunction) + used.add(new_alias) + change_map[alias] = new_alias + first = False + + # So that we don't exclude valid results in an "or" query combination, + # the first join that is exclusive to the lhs (self) must be converted + # to an outer join. + if not conjunction: + for alias in self.tables[1:]: + if self.alias_refcount[alias] == 1: + self.promote_alias(alias, True) + break + + # Now relabel a copy of the rhs where-clause and add it to the current + # one. + if rhs.where: + w = deepcopy(rhs.where) + w.relabel_aliases(change_map) + if not self.where: + # Since 'self' matches everything, add an explicit "include + # everything" where-constraint so that connections between the + # where clauses won't exclude valid results. + self.where.add(EverythingNode(), AND) + elif self.where: + # rhs has an empty where clause. + w = self.where_class() + w.add(EverythingNode(), AND) + else: + w = self.where_class() + self.where.add(w, connector) + + # Selection columns and extra extensions are those provided by 'rhs'. + self.select = [] + for col in rhs.select: + if isinstance(col, (list, tuple)): + self.select.append((change_map.get(col[0], col[0]), col[1])) + else: + item = deepcopy(col) + item.relabel_aliases(change_map) + self.select.append(item) + self.select_fields = rhs.select_fields[:] + + if connector == OR: + # It would be nice to be able to handle this, but the queries don't + # really make sense (or return consistent value sets). Not worth + # the extra complexity when you can write a real query instead. + if self.extra_select and rhs.extra_select: + raise ValueError("When merging querysets using 'or', you " + "cannot have extra(select=...) on both sides.") + if self.extra_where and rhs.extra_where: + raise ValueError("When merging querysets using 'or', you " + "cannot have extra(where=...) on both sides.") + self.extra_select.update(rhs.extra_select) + self.extra_tables += rhs.extra_tables + self.extra_where += rhs.extra_where + self.extra_params += rhs.extra_params + + # Ordering uses the 'rhs' ordering, unless it has none, in which case + # the current ordering is used. + self.order_by = rhs.order_by and rhs.order_by[:] or self.order_by + self.extra_order_by = rhs.extra_order_by or self.extra_order_by + + def pre_sql_setup(self): + """ + Does any necessary class setup immediately prior to producing SQL. This + is for things that can't necessarily be done in __init__ because we + might not have all the pieces in place at that time. + """ + if not self.tables: + self.join((None, self.model._meta.db_table, None, None)) + if self.select_related and not self.related_select_cols: + self.fill_related_selections() + + def get_columns(self, with_aliases=False): + """ + Return the list of columns to use in the select statement. If no + columns have been specified, returns all columns relating to fields in + the model. + + If 'with_aliases' is true, any column names that are duplicated + (without the table names) are given unique aliases. This is needed in + some cases to avoid ambiguitity with nested queries. + """ + qn = self.quote_name_unless_alias + qn2 = self.connection.ops.quote_name + result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in self.extra_select.iteritems()] + aliases = set(self.extra_select.keys()) + if with_aliases: + col_aliases = aliases.copy() + else: + col_aliases = set() + if self.select: + for col in self.select: + if isinstance(col, (list, tuple)): + r = '%s.%s' % (qn(col[0]), qn(col[1])) + if with_aliases and col[1] in col_aliases: + c_alias = 'Col%d' % len(col_aliases) + result.append('%s AS %s' % (r, c_alias)) + aliases.add(c_alias) + col_aliases.add(c_alias) + else: + result.append(r) + aliases.add(r) + col_aliases.add(col[1]) + else: + result.append(col.as_sql(quote_func=qn)) + if hasattr(col, 'alias'): + aliases.add(col.alias) + col_aliases.add(col.alias) + elif self.default_cols: + cols, new_aliases = self.get_default_columns(with_aliases, + col_aliases) + result.extend(cols) + aliases.update(new_aliases) + for table, col in self.related_select_cols: + r = '%s.%s' % (qn(table), qn(col)) + if with_aliases and col in col_aliases: + c_alias = 'Col%d' % len(col_aliases) + result.append('%s AS %s' % (r, c_alias)) + aliases.add(c_alias) + col_aliases.add(c_alias) + else: + result.append(r) + aliases.add(r) + col_aliases.add(col) + + self._select_aliases = aliases + return result + + def get_default_columns(self, with_aliases=False, col_aliases=None, + start_alias=None, opts=None, as_pairs=False): + """ + Computes the default columns for selecting every field in the base + model. + + Returns a list of strings, quoted appropriately for use in SQL + directly, as well as a set of aliases used in the select statement (if + 'as_pairs' is True, returns a list of (alias, col_name) pairs instead + of strings as the first component and None as the second component). + """ + result = [] + if opts is None: + opts = self.model._meta + if start_alias: + table_alias = start_alias + else: + table_alias = self.tables[0] + root_pk = opts.pk.column + seen = {None: table_alias} + qn = self.quote_name_unless_alias + qn2 = self.connection.ops.quote_name + aliases = set() + for field, model in opts.get_fields_with_model(): + try: + alias = seen[model] + except KeyError: + alias = self.join((table_alias, model._meta.db_table, + root_pk, model._meta.pk.column)) + seen[model] = alias + if as_pairs: + result.append((alias, field.column)) + continue + if with_aliases and field.column in col_aliases: + c_alias = 'Col%d' % len(col_aliases) + result.append('%s.%s AS %s' % (qn(alias), + qn2(field.column), c_alias)) + col_aliases.add(c_alias) + aliases.add(c_alias) + else: + r = '%s.%s' % (qn(alias), qn2(field.column)) + result.append(r) + aliases.add(r) + if with_aliases: + col_aliases.add(field.column) + if as_pairs: + return result, None + return result, aliases + + def get_from_clause(self): + """ + Returns a list of strings that are joined together to go after the + "FROM" part of the query, as well as a list any extra parameters that + need to be included. Sub-classes, can override this to create a + from-clause via a "select", for example (e.g. CountQuery). + + This should only be called after any SQL construction methods that + might change the tables we need. This means the select columns and + ordering must be done first. + """ + result = [] + qn = self.quote_name_unless_alias + qn2 = self.connection.ops.quote_name + first = True + for alias in self.tables: + if not self.alias_refcount[alias]: + continue + try: + name, alias, join_type, lhs, lhs_col, col, nullable = self.alias_map[alias] + except KeyError: + # Extra tables can end up in self.tables, but not in the + # alias_map if they aren't in a join. That's OK. We skip them. + continue + alias_str = (alias != name and ' %s' % alias or '') + if join_type and not first: + result.append('%s %s%s ON (%s.%s = %s.%s)' + % (join_type, qn(name), alias_str, qn(lhs), + qn2(lhs_col), qn(alias), qn2(col))) + else: + connector = not first and ', ' or '' + result.append('%s%s%s' % (connector, qn(name), alias_str)) + first = False + for t in self.extra_tables: + alias, unused = self.table_alias(t) + # Only add the alias if it's not already present (the table_alias() + # calls increments the refcount, so an alias refcount of one means + # this is the only reference. + if alias not in self.alias_map or self.alias_refcount[alias] == 1: + connector = not first and ', ' or '' + result.append('%s%s' % (connector, qn(alias))) + first = False + return result, [] + + def get_grouping(self): + """ + Returns a tuple representing the SQL elements in the "group by" clause. + """ + qn = self.quote_name_unless_alias + result = [] + for col in self.group_by: + if isinstance(col, (list, tuple)): + result.append('%s.%s' % (qn(col[0]), qn(col[1]))) + elif hasattr(col, 'as_sql'): + result.append(col.as_sql(qn)) + else: + result.append(str(col)) + return result + + def get_ordering(self): + """ + Returns list representing the SQL elements in the "order by" clause. + Also sets the ordering_aliases attribute on this instance to a list of + extra aliases needed in the select. + + Determining the ordering SQL can change the tables we need to include, + so this should be run *before* get_from_clause(). + """ + if self.extra_order_by: + ordering = self.extra_order_by + elif not self.default_ordering: + ordering = [] + else: + ordering = self.order_by or self.model._meta.ordering + qn = self.quote_name_unless_alias + qn2 = self.connection.ops.quote_name + distinct = self.distinct + select_aliases = self._select_aliases + result = [] + ordering_aliases = [] + if self.standard_ordering: + asc, desc = ORDER_DIR['ASC'] + else: + asc, desc = ORDER_DIR['DESC'] + for field in ordering: + if field == '?': + result.append(self.connection.ops.random_function_sql()) + continue + if isinstance(field, int): + if field < 0: + order = desc + field = -field + else: + order = asc + result.append('%s %s' % (field, order)) + continue + if '.' in field: + # This came in through an extra(order_by=...) addition. Pass it + # on verbatim. + col, order = get_order_dir(field, asc) + table, col = col.split('.', 1) + elt = '%s.%s' % (qn(table), col) + if not distinct or elt in select_aliases: + result.append('%s %s' % (elt, order)) + elif get_order_dir(field)[0] not in self.extra_select: + # 'col' is of the form 'field' or 'field1__field2' or + # '-field1__field2__field', etc. + for table, col, order in self.find_ordering_name(field, + self.model._meta, default_order=asc): + elt = '%s.%s' % (qn(table), qn2(col)) + if distinct and elt not in select_aliases: + ordering_aliases.append(elt) + result.append('%s %s' % (elt, order)) + else: + col, order = get_order_dir(field, asc) + elt = qn(col) + if distinct and elt not in select_aliases: + ordering_aliases.append(elt) + result.append('%s %s' % (elt, order)) + self.ordering_aliases = ordering_aliases + return result + + def find_ordering_name(self, name, opts, alias=None, default_order='ASC', + already_seen=None): + """ + Returns the table alias (the name might be ambiguous, the alias will + not be) and column name for ordering by the given 'name' parameter. + The 'name' is of the form 'field1__field2__...__fieldN'. + """ + name, order = get_order_dir(name, default_order) + pieces = name.split(LOOKUP_SEP) + if not alias: + alias = self.get_initial_alias() + field, target, opts, joins, last = self.setup_joins(pieces, opts, + alias, False) + alias = joins[-1] + col = target.column + if not field.rel: + # To avoid inadvertent trimming of a necessary alias, use the + # refcount to show that we are referencing a non-relation field on + # the model. + self.ref_alias(alias) + + # Must use left outer joins for nullable fields. + for join in joins: + self.promote_alias(join) + + # If we get to this point and the field is a relation to another model, + # append the default ordering for that model. + if field.rel and len(joins) > 1 and opts.ordering: + # Firstly, avoid infinite loops. + if not already_seen: + already_seen = set() + join_tuple = tuple([self.alias_map[j][TABLE_NAME] for j in joins]) + if join_tuple in already_seen: + raise FieldError('Infinite loop caused by ordering.') + already_seen.add(join_tuple) + + results = [] + for item in opts.ordering: + results.extend(self.find_ordering_name(item, opts, alias, + order, already_seen)) + return results + + if alias: + # We have to do the same "final join" optimisation as in + # add_filter, since the final column might not otherwise be part of + # the select set (so we can't order on it). + while 1: + join = self.alias_map[alias] + if col != join[RHS_JOIN_COL]: + break + self.unref_alias(alias) + alias = join[LHS_ALIAS] + col = join[LHS_JOIN_COL] + return [(alias, col, order)] + + def table_alias(self, table_name, create=False): + """ + Returns a table alias for the given table_name and whether this is a + new alias or not. + + If 'create' is true, a new alias is always created. Otherwise, the + most recently created alias for the table (if one exists) is reused. + """ + current = self.table_map.get(table_name) + if not create and current: + alias = current[0] + self.alias_refcount[alias] += 1 + return alias, False + + # Create a new alias for this table. + if current: + alias = '%s%d' % (self.alias_prefix, len(self.alias_map) + 1) + current.append(alias) + else: + # The first occurence of a table uses the table name directly. + alias = table_name + self.table_map[alias] = [alias] + self.alias_refcount[alias] = 1 + #self.alias_map[alias] = None + self.tables.append(alias) + return alias, True + + def ref_alias(self, alias): + """ Increases the reference count for this alias. """ + self.alias_refcount[alias] += 1 + + def unref_alias(self, alias): + """ Decreases the reference count for this alias. """ + self.alias_refcount[alias] -= 1 + + def promote_alias(self, alias, unconditional=False): + """ + Promotes the join type of an alias to an outer join if it's possible + for the join to contain NULL values on the left. If 'unconditional' is + False, the join is only promoted if it is nullable, otherwise it is + always promoted. + + Returns True if the join was promoted. + """ + if ((unconditional or self.alias_map[alias][NULLABLE]) and + self.alias_map[alias] != self.LOUTER): + data = list(self.alias_map[alias]) + data[JOIN_TYPE] = self.LOUTER + self.alias_map[alias] = tuple(data) + return True + return False + + def change_aliases(self, change_map): + """ + Changes the aliases in change_map (which maps old-alias -> new-alias), + relabelling any references to them in select columns and the where + clause. + """ + assert set(change_map.keys()).intersection(set(change_map.values())) == set() + + # 1. Update references in "select" and "where". + self.where.relabel_aliases(change_map) + for pos, col in enumerate(self.select): + if isinstance(col, (list, tuple)): + self.select[pos] = (change_map.get(old_alias, old_alias), col[1]) + else: + col.relabel_aliases(change_map) + + # 2. Rename the alias in the internal table/alias datastructures. + for old_alias, new_alias in change_map.iteritems(): + alias_data = list(self.alias_map[old_alias]) + alias_data[RHS_ALIAS] = new_alias + + t = self.rev_join_map[old_alias] + data = list(self.join_map[t]) + data[data.index(old_alias)] = new_alias + self.join_map[t] = tuple(data) + self.rev_join_map[new_alias] = t + del self.rev_join_map[old_alias] + self.alias_refcount[new_alias] = self.alias_refcount[old_alias] + del self.alias_refcount[old_alias] + self.alias_map[new_alias] = tuple(alias_data) + del self.alias_map[old_alias] + + table_aliases = self.table_map[alias_data[TABLE_NAME]] + for pos, alias in enumerate(table_aliases): + if alias == old_alias: + table_aliases[pos] = new_alias + break + for pos, alias in enumerate(self.tables): + if alias == old_alias: + self.tables[pos] = new_alias + break + + # 3. Update any joins that refer to the old alias. + for alias, data in self.alias_map.iteritems(): + lhs = data[LHS_ALIAS] + if lhs in change_map: + data = list(data) + data[LHS_ALIAS] = change_map[lhs] + self.alias_map[alias] = tuple(data) + + def bump_prefix(self, exceptions=()): + """ + Changes the alias prefix to the next letter in the alphabet and + relabels all the aliases. Even tables that previously had no alias will + get an alias after this call (it's mostly used for nested queries and + the outer query will already be using the non-aliased table name). + + Subclasses who create their own prefix should override this method to + produce a similar result (a new prefix and relabelled aliases). + + The 'exceptions' parameter is a container that holds alias names which + should not be changed. + """ + assert ord(self.alias_prefix) < ord('Z') + self.alias_prefix = chr(ord(self.alias_prefix) + 1) + change_map = {} + prefix = self.alias_prefix + for pos, alias in enumerate(self.tables): + if alias in exceptions: + continue + new_alias = '%s%d' % (prefix, pos) + change_map[alias] = new_alias + self.tables[pos] = new_alias + self.change_aliases(change_map) + + def get_initial_alias(self): + """ + Returns the first alias for this query, after increasing its reference + count. + """ + if self.tables: + alias = self.tables[0] + self.ref_alias(alias) + else: + alias = self.join((None, self.model._meta.db_table, None, None)) + return alias + + def count_active_tables(self): + """ + Returns the number of tables in this query with a non-zero reference + count. + """ + return len([1 for count in self.alias_refcount.itervalues() if count]) + + def join(self, connection, always_create=False, exclusions=(), + promote=False, outer_if_first=False, nullable=False, reuse=None): + """ + Returns an alias for the join in 'connection', either reusing an + existing alias for that join or creating a new one. 'connection' is a + tuple (lhs, table, lhs_col, col) where 'lhs' is either an existing + table alias or a table name. The join correspods to the SQL equivalent + of:: + + lhs.lhs_col = table.col + + If 'always_create' is True and 'reuse' is None, a new alias is always + created, regardless of whether one already exists or not. Otherwise + 'reuse' must be a set and a new join is created unless one of the + aliases in `reuse` can be used. + + If 'exclusions' is specified, it is something satisfying the container + protocol ("foo in exclusions" must work) and specifies a list of + aliases that should not be returned, even if they satisfy the join. + + If 'promote' is True, the join type for the alias will be LOUTER (if + the alias previously existed, the join type will be promoted from INNER + to LOUTER, if necessary). + + If 'outer_if_first' is True and a new join is created, it will have the + LOUTER join type. This is used when joining certain types of querysets + and Q-objects together. + + If 'nullable' is True, the join can potentially involve NULL values and + is a candidate for promotion (to "left outer") when combining querysets. + """ + lhs, table, lhs_col, col = connection + if lhs in self.alias_map: + lhs_table = self.alias_map[lhs][TABLE_NAME] + else: + lhs_table = lhs + + if reuse and always_create and table in self.table_map: + # Convert the 'reuse' to case to be "exclude everything but the + # reusable set, minus exclusions, for this table". + exclusions = set(self.table_map[table]).difference(reuse).union(set(exclusions)) + always_create = False + t_ident = (lhs_table, table, lhs_col, col) + if not always_create: + for alias in self.join_map.get(t_ident, ()): + if alias not in exclusions: + if lhs_table and not self.alias_refcount[self.alias_map[alias][LHS_ALIAS]]: + # The LHS of this join tuple is no longer part of the + # query, so skip this possibility. + continue + self.ref_alias(alias) + if promote: + self.promote_alias(alias) + return alias + + # No reuse is possible, so we need a new alias. + alias, _ = self.table_alias(table, True) + if not lhs: + # Not all tables need to be joined to anything. No join type + # means the later columns are ignored. + join_type = None + elif promote or outer_if_first: + join_type = self.LOUTER + else: + join_type = self.INNER + join = (table, alias, join_type, lhs, lhs_col, col, nullable) + self.alias_map[alias] = join + if t_ident in self.join_map: + self.join_map[t_ident] += (alias,) + else: + self.join_map[t_ident] = (alias,) + self.rev_join_map[alias] = t_ident + return alias + + def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, + used=None, requested=None, restricted=None, nullable=None, + dupe_set=None): + """ + Fill in the information needed for a select_related query. The current + depth is measured as the number of connections away from the root model + (for example, cur_depth=1 means we are looking at models with direct + connections to the root model). + """ + if not restricted and self.max_depth and cur_depth > self.max_depth: + # We've recursed far enough; bail out. + return + + if not opts: + opts = self.get_meta() + root_alias = self.get_initial_alias() + self.related_select_cols = [] + self.related_select_fields = [] + if not used: + used = set() + if dupe_set is None: + dupe_set = set() + orig_dupe_set = dupe_set + orig_used = used + + # Setup for the case when only particular related fields should be + # included in the related selection. + if requested is None and restricted is not False: + if isinstance(self.select_related, dict): + requested = self.select_related + restricted = True + else: + restricted = False + + for f, model in opts.get_fields_with_model(): + if not select_related_descend(f, restricted, requested): + continue + dupe_set = orig_dupe_set.copy() + used = orig_used.copy() + table = f.rel.to._meta.db_table + if nullable or f.null: + promote = True + else: + promote = False + if model: + int_opts = opts + alias = root_alias + for int_model in opts.get_base_chain(model): + lhs_col = int_opts.parents[int_model].column + dedupe = lhs_col in opts.duplicate_targets + if dedupe: + used.update(self.dupe_avoidance.get(id(opts), lhs_col), + ()) + dupe_set.add((opts, lhs_col)) + int_opts = int_model._meta + alias = self.join((alias, int_opts.db_table, lhs_col, + int_opts.pk.column), exclusions=used, + promote=promote) + for (dupe_opts, dupe_col) in dupe_set: + self.update_dupe_avoidance(dupe_opts, dupe_col, alias) + else: + alias = root_alias + + dedupe = f.column in opts.duplicate_targets + if dupe_set or dedupe: + used.update(self.dupe_avoidance.get((id(opts), f.column), ())) + if dedupe: + dupe_set.add((opts, f.column)) + + alias = self.join((alias, table, f.column, + f.rel.get_related_field().column), exclusions=used, + promote=promote) + used.add(alias) + self.related_select_cols.extend(self.get_default_columns( + start_alias=alias, opts=f.rel.to._meta, as_pairs=True)[0]) + self.related_select_fields.extend(f.rel.to._meta.fields) + if restricted: + next = requested.get(f.name, {}) + else: + next = False + if f.null is not None: + new_nullable = f.null + else: + new_nullable = None + for dupe_opts, dupe_col in dupe_set: + self.update_dupe_avoidance(dupe_opts, dupe_col, alias) + self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, + used, next, restricted, new_nullable, dupe_set) + + def add_filter(self, filter_expr, connector=AND, negate=False, trim=False, + can_reuse=None): + """ + Add a single filter to the query. The 'filter_expr' is a pair: + (filter_string, value). E.g. ('name__contains', 'fred') + + If 'negate' is True, this is an exclude() filter. It's important to + note that this method does not negate anything in the where-clause + object when inserting the filter constraints. This is because negated + filters often require multiple calls to add_filter() and the negation + should only happen once. So the caller is responsible for this (the + caller will normally be add_q(), so that as an example). + + If 'trim' is True, we automatically trim the final join group (used + internally when constructing nested queries). + + If 'can_reuse' is a set, we are processing a component of a + multi-component filter (e.g. filter(Q1, Q2)). In this case, 'can_reuse' + will be a set of table aliases that can be reused in this filter, even + if we would otherwise force the creation of new aliases for a join + (needed for nested Q-filters). The set is updated by this method. + """ + arg, value = filter_expr + parts = arg.split(LOOKUP_SEP) + if not parts: + raise FieldError("Cannot parse keyword query %r" % arg) + + # Work out the lookup type and remove it from 'parts', if necessary. + if len(parts) == 1 or parts[-1] not in self.query_terms: + lookup_type = 'exact' + else: + lookup_type = parts.pop() + + # Interpret '__exact=None' as the sql 'is NULL'; otherwise, reject all + # uses of None as a query value. + if value is None: + if lookup_type != 'exact': + raise ValueError("Cannot use None as a query value") + lookup_type = 'isnull' + value = True + elif callable(value): + value = value() + + opts = self.get_meta() + alias = self.get_initial_alias() + allow_many = trim or not negate + + try: + field, target, opts, join_list, last = self.setup_joins(parts, opts, + alias, True, allow_many, can_reuse=can_reuse) + except MultiJoin, e: + self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level])) + return + final = len(join_list) + penultimate = last.pop() + if penultimate == final: + penultimate = last.pop() + if trim and len(join_list) > 1: + extra = join_list[penultimate:] + join_list = join_list[:penultimate] + final = penultimate + penultimate = last.pop() + col = self.alias_map[extra[0]][LHS_JOIN_COL] + for alias in extra: + self.unref_alias(alias) + else: + col = target.column + alias = join_list[-1] + + while final > 1: + # An optimization: if the final join is against the same column as + # we are comparing against, we can go back one step in the join + # chain and compare against the lhs of the join instead (and then + # repeat the optimization). The result, potentially, involves less + # table joins. + join = self.alias_map[alias] + if col != join[RHS_JOIN_COL]: + break + self.unref_alias(alias) + alias = join[LHS_ALIAS] + col = join[LHS_JOIN_COL] + join_list = join_list[:-1] + final -= 1 + if final == penultimate: + penultimate = last.pop() + + if (lookup_type == 'isnull' and value is True and not negate and + final > 1): + # If the comparison is against NULL, we need to use a left outer + # join when connecting to the previous model. We make that + # adjustment here. We don't do this unless needed as it's less + # efficient at the database level. + self.promote_alias(join_list[penultimate]) + + if connector == OR: + # Some joins may need to be promoted when adding a new filter to a + # disjunction. We walk the list of new joins and where it diverges + # from any previous joins (ref count is 1 in the table list), we + # make the new additions (and any existing ones not used in the new + # join list) an outer join. + join_it = iter(join_list) + table_it = iter(self.tables) + join_it.next(), table_it.next() + table_promote = False + for join in join_it: + table = table_it.next() + if join == table and self.alias_refcount[join] > 1: + continue + join_promote = self.promote_alias(join) + if table != join: + table_promote = self.promote_alias(table) + break + for join in join_it: + if self.promote_alias(join, join_promote): + join_promote = True + for table in table_it: + # Some of these will have been promoted from the join_list, but + # that's harmless. + if self.promote_alias(table, table_promote): + table_promote = True + + self.where.add((alias, col, field, lookup_type, value), connector) + + if negate: + for alias in join_list: + self.promote_alias(alias) + if lookup_type != 'isnull': + if final > 1: + for alias in join_list: + if self.alias_map[alias][JOIN_TYPE] == self.LOUTER: + j_col = self.alias_map[alias][RHS_JOIN_COL] + entry = self.where_class() + entry.add((alias, j_col, None, 'isnull', True), AND) + entry.negate() + self.where.add(entry, AND) + break + elif not (lookup_type == 'in' and not value) and field.null: + # Leaky abstraction artifact: We have to specifically + # exclude the "foo__in=[]" case from this handling, because + # it's short-circuited in the Where class. + entry = self.where_class() + entry.add((alias, col, None, 'isnull', True), AND) + entry.negate() + self.where.add(entry, AND) + + if can_reuse is not None: + can_reuse.update(join_list) + + def add_q(self, q_object, used_aliases=None): + """ + Adds a Q-object to the current filter. + + Can also be used to add anything that has an 'add_to_query()' method. + """ + if used_aliases is None: + used_aliases = self.used_aliases + if hasattr(q_object, 'add_to_query'): + # Complex custom objects are responsible for adding themselves. + q_object.add_to_query(self, used_aliases) + else: + if self.where and q_object.connector != AND and len(q_object) > 1: + self.where.start_subtree(AND) + subtree = True + else: + subtree = False + connector = AND + for child in q_object.children: + if isinstance(child, Node): + self.where.start_subtree(connector) + self.add_q(child, used_aliases) + self.where.end_subtree() + else: + self.add_filter(child, connector, q_object.negated, + can_reuse=used_aliases) + connector = q_object.connector + if q_object.negated: + self.where.negate() + if subtree: + self.where.end_subtree() + if self.filter_is_sticky: + self.used_aliases = used_aliases + + def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True, + allow_explicit_fk=False, can_reuse=None): + """ + Compute the necessary table joins for the passage through the fields + given in 'names'. 'opts' is the Options class for the current model + (which gives the table we are joining to), 'alias' is the alias for the + table we are joining to. If dupe_multis is True, any many-to-many or + many-to-one joins will always create a new alias (necessary for + disjunctive filters). If can_reuse is not None, it's a list of aliases + that can be reused in these joins (nothing else can be reused in this + case). + + Returns the final field involved in the join, the target database + column (used for any 'where' constraint), the final 'opts' value and the + list of tables joined. + """ + joins = [alias] + last = [0] + dupe_set = set() + exclusions = set() + for pos, name in enumerate(names): + try: + exclusions.add(int_alias) + except NameError: + pass + exclusions.add(alias) + last.append(len(joins)) + if name == 'pk': + name = opts.pk.name + + try: + field, model, direct, m2m = opts.get_field_by_name(name) + except FieldDoesNotExist: + for f in opts.fields: + if allow_explicit_fk and name == f.attname: + # XXX: A hack to allow foo_id to work in values() for + # backwards compatibility purposes. If we dropped that + # feature, this could be removed. + field, model, direct, m2m = opts.get_field_by_name(f.name) + break + else: + names = opts.get_all_field_names() + raise FieldError("Cannot resolve keyword %r into field. " + "Choices are: %s" % (name, ", ".join(names))) + + if not allow_many and (m2m or not direct): + for alias in joins: + self.unref_alias(alias) + raise MultiJoin(pos + 1) + if model: + # The field lives on a base class of the current model. + for int_model in opts.get_base_chain(model): + lhs_col = opts.parents[int_model].column + dedupe = lhs_col in opts.duplicate_targets + if dedupe: + exclusions.update(self.dupe_avoidance.get( + (id(opts), lhs_col), ())) + dupe_set.add((opts, lhs_col)) + opts = int_model._meta + alias = self.join((alias, opts.db_table, lhs_col, + opts.pk.column), exclusions=exclusions) + joins.append(alias) + exclusions.add(alias) + for (dupe_opts, dupe_col) in dupe_set: + self.update_dupe_avoidance(dupe_opts, dupe_col, alias) + cached_data = opts._join_cache.get(name) + orig_opts = opts + dupe_col = direct and field.column or field.field.column + dedupe = dupe_col in opts.duplicate_targets + if dupe_set or dedupe: + if dedupe: + dupe_set.add((opts, dupe_col)) + exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col), + ())) + + if direct: + if m2m: + # Many-to-many field defined on the current model. + if cached_data: + (table1, from_col1, to_col1, table2, from_col2, + to_col2, opts, target) = cached_data + else: + table1 = field.m2m_db_table() + from_col1 = opts.pk.column + to_col1 = field.m2m_column_name() + opts = field.rel.to._meta + table2 = opts.db_table + from_col2 = field.m2m_reverse_name() + to_col2 = opts.pk.column + target = opts.pk + orig_opts._join_cache[name] = (table1, from_col1, + to_col1, table2, from_col2, to_col2, opts, + target) + + int_alias = self.join((alias, table1, from_col1, to_col1), + dupe_multis, exclusions, nullable=True, + reuse=can_reuse) + alias = self.join((int_alias, table2, from_col2, to_col2), + dupe_multis, exclusions, nullable=True, + reuse=can_reuse) + joins.extend([int_alias, alias]) + elif field.rel: + # One-to-one or many-to-one field + if cached_data: + (table, from_col, to_col, opts, target) = cached_data + else: + opts = field.rel.to._meta + target = field.rel.get_related_field() + table = opts.db_table + from_col = field.column + to_col = target.column + orig_opts._join_cache[name] = (table, from_col, to_col, + opts, target) + + alias = self.join((alias, table, from_col, to_col), + exclusions=exclusions, nullable=field.null) + joins.append(alias) + else: + # Non-relation fields. + target = field + break + else: + orig_field = field + field = field.field + if m2m: + # Many-to-many field defined on the target model. + if cached_data: + (table1, from_col1, to_col1, table2, from_col2, + to_col2, opts, target) = cached_data + else: + table1 = field.m2m_db_table() + from_col1 = opts.pk.column + to_col1 = field.m2m_reverse_name() + opts = orig_field.opts + table2 = opts.db_table + from_col2 = field.m2m_column_name() + to_col2 = opts.pk.column + target = opts.pk + orig_opts._join_cache[name] = (table1, from_col1, + to_col1, table2, from_col2, to_col2, opts, + target) + + int_alias = self.join((alias, table1, from_col1, to_col1), + dupe_multis, exclusions, nullable=True, + reuse=can_reuse) + alias = self.join((int_alias, table2, from_col2, to_col2), + dupe_multis, exclusions, nullable=True, + reuse=can_reuse) + joins.extend([int_alias, alias]) + else: + # One-to-many field (ForeignKey defined on the target model) + if cached_data: + (table, from_col, to_col, opts, target) = cached_data + else: + local_field = opts.get_field_by_name( + field.rel.field_name)[0] + opts = orig_field.opts + table = opts.db_table + from_col = local_field.column + to_col = field.column + target = opts.pk + orig_opts._join_cache[name] = (table, from_col, to_col, + opts, target) + + alias = self.join((alias, table, from_col, to_col), + dupe_multis, exclusions, nullable=True, + reuse=can_reuse) + joins.append(alias) + + for (dupe_opts, dupe_col) in dupe_set: + try: + self.update_dupe_avoidance(dupe_opts, dupe_col, int_alias) + except NameError: + self.update_dupe_avoidance(dupe_opts, dupe_col, alias) + + if pos != len(names) - 1: + raise FieldError("Join on field %r not permitted." % name) + + return field, target, opts, joins, last + + def update_dupe_avoidance(self, opts, col, alias): + """ + For a column that is one of multiple pointing to the same table, update + the internal data structures to note that this alias shouldn't be used + for those other columns. + """ + ident = id(opts) + for name in opts.duplicate_targets[col]: + try: + self.dupe_avoidance[ident, name].add(alias) + except KeyError: + self.dupe_avoidance[ident, name] = set([alias]) + + def split_exclude(self, filter_expr, prefix): + """ + When doing an exclude against any kind of N-to-many relation, we need + to use a subquery. This method constructs the nested query, given the + original exclude filter (filter_expr) and the portion up to the first + N-to-many relation field. + """ + query = Query(self.model, self.connection) + query.add_filter(filter_expr) + query.set_start(prefix) + query.clear_ordering(True) + self.add_filter(('%s__in' % prefix, query), negate=True, trim=True) + + def set_limits(self, low=None, high=None): + """ + Adjusts the limits on the rows retrieved. We use low/high to set these, + as it makes it more Pythonic to read and write. When the SQL query is + created, they are converted to the appropriate offset and limit values. + + Any limits passed in here are applied relative to the existing + constraints. So low is added to the current low value and both will be + clamped to any existing high value. + """ + if high is not None: + if self.high_mark: + self.high_mark = min(self.high_mark, self.low_mark + high) + else: + self.high_mark = self.low_mark + high + if low is not None: + if self.high_mark: + self.low_mark = min(self.high_mark, self.low_mark + low) + else: + self.low_mark = self.low_mark + low + + def clear_limits(self): + """ + Clears any existing limits. + """ + self.low_mark, self.high_mark = 0, None + + def can_filter(self): + """ + Returns True if adding filters to this instance is still possible. + + Typically, this means no limits or offsets have been put on the results. + """ + return not (self.low_mark or self.high_mark) + + def add_fields(self, field_names, allow_m2m=True): + """ + Adds the given (model) fields to the select set. The field names are + added in the order specified. + """ + alias = self.get_initial_alias() + opts = self.get_meta() + try: + for name in field_names: + field, target, u2, joins, u3 = self.setup_joins( + name.split(LOOKUP_SEP), opts, alias, False, allow_m2m, + True) + final_alias = joins[-1] + col = target.column + if len(joins) > 1: + join = self.alias_map[final_alias] + if col == join[RHS_JOIN_COL]: + self.unref_alias(final_alias) + final_alias = join[LHS_ALIAS] + col = join[LHS_JOIN_COL] + joins = joins[:-1] + promote = False + for join in joins[1:]: + # Only nullable aliases are promoted, so we don't end up + # doing unnecessary left outer joins here. + if self.promote_alias(join, promote): + promote = True + self.select.append((final_alias, col)) + self.select_fields.append(field) + except MultiJoin: + raise FieldError("Invalid field name: '%s'" % name) + except FieldError: + names = opts.get_all_field_names() + self.extra_select.keys() + names.sort() + raise FieldError("Cannot resolve keyword %r into field. " + "Choices are: %s" % (name, ", ".join(names))) + + def add_ordering(self, *ordering): + """ + Adds items from the 'ordering' sequence to the query's "order by" + clause. These items are either field names (not column names) -- + possibly with a direction prefix ('-' or '?') -- or ordinals, + corresponding to column positions in the 'select' list. + + If 'ordering' is empty, all ordering is cleared from the query. + """ + errors = [] + for item in ordering: + if not ORDER_PATTERN.match(item): + errors.append(item) + if errors: + raise FieldError('Invalid order_by arguments: %s' % errors) + if ordering: + self.order_by.extend(ordering) + else: + self.default_ordering = False + + def clear_ordering(self, force_empty=False): + """ + Removes any ordering settings. If 'force_empty' is True, there will be + no ordering in the resulting query (not even the model's default). + """ + self.order_by = [] + self.extra_order_by = () + if force_empty: + self.default_ordering = False + + def add_count_column(self): + """ + Converts the query to do count(...) or count(distinct(pk)) in order to + get its size. + """ + # TODO: When group_by support is added, this needs to be adjusted so + # that it doesn't totally overwrite the select list. + if not self.distinct: + if not self.select: + select = Count() + else: + assert len(self.select) == 1, \ + "Cannot add count col with multiple cols in 'select': %r" % self.select + select = Count(self.select[0]) + else: + opts = self.model._meta + if not self.select: + select = Count((self.join((None, opts.db_table, None, None)), + opts.pk.column), True) + else: + # Because of SQL portability issues, multi-column, distinct + # counts need a sub-query -- see get_count() for details. + assert len(self.select) == 1, \ + "Cannot add count col with multiple cols in 'select'." + select = Count(self.select[0], True) + + # Distinct handling is done in Count(), so don't do it at this + # level. + self.distinct = False + self.select = [select] + self.select_fields = [None] + self.extra_select = {} + + def add_select_related(self, fields): + """ + Sets up the select_related data structure so that we only select + certain related models (as opposed to all models, when + self.select_related=True). + """ + field_dict = {} + for field in fields: + d = field_dict + for part in field.split(LOOKUP_SEP): + d = d.setdefault(part, {}) + self.select_related = field_dict + self.related_select_cols = [] + self.related_select_fields = [] + + def add_extra(self, select, select_params, where, params, tables, order_by): + """ + Adds data to the various extra_* attributes for user-created additions + to the query. + """ + if select: + # We need to pair any placeholder markers in the 'select' + # dictionary with their parameters in 'select_params' so that + # subsequent updates to the select dictionary also adjust the + # parameters appropriately. + select_pairs = SortedDict() + if select_params: + param_iter = iter(select_params) + else: + param_iter = iter([]) + for name, entry in select.items(): + entry = force_unicode(entry) + entry_params = [] + pos = entry.find("%s") + while pos != -1: + entry_params.append(param_iter.next()) + pos = entry.find("%s", pos + 2) + select_pairs[name] = (entry, entry_params) + # This is order preserving, since self.extra_select is a SortedDict. + self.extra_select.update(select_pairs) + if where: + self.extra_where += tuple(where) + if params: + self.extra_params += tuple(params) + if tables: + self.extra_tables += tuple(tables) + if order_by: + self.extra_order_by = order_by + + def trim_extra_select(self, names): + """ + Removes any aliases in the extra_select dictionary that aren't in + 'names'. + + This is needed if we are selecting certain values that don't incldue + all of the extra_select names. + """ + for key in set(self.extra_select).difference(set(names)): + del self.extra_select[key] + + def set_start(self, start): + """ + Sets the table from which to start joining. The start position is + specified by the related attribute from the base model. This will + automatically set to the select column to be the column linked from the + previous table. + + This method is primarily for internal use and the error checking isn't + as friendly as add_filter(). Mostly useful for querying directly + against the join table of many-to-many relation in a subquery. + """ + opts = self.model._meta + alias = self.get_initial_alias() + field, col, opts, joins, last = self.setup_joins( + start.split(LOOKUP_SEP), opts, alias, False) + alias = joins[last[-1]] + self.select = [(alias, self.alias_map[alias][RHS_JOIN_COL])] + self.select_fields = [field] + self.start_meta = opts + + # The call to setup_joins add an extra reference to everything in + # joins. So we need to unref everything once, and everything prior to + # the final join a second time. + for alias in joins: + self.unref_alias(alias) + for alias in joins[:last[-1]]: + self.unref_alias(alias) + + def execute_sql(self, result_type=MULTI): + """ + Run the query against the database and returns the result(s). The + return value is a single data item if result_type is SINGLE, or an + iterator over the results if the result_type is MULTI. + + result_type is either MULTI (use fetchmany() to retrieve all rows), + SINGLE (only retrieve a single row), or None (no results expected, but + the cursor is returned, since it's used by subclasses such as + InsertQuery). + """ + try: + sql, params = self.as_sql() + if not sql: + raise EmptyResultSet + except EmptyResultSet: + if result_type == MULTI: + return empty_iter() + else: + return + + cursor = self.connection.cursor() + cursor.execute(sql, params) + + if not result_type: + return cursor + if result_type == SINGLE: + if self.ordering_aliases: + return cursor.fetchone()[:-len(results.ordering_aliases)] + return cursor.fetchone() + + # The MULTI case. + if self.ordering_aliases: + result = order_modified_iter(cursor, len(self.ordering_aliases), + self.connection.features.empty_fetchmany_value) + else: + result = iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), + self.connection.features.empty_fetchmany_value) + if not self.connection.features.can_use_chunked_reads: + # If we are using non-chunked reads, we return the same data + # structure as normally, but ensure it is all read into memory + # before going any further. + return list(result) + return result + +# Use the backend's custom Query class if it defines one. Otherwise, use the +# default. +if connection.features.uses_custom_query_class: + Query = connection.ops.query_class(Query) + +def get_order_dir(field, default='ASC'): + """ + Returns the field name and direction for an order specification. For + example, '-foo' is returned as ('foo', 'DESC'). + + The 'default' param is used to indicate which way no prefix (or a '+' + prefix) should sort. The '-' prefix always sorts the opposite way. + """ + dirn = ORDER_DIR[default] + if field[0] == '-': + return field[1:], dirn[1] + return field, dirn[0] + +def empty_iter(): + """ + Returns an iterator containing no results. + """ + yield iter([]).next() + +def order_modified_iter(cursor, trim, sentinel): + """ + Yields blocks of rows from a cursor. We use this iterator in the special + case when extra output columns have been added to support ordering + requirements. We must trim those extra columns before anything else can use + the results, since they're only needed to make the SQL valid. + """ + for rows in iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), + sentinel): + yield [r[:-trim] for r in rows] + +def setup_join_cache(sender, **kwargs): + """ + The information needed to join between model fields is something that is + invariant over the life of the model, so we cache it in the model's Options + class, rather than recomputing it all the time. + + This method initialises the (empty) cache when the model is created. + """ + sender._meta._join_cache = {} + +signals.class_prepared.connect(setup_join_cache) + diff --git a/webapp/django/db/models/sql/subqueries.py b/webapp/django/db/models/sql/subqueries.py new file mode 100644 index 0000000000..cea2a58d39 --- /dev/null +++ b/webapp/django/db/models/sql/subqueries.py @@ -0,0 +1,411 @@ +""" +Query subclasses which provide extra functionality beyond simple data retrieval. +""" + +from django.core.exceptions import FieldError +from django.db.models.sql.constants import * +from django.db.models.sql.datastructures import Date +from django.db.models.sql.query import Query +from django.db.models.sql.where import AND + +__all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery', + 'CountQuery'] + +class DeleteQuery(Query): + """ + Delete queries are done through this class, since they are more constrained + than general queries. + """ + def as_sql(self): + """ + Creates the SQL for this query. Returns the SQL string and list of + parameters. + """ + assert len(self.tables) == 1, \ + "Can only delete from one table at a time." + result = ['DELETE FROM %s' % self.quote_name_unless_alias(self.tables[0])] + where, params = self.where.as_sql() + result.append('WHERE %s' % where) + return ' '.join(result), tuple(params) + + def do_query(self, table, where): + self.tables = [table] + self.where = where + self.execute_sql(None) + + def delete_batch_related(self, pk_list): + """ + Set up and execute delete queries for all the objects related to the + primary key values in pk_list. To delete the objects themselves, use + the delete_batch() method. + + More than one physical query may be executed if there are a + lot of values in pk_list. + """ + from django.contrib.contenttypes import generic + cls = self.model + for related in cls._meta.get_all_related_many_to_many_objects(): + if not isinstance(related.field, generic.GenericRelation): + for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): + where = self.where_class() + where.add((None, related.field.m2m_reverse_name(), + related.field, 'in', + pk_list[offset : offset+GET_ITERATOR_CHUNK_SIZE]), + AND) + self.do_query(related.field.m2m_db_table(), where) + + for f in cls._meta.many_to_many: + w1 = self.where_class() + if isinstance(f, generic.GenericRelation): + from django.contrib.contenttypes.models import ContentType + field = f.rel.to._meta.get_field(f.content_type_field_name) + w1.add((None, field.column, field, 'exact', + ContentType.objects.get_for_model(cls).id), AND) + for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): + where = self.where_class() + where.add((None, f.m2m_column_name(), f, 'in', + pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), + AND) + if w1: + where.add(w1, AND) + self.do_query(f.m2m_db_table(), where) + + def delete_batch(self, pk_list): + """ + Set up and execute delete queries for all the objects in pk_list. This + should be called after delete_batch_related(), if necessary. + + More than one physical query may be executed if there are a + lot of values in pk_list. + """ + for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): + where = self.where_class() + field = self.model._meta.pk + where.add((None, field.column, field, 'in', + pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), AND) + self.do_query(self.model._meta.db_table, where) + +class UpdateQuery(Query): + """ + Represents an "update" SQL query. + """ + def __init__(self, *args, **kwargs): + super(UpdateQuery, self).__init__(*args, **kwargs) + self._setup_query() + + def _setup_query(self): + """ + Runs on initialization and after cloning. Any attributes that would + normally be set in __init__ should go in here, instead, so that they + are also set up after a clone() call. + """ + self.values = [] + self.related_ids = None + if not hasattr(self, 'related_updates'): + self.related_updates = {} + + def clone(self, klass=None, **kwargs): + return super(UpdateQuery, self).clone(klass, + related_updates=self.related_updates.copy, **kwargs) + + def execute_sql(self, result_type=None): + """ + Execute the specified update. Returns the number of rows affected by + the primary update query (there could be other updates on related + tables, but their rowcounts are not returned). + """ + cursor = super(UpdateQuery, self).execute_sql(result_type) + rows = cursor.rowcount + del cursor + for query in self.get_related_updates(): + query.execute_sql(result_type) + return rows + + def as_sql(self): + """ + Creates the SQL for this query. Returns the SQL string and list of + parameters. + """ + self.pre_sql_setup() + if not self.values: + return '', () + table = self.tables[0] + qn = self.quote_name_unless_alias + result = ['UPDATE %s' % qn(table)] + result.append('SET') + values, update_params = [], [] + for name, val, placeholder in self.values: + if val is not None: + values.append('%s = %s' % (qn(name), placeholder)) + update_params.append(val) + else: + values.append('%s = NULL' % qn(name)) + result.append(', '.join(values)) + where, params = self.where.as_sql() + if where: + result.append('WHERE %s' % where) + return ' '.join(result), tuple(update_params + params) + + def pre_sql_setup(self): + """ + If the update depends on results from other tables, we need to do some + munging of the "where" conditions to match the format required for + (portable) SQL updates. That is done here. + + Further, if we are going to be running multiple updates, we pull out + the id values to update at this point so that they don't change as a + result of the progressive updates. + """ + self.select_related = False + self.clear_ordering(True) + super(UpdateQuery, self).pre_sql_setup() + count = self.count_active_tables() + if not self.related_updates and count == 1: + return + + # We need to use a sub-select in the where clause to filter on things + # from other tables. + query = self.clone(klass=Query) + query.bump_prefix() + query.extra_select = {} + first_table = query.tables[0] + if query.alias_refcount[first_table] == 1: + # We can remove one table from the inner query. + query.unref_alias(first_table) + for i in xrange(1, len(query.tables)): + table = query.tables[i] + if query.alias_refcount[table]: + break + join_info = query.alias_map[table] + query.select = [(join_info[RHS_ALIAS], join_info[RHS_JOIN_COL])] + must_pre_select = False + else: + query.select = [] + query.add_fields([query.model._meta.pk.name]) + must_pre_select = not self.connection.features.update_can_self_select + + # Now we adjust the current query: reset the where clause and get rid + # of all the tables we don't need (since they're in the sub-select). + self.where = self.where_class() + if self.related_updates or must_pre_select: + # Either we're using the idents in multiple update queries (so + # don't want them to change), or the db backend doesn't support + # selecting from the updating table (e.g. MySQL). + idents = [] + for rows in query.execute_sql(MULTI): + idents.extend([r[0] for r in rows]) + self.add_filter(('pk__in', idents)) + self.related_ids = idents + else: + # The fast path. Filters and updates in one query. + self.add_filter(('pk__in', query)) + for alias in self.tables[1:]: + self.alias_refcount[alias] = 0 + + def clear_related(self, related_field, pk_list): + """ + Set up and execute an update query that clears related entries for the + keys in pk_list. + + This is used by the QuerySet.delete_objects() method. + """ + for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): + self.where = self.where_class() + f = self.model._meta.pk + self.where.add((None, f.column, f, 'in', + pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), + AND) + self.values = [(related_field.column, None, '%s')] + self.execute_sql(None) + + def add_update_values(self, values): + """ + Convert a dictionary of field name to value mappings into an update + query. This is the entry point for the public update() method on + querysets. + """ + values_seq = [] + for name, val in values.iteritems(): + field, model, direct, m2m = self.model._meta.get_field_by_name(name) + if not direct or m2m: + raise FieldError('Cannot update model field %r (only non-relations and foreign keys permitted).' % field) + values_seq.append((field, model, val)) + return self.add_update_fields(values_seq) + + def add_update_fields(self, values_seq): + """ + Turn a sequence of (field, model, value) triples into an update query. + Used by add_update_values() as well as the "fast" update path when + saving models. + """ + from django.db.models.base import Model + for field, model, val in values_seq: + # FIXME: Some sort of db_prep_* is probably more appropriate here. + if field.rel and isinstance(val, Model): + val = val.pk + + # Getting the placeholder for the field. + if hasattr(field, 'get_placeholder'): + placeholder = field.get_placeholder(val) + else: + placeholder = '%s' + + if model: + self.add_related_update(model, field.column, val, placeholder) + else: + self.values.append((field.column, val, placeholder)) + + def add_related_update(self, model, column, value, placeholder): + """ + Adds (name, value) to an update query for an ancestor model. + + Updates are coalesced so that we only run one update query per ancestor. + """ + try: + self.related_updates[model].append((column, value, placeholder)) + except KeyError: + self.related_updates[model] = [(column, value, placeholder)] + + def get_related_updates(self): + """ + Returns a list of query objects: one for each update required to an + ancestor model. Each query will have the same filtering conditions as + the current query but will only update a single table. + """ + if not self.related_updates: + return [] + result = [] + for model, values in self.related_updates.iteritems(): + query = UpdateQuery(model, self.connection) + query.values = values + if self.related_ids: + query.add_filter(('pk__in', self.related_ids)) + result.append(query) + return result + +class InsertQuery(Query): + def __init__(self, *args, **kwargs): + super(InsertQuery, self).__init__(*args, **kwargs) + self.columns = [] + self.values = [] + self.params = () + + def clone(self, klass=None, **kwargs): + extras = {'columns': self.columns[:], 'values': self.values[:], + 'params': self.params} + return super(InsertQuery, self).clone(klass, extras) + + def as_sql(self): + # We don't need quote_name_unless_alias() here, since these are all + # going to be column names (so we can avoid the extra overhead). + qn = self.connection.ops.quote_name + result = ['INSERT INTO %s' % qn(self.model._meta.db_table)] + result.append('(%s)' % ', '.join([qn(c) for c in self.columns])) + result.append('VALUES (%s)' % ', '.join(self.values)) + return ' '.join(result), self.params + + def execute_sql(self, return_id=False): + cursor = super(InsertQuery, self).execute_sql(None) + if return_id: + return self.connection.ops.last_insert_id(cursor, + self.model._meta.db_table, self.model._meta.pk.column) + + def insert_values(self, insert_values, raw_values=False): + """ + Set up the insert query from the 'insert_values' dictionary. The + dictionary gives the model field names and their target values. + + If 'raw_values' is True, the values in the 'insert_values' dictionary + are inserted directly into the query, rather than passed as SQL + parameters. This provides a way to insert NULL and DEFAULT keywords + into the query, for example. + """ + placeholders, values = [], [] + for field, val in insert_values: + if hasattr(field, 'get_placeholder'): + # Some fields (e.g. geo fields) need special munging before + # they can be inserted. + placeholders.append(field.get_placeholder(val)) + else: + placeholders.append('%s') + + self.columns.append(field.column) + values.append(val) + if raw_values: + self.values.extend(values) + else: + self.params += tuple(values) + self.values.extend(placeholders) + +class DateQuery(Query): + """ + A DateQuery is a normal query, except that it specifically selects a single + date field. This requires some special handling when converting the results + back to Python objects, so we put it in a separate class. + """ + def __getstate__(self): + """ + Special DateQuery-specific pickle handling. + """ + for elt in self.select: + if isinstance(elt, Date): + # Eliminate a method reference that can't be pickled. The + # __setstate__ method restores this. + elt.date_sql_func = None + return super(DateQuery, self).__getstate__() + + def __setstate__(self, obj_dict): + super(DateQuery, self).__setstate__(obj_dict) + for elt in self.select: + if isinstance(elt, Date): + self.date_sql_func = self.connection.ops.date_trunc_sql + + def results_iter(self): + """ + Returns an iterator over the results from executing this query. + """ + resolve_columns = hasattr(self, 'resolve_columns') + if resolve_columns: + from django.db.models.fields import DateTimeField + fields = [DateTimeField()] + else: + from django.db.backends.util import typecast_timestamp + needs_string_cast = self.connection.features.needs_datetime_string_cast + + offset = len(self.extra_select) + for rows in self.execute_sql(MULTI): + for row in rows: + date = row[offset] + if resolve_columns: + date = self.resolve_columns([date], fields)[0] + elif needs_string_cast: + date = typecast_timestamp(str(date)) + yield date + + def add_date_select(self, field, lookup_type, order='ASC'): + """ + Converts the query into a date extraction query. + """ + result = self.setup_joins([field.name], self.get_meta(), + self.get_initial_alias(), False) + alias = result[3][-1] + select = Date((alias, field.column), lookup_type, + self.connection.ops.date_trunc_sql) + self.select = [select] + self.select_fields = [None] + self.select_related = False # See #7097. + self.distinct = True + self.order_by = order == 'ASC' and [1] or [-1] + +class CountQuery(Query): + """ + A CountQuery knows how to take a normal query which would select over + multiple distinct columns and turn it into SQL that can be used on a + variety of backends (it requires a select in the FROM clause). + """ + def get_from_clause(self): + result, params = self._query.as_sql() + return ['(%s) A1' % result], params + + def get_ordering(self): + return () diff --git a/webapp/django/db/models/sql/where.py b/webapp/django/db/models/sql/where.py new file mode 100644 index 0000000000..662d99a4a2 --- /dev/null +++ b/webapp/django/db/models/sql/where.py @@ -0,0 +1,213 @@ +""" +Code to manage the creation and SQL rendering of 'where' constraints. +""" +import datetime + +from django.utils import tree +from django.db import connection +from django.db.models.fields import Field +from django.db.models.query_utils import QueryWrapper +from datastructures import EmptyResultSet, FullResultSet + +# Connection types +AND = 'AND' +OR = 'OR' + +class WhereNode(tree.Node): + """ + Used to represent the SQL where-clause. + + The class is tied to the Query class that created it (in order to create + the correct SQL). + + The children in this tree are usually either Q-like objects or lists of + [table_alias, field_name, db_type, lookup_type, value_annotation, + params]. However, a child could also be any class with as_sql() and + relabel_aliases() methods. + """ + default = AND + + def add(self, data, connector): + """ + Add a node to the where-tree. If the data is a list or tuple, it is + expected to be of the form (alias, col_name, field_obj, lookup_type, + value), which is then slightly munged before being stored (to avoid + storing any reference to field objects). Otherwise, the 'data' is + stored unchanged and can be anything with an 'as_sql()' method. + """ + # Because of circular imports, we need to import this here. + from django.db.models.base import ObjectDoesNotExist + + if not isinstance(data, (list, tuple)): + super(WhereNode, self).add(data, connector) + return + + alias, col, field, lookup_type, value = data + try: + if field: + params = field.get_db_prep_lookup(lookup_type, value) + db_type = field.db_type() + else: + # This is possible when we add a comparison to NULL sometimes + # (we don't really need to waste time looking up the associated + # field object). + params = Field().get_db_prep_lookup(lookup_type, value) + db_type = None + except ObjectDoesNotExist: + # This can happen when trying to insert a reference to a null pk. + # We break out of the normal path and indicate there's nothing to + # match. + super(WhereNode, self).add(NothingNode(), connector) + return + if isinstance(value, datetime.datetime): + annotation = datetime.datetime + else: + annotation = bool(value) + super(WhereNode, self).add((alias, col, db_type, lookup_type, + annotation, params), connector) + + def as_sql(self, qn=None): + """ + Returns the SQL version of the where clause and the value to be + substituted in. Returns None, None if this node is empty. + + If 'node' is provided, that is the root of the SQL generation + (generally not needed except by the internal implementation for + recursion). + """ + if not qn: + qn = connection.ops.quote_name + if not self.children: + return None, [] + result = [] + result_params = [] + empty = True + for child in self.children: + try: + if hasattr(child, 'as_sql'): + sql, params = child.as_sql(qn=qn) + else: + # A leaf node in the tree. + sql, params = self.make_atom(child, qn) + except EmptyResultSet: + if self.connector == AND and not self.negated: + # We can bail out early in this particular case (only). + raise + elif self.negated: + empty = False + continue + except FullResultSet: + if self.connector == OR: + if self.negated: + empty = True + break + # We match everything. No need for any constraints. + return '', [] + if self.negated: + empty = True + continue + empty = False + if sql: + result.append(sql) + result_params.extend(params) + if empty: + raise EmptyResultSet + + conn = ' %s ' % self.connector + sql_string = conn.join(result) + if sql_string: + if self.negated: + sql_string = 'NOT (%s)' % sql_string + elif len(self.children) != 1: + sql_string = '(%s)' % sql_string + return sql_string, result_params + + def make_atom(self, child, qn): + """ + Turn a tuple (table_alias, column_name, db_type, lookup_type, + value_annot, params) into valid SQL. + + Returns the string for the SQL fragment and the parameters to use for + it. + """ + table_alias, name, db_type, lookup_type, value_annot, params = child + if table_alias: + lhs = '%s.%s' % (qn(table_alias), qn(name)) + else: + lhs = qn(name) + field_sql = connection.ops.field_cast_sql(db_type) % lhs + + if value_annot is datetime.datetime: + cast_sql = connection.ops.datetime_cast_sql() + else: + cast_sql = '%s' + + if isinstance(params, QueryWrapper): + extra, params = params.data + else: + extra = '' + + if lookup_type in connection.operators: + format = "%s %%s %s" % (connection.ops.lookup_cast(lookup_type), + extra) + return (format % (field_sql, + connection.operators[lookup_type] % cast_sql), params) + + if lookup_type == 'in': + if not value_annot: + raise EmptyResultSet + if extra: + return ('%s IN %s' % (field_sql, extra), params) + return ('%s IN (%s)' % (field_sql, ', '.join(['%s'] * len(params))), + params) + elif lookup_type in ('range', 'year'): + return ('%s BETWEEN %%s and %%s' % field_sql, params) + elif lookup_type in ('month', 'day'): + return ('%s = %%s' % connection.ops.date_extract_sql(lookup_type, + field_sql), params) + elif lookup_type == 'isnull': + return ('%s IS %sNULL' % (field_sql, + (not value_annot and 'NOT ' or '')), ()) + elif lookup_type == 'search': + return (connection.ops.fulltext_search_sql(field_sql), params) + elif lookup_type in ('regex', 'iregex'): + return connection.ops.regex_lookup(lookup_type) % (field_sql, cast_sql), params + + raise TypeError('Invalid lookup_type: %r' % lookup_type) + + def relabel_aliases(self, change_map, node=None): + """ + Relabels the alias values of any children. 'change_map' is a dictionary + mapping old (current) alias values to the new values. + """ + if not node: + node = self + for pos, child in enumerate(node.children): + if hasattr(child, 'relabel_aliases'): + child.relabel_aliases(change_map) + elif isinstance(child, tree.Node): + self.relabel_aliases(change_map, child) + else: + if child[0] in change_map: + node.children[pos] = (change_map[child[0]],) + child[1:] + +class EverythingNode(object): + """ + A node that matches everything. + """ + def as_sql(self, qn=None): + raise FullResultSet + + def relabel_aliases(self, change_map, node=None): + return + +class NothingNode(object): + """ + A node that matches nothing. + """ + def as_sql(self, qn=None): + raise EmptyResultSet + + def relabel_aliases(self, change_map, node=None): + return + diff --git a/webapp/django/db/transaction.py b/webapp/django/db/transaction.py new file mode 100644 index 0000000000..506074f1b3 --- /dev/null +++ b/webapp/django/db/transaction.py @@ -0,0 +1,267 @@ +""" +This module implements a transaction manager that can be used to define +transaction handling in a request or view function. It is used by transaction +control middleware and decorators. + +The transaction manager can be in managed or in auto state. Auto state means the +system is using a commit-on-save strategy (actually it's more like +commit-on-change). As soon as the .save() or .delete() (or related) methods are +called, a commit is made. + +Managed transactions don't do those commits, but will need some kind of manual +or implicit commits or rollbacks. +""" + +try: + import thread +except ImportError: + import dummy_thread as thread +try: + from functools import wraps +except ImportError: + from django.utils.functional import wraps # Python 2.3, 2.4 fallback. +from django.db import connection +from django.conf import settings + +class TransactionManagementError(Exception): + """ + This exception is thrown when something bad happens with transaction + management. + """ + pass + +# The states are dictionaries of lists. The key to the dict is the current +# thread and the list is handled as a stack of values. +state = {} +savepoint_state = {} + +# The dirty flag is set by *_unless_managed functions to denote that the +# code under transaction management has changed things to require a +# database commit. +dirty = {} + +def enter_transaction_management(): + """ + Enters transaction management for a running thread. It must be balanced with + the appropriate leave_transaction_management call, since the actual state is + managed as a stack. + + The state and dirty flag are carried over from the surrounding block or + from the settings, if there is no surrounding block (dirty is always false + when no current block is running). + """ + thread_ident = thread.get_ident() + if thread_ident in state and state[thread_ident]: + state[thread_ident].append(state[thread_ident][-1]) + else: + state[thread_ident] = [] + state[thread_ident].append(settings.TRANSACTIONS_MANAGED) + if thread_ident not in dirty: + dirty[thread_ident] = False + +def leave_transaction_management(): + """ + Leaves transaction management for a running thread. A dirty flag is carried + over to the surrounding block, as a commit will commit all changes, even + those from outside. (Commits are on connection level.) + """ + thread_ident = thread.get_ident() + if thread_ident in state and state[thread_ident]: + del state[thread_ident][-1] + else: + raise TransactionManagementError("This code isn't under transaction management") + if dirty.get(thread_ident, False): + rollback() + raise TransactionManagementError("Transaction managed block ended with pending COMMIT/ROLLBACK") + dirty[thread_ident] = False + +def is_dirty(): + """ + Returns True if the current transaction requires a commit for changes to + happen. + """ + return dirty.get(thread.get_ident(), False) + +def set_dirty(): + """ + Sets a dirty flag for the current thread and code streak. This can be used + to decide in a managed block of code to decide whether there are open + changes waiting for commit. + """ + thread_ident = thread.get_ident() + if thread_ident in dirty: + dirty[thread_ident] = True + else: + raise TransactionManagementError("This code isn't under transaction management") + +def set_clean(): + """ + Resets a dirty flag for the current thread and code streak. This can be used + to decide in a managed block of code to decide whether a commit or rollback + should happen. + """ + thread_ident = thread.get_ident() + if thread_ident in dirty: + dirty[thread_ident] = False + else: + raise TransactionManagementError("This code isn't under transaction management") + clean_savepoints() + +def clean_savepoints(): + thread_ident = thread.get_ident() + if thread_ident in savepoint_state: + del savepoint_state[thread_ident] + +def is_managed(): + """ + Checks whether the transaction manager is in manual or in auto state. + """ + thread_ident = thread.get_ident() + if thread_ident in state: + if state[thread_ident]: + return state[thread_ident][-1] + return settings.TRANSACTIONS_MANAGED + +def managed(flag=True): + """ + Puts the transaction manager into a manual state: managed transactions have + to be committed explicitly by the user. If you switch off transaction + management and there is a pending commit/rollback, the data will be + commited. + """ + thread_ident = thread.get_ident() + top = state.get(thread_ident, None) + if top: + top[-1] = flag + if not flag and is_dirty(): + connection._commit() + set_clean() + else: + raise TransactionManagementError("This code isn't under transaction management") + +def commit_unless_managed(): + """ + Commits changes if the system is not in managed transaction mode. + """ + if not is_managed(): + connection._commit() + clean_savepoints() + else: + set_dirty() + +def rollback_unless_managed(): + """ + Rolls back changes if the system is not in managed transaction mode. + """ + if not is_managed(): + connection._rollback() + else: + set_dirty() + +def commit(): + """ + Does the commit itself and resets the dirty flag. + """ + connection._commit() + set_clean() + +def rollback(): + """ + This function does the rollback itself and resets the dirty flag. + """ + connection._rollback() + set_clean() + +def savepoint(): + """ + Creates a savepoint (if supported and required by the backend) inside the + current transaction. Returns an identifier for the savepoint that will be + used for the subsequent rollback or commit. + """ + thread_ident = thread.get_ident() + if thread_ident in savepoint_state: + savepoint_state[thread_ident].append(None) + else: + savepoint_state[thread_ident] = [None] + tid = str(thread_ident).replace('-', '') + sid = "s%s_x%d" % (tid, len(savepoint_state[thread_ident])) + connection._savepoint(sid) + return sid + +def savepoint_rollback(sid): + """ + Rolls back the most recent savepoint (if one exists). Does nothing if + savepoints are not supported. + """ + if thread.get_ident() in savepoint_state: + connection._savepoint_rollback(sid) + +def savepoint_commit(sid): + """ + Commits the most recent savepoint (if one exists). Does nothing if + savepoints are not supported. + """ + if thread.get_ident() in savepoint_state: + connection._savepoint_commit(sid) + +############## +# DECORATORS # +############## + +def autocommit(func): + """ + Decorator that activates commit on save. This is Django's default behavior; + this decorator is useful if you globally activated transaction management in + your settings file and want the default behavior in some view functions. + """ + def _autocommit(*args, **kw): + try: + enter_transaction_management() + managed(False) + return func(*args, **kw) + finally: + leave_transaction_management() + return wraps(func)(_autocommit) + +def commit_on_success(func): + """ + This decorator activates commit on response. This way, if the view function + runs successfully, a commit is made; if the viewfunc produces an exception, + a rollback is made. This is one of the most common ways to do transaction + control in web apps. + """ + def _commit_on_success(*args, **kw): + try: + enter_transaction_management() + managed(True) + try: + res = func(*args, **kw) + except: + # All exceptions must be handled here (even string ones). + if is_dirty(): + rollback() + raise + else: + if is_dirty(): + commit() + return res + finally: + leave_transaction_management() + return wraps(func)(_commit_on_success) + +def commit_manually(func): + """ + Decorator that activates manual transaction control. It just disables + automatic transaction control and doesn't do any commit/rollback of its + own -- it's up to the user to call the commit and rollback functions + themselves. + """ + def _commit_manually(*args, **kw): + try: + enter_transaction_management() + managed(True) + return func(*args, **kw) + finally: + leave_transaction_management() + + return wraps(func)(_commit_manually) |