#!/usr/bin/env python2.7
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#




"""Generate Datastore Stats over Dev mode appserver's datastore."""









import datetime
import logging

from google.appengine.api import datastore
from google.appengine.api import datastore_admin
from google.appengine.api import datastore_types
from google.appengine.api import users
from google.appengine.ext.db import stats

DELETE_BATCH_SIZE = 100


_GLOBAL_KEY = (stats.GlobalStat, 'total_entity_usage', '')






_PROPERTY_TYPE_TO_DSS_NAME = {
    unicode: ('String', 'STRING'),
    bool: ('Boolean', 'BOOLEAN'),
    long: ('Integer', 'INT64'),
    type(None): ('NULL', 'NULL'),
    float: ('Float', 'DOUBLE'),
    datastore_types.Key: ('Key', 'REFERENCE'),
    datastore_types.Blob: ('Blob', 'STRING'),
    datastore_types.EmbeddedEntity: ('EmbeddedEntity', 'STRING'),
    datastore_types.ByteString: ('ShortBlob', 'STRING'),
    datastore_types.Text: ('Text', 'STRING'),
    users.User: ('User', 'USER'),
    datastore_types.Category: ('Category', 'STRING'),
    datastore_types.Link: ('Link', 'STRING'),
    datastore_types.Email: ('Email', 'STRING'),
    datetime.datetime: ('Date/Time', 'INT64'),
    datastore_types.GeoPt: ('GeoPt', 'POINT'),
    datastore_types.IM: ('IM', 'STRING'),
    datastore_types.PhoneNumber: ('PhoneNumber', 'STRING'),
    datastore_types.PostalAddress: ('PostalAddress', 'STRING'),
    datastore_types.Rating: ('Rating', 'INT64'),
    datastore_types.BlobKey: ('BlobKey', 'STRING'),
    }




class DatastoreStatsProcessor(object):
  """Generates datastore stats for an app's an datastore entities."""

  def __init__(self, _app=None):
    self.app_id = datastore_types.ResolveAppId(_app)


    self.whole_app_stats = {}



    self.namespace_stats = {}
    self.found_non_empty_namespace = False


    self.old_stat_keys = []


    self.timestamp = datetime.datetime.utcnow()

  def __ScanAllNamespaces(self):
    """Scans all the namespaces and processes each namespace."""
    namespace_query = datastore.Query('__namespace__', _app=self.app_id)

    for namespace_entity in namespace_query.Run():
      name = namespace_entity.key().name()
      if name is None:
        name = ''
      self.__ProcessNamespace(name)

  def __ProcessNamespace(self, namespace):
    """Process all the entities in a given namespace."""

    all_query = datastore.Query(namespace=namespace, _app=self.app_id)
    all_query['__key__ >='] = datastore_types.Key.from_path(
        ' ', 1L, namespace=namespace, _app=self.app_id)


    for entity in all_query.Run():
      self.found_non_empty_namespace |= (namespace != '')
      proto = entity.ToPb()
      proto_size = len(proto.SerializeToString())

      if entity.key().kind() in stats._DATASTORE_STATS_CLASSES_BY_KIND:



        stat_kind = stats._DATASTORE_STATS_CLASSES_BY_KIND[entity.key().kind()]

        self.old_stat_keys.append(entity.key())
        self.__AggregateTotal(proto_size, entity.key(), proto, namespace,
                              stat_kind)
      else:
        self.__ProcessUserEntity(proto_size, entity.key(), proto, namespace)

  def __GetPropertyIndexStat(self, namespace, kind_name,
                             entity_key_size, prop):
    """Return the size and count of indexes for a property of an EntityProto."""

    property_index_size = (len(self.app_id) + len(kind_name) +
                           len(prop.value().SerializeToString()) +
                           len(namespace) + entity_key_size)

    return (property_index_size, 2)

  def __GetTypeIndexStat(self, namespace, kind_name, entity_key_size):
    """Return the size and count of indexes by type of an EntityProto."""
    type_index_size = (len(self.app_id) + len(kind_name) + entity_key_size
                       + len(namespace))
    return (type_index_size, 1)

  def __ProcessUserEntity(self, proto_size, key, proto, namespace):
    """Increment datastore stats for a non stats record."""
    self.__AggregateTotal(proto_size, key, proto, namespace, None)

    kind_name = key.kind()

    entity_key_size = (len(proto.key().app()) + len(namespace) +
                       len(proto.key().path().SerializeToString()) +
                       len(proto.entity_group().SerializeToString()))

    self.__AggregateCompositeIndices(proto, namespace, kind_name,
                                     entity_key_size)

    type_index_size, type_index_count = self.__GetTypeIndexStat(namespace,
                                                                kind_name,
                                                                entity_key_size)
    property_index_count = 0
    property_index_size = 0


    for prop_list in (proto.property_list(), proto.raw_property_list()):
      for prop in prop_list:
        index_size, index_count = self.__GetPropertyIndexStat(namespace,
                                                              kind_name,
                                                              entity_key_size,
                                                              prop)
        property_index_size += index_size
        property_index_count += index_count

    builtin_index_size = type_index_size + property_index_size
    builtin_index_count = type_index_count + property_index_count

    self.__Increment(self.whole_app_stats, 1,
                     (stats.KindStat, kind_name, ''),
                     proto_size,
                     builtin_index_count=builtin_index_count,
                     builtin_index_size=builtin_index_size,
                     kind_name=kind_name)

    self.__Increment(self.namespace_stats, 1,
                     (stats.NamespaceKindStat, kind_name, namespace),
                     proto_size,
                     builtin_index_count=builtin_index_count,
                     builtin_index_size=builtin_index_size,
                     kind_name=kind_name)



    if key.parent() is None:
      whole_app_model = stats.KindRootEntityStat
      namespace_model = stats.NamespaceKindRootEntityStat
    else:
      whole_app_model = stats.KindNonRootEntityStat
      namespace_model = stats.NamespaceKindNonRootEntityStat

    self.__Increment(self.whole_app_stats, 1,
                     (whole_app_model, kind_name, ''),
                     proto_size,
                     kind_name=kind_name)

    self.__Increment(self.namespace_stats, 1,
                     (namespace_model, kind_name, namespace),
                     proto_size,
                     kind_name=kind_name)

    self.__ProcessProperties(
        kind_name,
        namespace,
        entity_key_size,
        (proto.property_list(), proto.raw_property_list()))

  def __ProcessProperties(self, kind_name, namespace, entity_key_size,
                          prop_lists):
    for prop_list in prop_lists:
      for prop in prop_list:
        try:
          value = datastore_types.FromPropertyPb(prop)
          self.__AggregateProperty(kind_name, namespace, entity_key_size,
                                   prop, value)
        except (AssertionError, AttributeError, TypeError, ValueError), e:
          logging.error('Cannot process property %r, exception %s' %
                        (prop, e))

  def __AggregateProperty(self, kind_name, namespace, entity_key_size,
                          prop, value):
    property_name = prop.name()
    property_type = _PROPERTY_TYPE_TO_DSS_NAME[type(value)][0]
    index_property_type = _PROPERTY_TYPE_TO_DSS_NAME[type(value)][1]
    size = len(prop.SerializeToString())


    index_size, index_count = self.__GetPropertyIndexStat(namespace, kind_name,
                                                          entity_key_size, prop)





    self.__Increment(self.whole_app_stats, 1,
                     (stats.PropertyTypeStat, property_type, ''),
                     size,
                     builtin_index_count=0,
                     builtin_index_size=0,
                     property_type=property_type)

    self.__Increment(self.whole_app_stats, 0,
                     (stats.PropertyTypeStat, index_property_type, ''),
                     0,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_type=index_property_type)

    self.__Increment(self.namespace_stats, 1,
                     (stats.NamespacePropertyTypeStat,
                      property_type, namespace),
                     size,
                     builtin_index_count=0,
                     builtin_index_size=0,
                     property_type=property_type)

    self.__Increment(self.namespace_stats, 0,
                     (stats.NamespacePropertyTypeStat,
                      index_property_type, namespace),
                     0,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_type=index_property_type)


    self.__Increment(self.whole_app_stats, 1,
                     (stats.KindPropertyTypeStat,
                      property_type + '_' + kind_name, ''),
                     size,
                     builtin_index_count=0,
                     builtin_index_size=0,
                     property_type=property_type, kind_name=kind_name)

    self.__Increment(self.whole_app_stats, 0,
                     (stats.KindPropertyTypeStat,
                      index_property_type + '_' + kind_name, ''),
                     0,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_type=index_property_type, kind_name=kind_name)

    self.__Increment(self.namespace_stats, 1,
                     (stats.NamespaceKindPropertyTypeStat,
                      property_type + '_' + kind_name, namespace),
                     size,
                     builtin_index_count=0,
                     builtin_index_size=0,
                     property_type=property_type, kind_name=kind_name)

    self.__Increment(self.namespace_stats, 0,
                     (stats.NamespaceKindPropertyTypeStat,
                      index_property_type + '_' + kind_name, namespace),
                     0,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_type=index_property_type, kind_name=kind_name)


    self.__Increment(self.whole_app_stats, 1,
                     (stats.KindPropertyNameStat,
                      property_name + '_' + kind_name, ''),
                     size,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_name=property_name, kind_name=kind_name)

    self.__Increment(self.namespace_stats, 1,
                     (stats.NamespaceKindPropertyNameStat,
                      property_name + '_' + kind_name, namespace),
                     size,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_name=property_name, kind_name=kind_name)


    self.__Increment(self.whole_app_stats, 1,
                     (stats.KindPropertyNamePropertyTypeStat,
                      property_type + '_' + property_name + '_' + kind_name,
                      ''), size,
                     builtin_index_count=0,
                     builtin_index_size=0,
                     property_type=property_type,
                     property_name=property_name, kind_name=kind_name)

    self.__Increment(self.whole_app_stats, 0,
                     (stats.KindPropertyNamePropertyTypeStat,
                      index_property_type + '_' + property_name + '_' +
                      kind_name,
                      ''), 0,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_type=index_property_type,
                     property_name=property_name, kind_name=kind_name)

    self.__Increment(self.namespace_stats, 1,
                     (stats.NamespaceKindPropertyNamePropertyTypeStat,
                      property_type + '_' + property_name + '_' + kind_name,
                      namespace),
                     size,
                     builtin_index_count=0,
                     builtin_index_size=0,
                     property_type=property_type,
                     property_name=property_name, kind_name=kind_name)

    self.__Increment(self.namespace_stats, 0,
                     (stats.NamespaceKindPropertyNamePropertyTypeStat,
                      index_property_type + '_' + property_name + '_' +
                      kind_name,
                      namespace),
                     0,
                     builtin_index_count=index_count,
                     builtin_index_size=index_size,
                     property_type=index_property_type,
                     property_name=property_name, kind_name=kind_name)

  def __GetCompositeIndexStat(self, definition, proto, namespace, kind_name,
                              entity_key_size):
    """Get statistics of composite index for a index definition of an entity."""






    property_list = proto.property_list()
    property_count = []
    property_size = []
    index_count = 1
    for indexed_prop in definition.property_list():
      name = indexed_prop.name()
      count = 0
      prop_size = 0
      for prop in property_list:
        if prop.name() == name:
          count += 1
          prop_size += len(prop.SerializeToString())

      property_count.append(count)
      property_size.append(prop_size)
      index_count *= count

    if index_count == 0:
      return (0, 0)

    index_only_size = 0
    for i in range(len(property_size)):
      index_only_size += property_size[i] * (index_count / property_count[i])





    index_size = (index_count * (entity_key_size + len(kind_name) +
                                 len(self.app_id) + len(namespace)) +
                  index_only_size * 2)

    return (index_size, index_count)

  def __AggregateCompositeIndices(self, proto, namespace, kind_name,
                                  entity_key_size):
    """Aggregate statistics of composite indexes for an entity."""
    composite_indices = datastore_admin.GetIndices(self.app_id)
    for index in composite_indices:
      definition = index.definition()
      if kind_name != definition.entity_type():
        continue

      index_size, index_count = self.__GetCompositeIndexStat(definition, proto,
                                                             namespace,
                                                             kind_name,
                                                             entity_key_size)

      if index_count == 0:
        continue


      name_id = namespace
      if not name_id:
        name_id = 1


      self.__Increment(self.whole_app_stats, 0, _GLOBAL_KEY, 0,
                       composite_index_count=index_count,
                       composite_index_size=index_size)

      self.__Increment(self.whole_app_stats, 0,
                       (stats.NamespaceStat, name_id, ''), 0,
                       composite_index_count=index_count,
                       composite_index_size=index_size,
                       subject_namespace=namespace)

      self.__Increment(self.namespace_stats, 0,
                       (stats.NamespaceGlobalStat, 'total_entity_usage',
                        namespace), 0,
                       composite_index_count=index_count,
                       composite_index_size=index_size)


      self.__Increment(self.whole_app_stats, 0,
                       (stats.KindStat, kind_name, ''), 0,
                       composite_index_count=index_count,
                       composite_index_size=index_size,
                       kind_name=kind_name)

      self.__Increment(self.namespace_stats, 0,
                       (stats.NamespaceKindStat, kind_name, namespace), 0,
                       composite_index_count=index_count,
                       composite_index_size=index_size,
                       kind_name=kind_name)


      index_id = index.id()
      self.__Increment(self.whole_app_stats, index_count,
                       (stats.KindCompositeIndexStat,
                        kind_name + '_%s' % index_id, ''), index_size,
                       kind_name=kind_name, index_id=index_id)

      self.__Increment(self.namespace_stats, index_count,
                       (stats.NamespaceKindCompositeIndexStat,
                        kind_name + '_%s' % index_id, namespace), index_size,
                       kind_name=kind_name, index_id=index_id)

  def __AggregateTotal(self, size, key, proto, namespace, stat_kind):
    """Aggregate total datastore stats."""
    kind_name = key.kind()

    entity_key_size = (len(proto.key().app()) +
                       len(proto.key().path().SerializeToString()) +
                       len(proto.entity_group().SerializeToString()))

    type_index_size, type_index_count = self.__GetTypeIndexStat(namespace,
                                                                kind_name,
                                                                entity_key_size)
    property_index_count = 0
    property_index_size = 0
    for prop_list in (proto.property_list(), proto.raw_property_list()):
      for prop in prop_list:
        index_size, index_count = self.__GetPropertyIndexStat(namespace,
                                                              kind_name,
                                                              entity_key_size,
                                                              prop)
        property_index_size += index_size
        property_index_count += index_count

    builtin_index_size = type_index_size + property_index_size
    builtin_index_count = type_index_count + property_index_count


    if stat_kind == stats.GlobalStat:
      count = 0
    else:
      count = 1


    self.__Increment(self.whole_app_stats, count, _GLOBAL_KEY, size,
                     builtin_index_count=builtin_index_count,
                     builtin_index_size=builtin_index_size)


    name_id = namespace
    if not name_id:
      name_id = 1

    if (stat_kind == stats.NamespaceStat) and (namespace == ''):
      count = 0


    self.__Increment(self.whole_app_stats, count,
                     (stats.NamespaceStat, name_id, ''),
                     size,
                     builtin_index_count=builtin_index_count,
                     builtin_index_size=builtin_index_size,
                     subject_namespace=namespace)

    if stat_kind == stats.NamespaceGlobalStat:
      count = 0


    self.__Increment(
        self.namespace_stats, count,
        (stats.NamespaceGlobalStat, 'total_entity_usage', namespace), size,
        builtin_index_count=builtin_index_count,
        builtin_index_size=builtin_index_size)

  def __Increment(self, stats_dict, count, stat_key, size,
                  builtin_index_count=0, builtin_index_size=0,
                  composite_index_count=0, composite_index_size=0, **kwds):
    """Increment stats for a particular kind.

    Args:
        stats_dict: The dictionary where the entities are held.
          The entities are keyed by stat_key. e.g. The
          __Stat_Total__ entity will be found in stats_dict[_GLOBAL_KEY].
        count: The amount to increment the datastore stat by.
        stat_key: A tuple of (db.Model of the stat, key value, namespace).
        size: The "bytes" to increment the size by.
        builtin_index_count: The bytes of builtin index to add in to a stat.
        builtin_index_size: The count of builtin index to add in to a stat.
        composite_index_count: The bytes of composite index to add in to a stat.
        composite_index_size: The count of composite index to add in to a stat.
        kwds: Name value pairs that are set on the created entities.
    """

    if stat_key not in stats_dict:
      stat_model = stat_key[0](
          key=datastore_types.Key.from_path(stat_key[0].STORED_KIND_NAME,
                                            stat_key[1],
                                            namespace=stat_key[2],
                                            _app=self.app_id),
          _app=self.app_id)
      stats_dict[stat_key] = stat_model
      for field, value in kwds.iteritems():
        setattr(stat_model, field, value)
      stat_model.count = count
      if size:
        stat_model.entity_bytes = size
      if builtin_index_size:
        stat_model.builtin_index_bytes = builtin_index_size
        stat_model.builtin_index_count = builtin_index_count
      if composite_index_size:
        stat_model.composite_index_bytes = composite_index_size
        stat_model.composite_index_count = composite_index_count
      stat_model.bytes = size + builtin_index_size + composite_index_size
      stat_model.timestamp = self.timestamp
    else:
      stat_model = stats_dict[stat_key]
      stat_model.count += count
      if size:
        stat_model.entity_bytes += size
      if builtin_index_size:
        stat_model.builtin_index_bytes += builtin_index_size
        stat_model.builtin_index_count += builtin_index_count
      if composite_index_size:
        stat_model.composite_index_bytes += composite_index_size
        stat_model.composite_index_count += composite_index_count
      stat_model.bytes += size + builtin_index_size + composite_index_size

  def __Finalize(self):
    """Finishes processing, deletes all old stats and writes new ones."""

    for i in range(0, len(self.old_stat_keys), DELETE_BATCH_SIZE):
      datastore.Delete(self.old_stat_keys[i:i+DELETE_BATCH_SIZE])

    self.written = 0

    for stat in self.whole_app_stats.itervalues():
      if stat.count or not (isinstance(stat, stats.GlobalStat) or
                            isinstance(stat, stats.NamespaceStat)):
        stat.put()
        self.written += 1



    if self.found_non_empty_namespace:
      for stat in self.namespace_stats.itervalues():
        if stat.count or not isinstance(stat, stats.NamespaceGlobalStat):
          stat.put()
          self.written += 1

  def Run(self):
    """Scans the datastore, computes new stats and writes them."""
    self.__ScanAllNamespaces()
    self.__Finalize()
    return self

  def Report(self):
    """Produce a small report about the result."""
    stat = self.whole_app_stats.get(_GLOBAL_KEY, None)
    entity_size = 0
    entity_count = 0
    builtin_index_size = 0
    builtin_index_count = 0
    composite_index_size = 0
    composite_index_count = 0
    if stat:
      entity_size = stat.entity_bytes
      entity_count = stat.count
      builtin_index_size = stat.builtin_index_bytes
      builtin_index_count = stat.builtin_index_count
      composite_index_size = stat.composite_index_bytes
      composite_index_count = stat.composite_index_count

      if not entity_count:
        entity_count = 1

    return ('Scanned %d entities of total %d bytes, %d index entries of total '
            '%d bytes and %d composite index entries of total %d bytes. '
            'Inserted %d new records.'
            % (entity_count, entity_size, builtin_index_count,
               builtin_index_size, composite_index_count, composite_index_size,
               self.written))
