class Stream(DeletedMarkerModel):
  """
  extra:  see api.stream_create()
  """
  owner = models.StringProperty()     # ref
  title = models.StringProperty()
  type = models.StringProperty()
  slug = models.StringProperty()
  read = models.IntegerProperty()     # TODO: document this
  write = models.IntegerProperty()
  extra = properties.DictProperty()

  key_template = 'stream/%(owner)s/%(slug)s'

  def is_public(self):
    return self.read == PRIVACY_PUBLIC

  def is_restricted(self):
    return self.read == PRIVACY_CONTACTS

  def keyname(self):
    """Returns the key name"""
    return self.key().name() class StreamEntry(DeletedMarkerModel):
  """
  extra :
    title - 
    location - 
    icon - 
    content - 
    entry_stream - 
    entry_stream_type - 
    entry_title - 
    entry_uuid - 
    comment_count - 
  """
  stream = models.StringProperty()    # ref - the stream this belongs to
  owner = models.StringProperty()     # ref - the actor who owns the stream
  actor = models.StringProperty()     # ref - the actor who wrote this
  entry = models.StringProperty()     # ref - the parent of this,
                                      #       should it be a comment
  uuid = models.StringProperty()
  created_at = properties.DateTimeProperty(auto_now_add=True)
  extra = properties.DictProperty()

  key_template = '%(stream)s/%(uuid)s'

  def url(self, with_anchor=True, request=None, mobile=False):
    if self.entry:
      # TODO bad?
      slug = self.entry.split("/")[-1]
      anchor = "#c-%s" % self.uuid
    else:
      # TODO(termie): add slug property
      slug = self.uuid
      anchor = ""
    path = "/%s/%s" % ('presence', slug)
    if with_anchor:
      path = "%s%s" % (path, anchor)
    return actor_url(_get_actor_urlnick_from_nick(self.owner),
                     _get_actor_type_from_nick(self.owner),
                     path=path,
                     request=request,
                     mobile=mobile)

  def keyname(self):
    """Returns the key name"""
    return self.key().name()

  def title(self):
    """ build a title for this entry, for a presence entry it will just be
    the title, but for a comment it will look like:

    Comment from [commenter nick] on [entry title] by [nick]

    Comment from [commenter nick] on [entry title] by [nick] to #[channel name]
    """

    if not self.is_comment():
      return self.extra.get('title')

    template = "Comment from %(actor)s on %(entry_title)s by %(entry_actor)s"
    actor = _get_actor_urlnick_from_nick(self.actor)
    entry_title = self.extra.get('entry_title')
    entry_actor = _get_actor_urlnick_from_nick(self.extra.get('entry_actor'))
    entry_owner_nick = util.get_user_from_topic(self.entry)
    entry_type = _get_actor_type_from_nick(entry_owner_nick)

    v = {'actor': actor,
         'entry_title': entry_title,
         'entry_actor': entry_actor,
         }

    if entry_type == 'channel':
      template += ' to #%(channel)s'
      channel = _get_actor_urlnick_from_nick(entry_owner_nick)
      v['channel'] = channel

    return template % v

  def is_comment(self):
    return (self.entry != None)

  def is_channel(self):
    return self.owner.startswith('#')

  def entry_actor(self):
    if self.entry:
      return util.get_user_from_topic(self.entry)
    return None class InboxEntry(CachingModel):
  """This is the inbox index for an entry.

  the index allows us to quickly pull the overview for a user. There may be
  items in the results that are later filtered out - deleted items or items
  whose privacy has changed.
  """
  inbox = models.StringListProperty()   # ref - who this is the inbox for
  stream = models.StringProperty()      # ref - the stream this belongs to
  stream_type = models.StringProperty() # separate because we may filter on it
  entry = models.StringProperty()       # ref - the entry if this is a comment
  created_at = properties.DateTimeProperty()
  uuid = models.StringProperty()
  shard = models.StringProperty()       # an identifier for this portion of
                                        # inboxes

  key_template = 'inboxentry/%(stream)s/%(uuid)s/%(shard)s'

  def stream_entry_keyname(self):
    """Returns the key name of the corresponding StreamEntry"""
    return "%s/%s" % (self.stream, self.uuid) class Actor(DeletedMarkerModel):
  """
  extra:
    contact_count - int; number of contacts
    follower_count - int; number of followers
    icon - string; avatar path
    bg_image - string; image for background (takes precedence over bg_color)
    bg_color - string; color for background
    bg_repeat - whether to repeat bg_image
    description [channel] - string; Channel description
    external_url [channel] - string; External url related ot channel
    member_count [channel] - int; number of members
    admin_count [channel] - int; number of admins
    email_notify [user] - boolean; does the user want email notifications?
    given_name [user] - string; First name
    family_name [user] - string; Last Name
    comments_hide [user] - boolean; Whether comments should be hidden on 
                             overview
  """
  nick = models.StringProperty()
  # the appengine datastore is case-sensitive whereas human brains are not, 
  # Paul is not different from paul to regular people so we need a way to 
  # prevent duplicate names from cropping up, this adds an additional indexed 
  # property to support that
  normalized_nick = models.StringProperty()
  password = models.StringProperty()
  privacy = models.IntegerProperty()
  type = models.StringProperty()
  extra = properties.DictProperty()
  # avatar_updated_at is used by DJabberd to get a list of changed avatar. We
  # set the default to a date before the launch so that initial avatars have an
  # updated_at that is less than any real changes.
  avatar_updated_at = properties.DateTimeProperty(
      default=datetime.datetime(2009, 01, 01))

  key_template = 'actor/%(nick)s'

  def url(self, path="", request=None, mobile=False):
    """ returns a url, with optional path appended
    
    NOTE: if appending a path, it should start with '/'
    """
    return actor_url(_get_actor_urlnick_from_nick(self.nick),
                     self.type,
                     path=path,
                     request=request,
                     mobile=mobile)

  def shortnick(self):
    return _get_actor_urlnick_from_nick(self.nick)

  def display_nick(self):
    return self.nick.split("@")[0]
    return _get_actor_urlnick_from_nick(self.nick)

  def to_api(self):
    rv = super(Actor, self).to_api()
    del rv['password']
    del rv['normalized_nick']
    extra = {}
    for k, v in rv['extra'].iteritems():
      if k in ACTOR_ALLOWED_EXTRA:
        extra[k] = v
    rv['extra'] = extra
    return rv

  def to_api_limited(self):
    rv = self.to_api()
    extra = {}
    for k, v in rv['extra'].iteritems():
      if k in ACTOR_LIMITED_EXTRA:
        extra[k] = v
    rv['extra'] = extra
    return rv

  def is_channel(self):
    return self.type == 'channel'

  def is_public(self):
    return self.privacy == PRIVACY_PUBLIC

  def is_restricted(self):
    return self.privacy == PRIVACY_CONTACTS

  def __repr__(self):
    # Get all properties, but not directly as property objects, because
    # constructor requires values to be passed in.
    d = dict([(k, self.__getattribute__(k)) for k in self.properties().keys()])
    return "%s(**%s)" % (self.__class__.__name__, repr(d)) class Relation(CachingModel):
  owner = models.StringProperty()     # ref - actor nick
  relation = models.StringProperty()  # what type of relationship this is
  target = models.StringProperty()    # ref - actor nick

  key_template = 'relation/%(relation)s/%(owner)s/%(target)s' class Subscription(CachingModel):
  """this represents a topic, usually a stream, that a subscriber
     (usually an inbox) would like to receive updates to
  """
  topic = models.StringProperty()       # ref - the stream being subscribed to
  subscriber = models.StringProperty()  # ref - the subscriber (actor)
  target = models.StringProperty()      # where to dump this
  state = models.StringProperty() # The status of remote subs, see XEP-0060
                                  # sect 4.2. The 'pending' state is ignored if
                                  # the target of the subscription is used.
                                  # The design is for performance: on public 
                                  # entries
                                  # the state is ignored and listing the
                                  # subscriptions is a single query; for
                                  # contacts-only entries the state is used but
                                  # it is also kept up-to-date regarding buddy
                                  # relationships, so a single query for
                                  # state='subscribed' can again be used.
  extra = properties.DictProperty()     # holds a bunch of stuff
  created_at = properties.DateTimeProperty(auto_now_add=True) 
                                  # for ordering someday
  key_template = '%(topic)s/%(target)s'

  def is_subscribed(self):
    # LEGACY COMPAT: the 'or' here is for legacy compat
    return (self.state == 'subscribed' or self.state == None) class Invite(CachingModel):
  code = models.StringProperty() # the code for the invite
  email = models.StringProperty() # the email this invite went to
  to_actor = models.StringProperty() # ref - the actor this invite was sent to
  from_actor = models.StringProperty() # ref - who sent this invite
  for_actor = models.StringProperty() # ref - invited to what, probs a channel
  status = models.StringProperty(default="active") # enum - active, blocked

  key_template = 'invite/%(code)s' class OAuthConsumer(CachingModel):
  key_ = models.StringProperty()      # the consumer key
  secret = models.StringProperty()    # the consumer secret
  actor = models.StringProperty()     # the actor who owns this
  status = models.StringProperty()    # active / pending / inactive
  type = models.StringProperty()      # web / desktop / mobile
  commercial = models.IntegerProperty()   # is this a commercial key?
  app_name = models.StringProperty()  # the name of the app this is for,
                                      # to be displayed to the user
  created_at = properties.DateTimeProperty(auto_now_add=True)
  
  key_template = 'oauth/consumer/%(key_)s'
  
  def url(self):
    return '/api/keys/%s' % self.key_ class OAuthRequestToken(CachingModel):
  key_ = models.StringProperty()      # the token key
  secret = models.StringProperty()    # the token secret
  consumer = models.StringProperty()  # the consumer this key is assigned to
  actor = models.StringProperty()     # the actor this key authenticates for
  authorized = models.IntegerProperty()   # has the actor authorized this token?
  created_at = properties.DateTimeProperty(auto_now_add=True)     
                                      # when this was created
  perms = models.StringProperty()     # read / write / delete

  key_template = 'oauth/requesttoken/%(key_)s'

  def to_string(self):
    token = oauth.OAuthToken(self.key_, self.secret)
    return token.to_string() class OAuthAccessToken(CachingModel):
  key_ = models.StringProperty()      # the token key
  secret = models.StringProperty()    # the token secret
  consumer = models.StringProperty()  # the consumer this key is assigned to
  actor = models.StringProperty()     # the actor this key authenticates for
  created_at = properties.DateTimeProperty(auto_now_add=True)     
                                      # when this was created
  perms = models.StringProperty()     # read / write / delete

  key_template = 'oauth/accesstoken/%(key_)s'

  def to_string(self):
    token = oauth.OAuthToken(self.key_, self.secret)
    return token.to_string() class Image(CachingModel):
  actor = models.StringProperty()     # whose image is this?
  content = models.BlobProperty()     # the image itself
  size = models.StringProperty()      # see api.avatar_upload

  # TODO(termie): key_template plans don't really work very well here
  #               because we haven't been storing the path :/ class Activation(CachingModel):
  actor = models.StringProperty()
  content = models.StringProperty()
  code = models.StringProperty()
  type = models.StringProperty()

  key_template = 'activation/%(actor)s/%(type)s/%(content)s' class KeyValue(CachingModel):
  actor = models.StringProperty()
  keyname = models.StringProperty()
  value = models.TextProperty()

  key_template = 'keyvalue/%(actor)s/%(keyname)s' class Presence(CachingModel):
  """This represents all the presence data for an actor at a moment in time.
  extra:
    status - string; message (like an "away message")
    location - string; TODO(tyler): Consider gps / cell / structured data
    availability - string; TODO(tyler): Define structure
  """
  actor = models.StringProperty()     # The actor whose presence this is
  updated_at = properties.DateTimeProperty(auto_now_add=True)
                                      # The moment we got the update
  uuid = models.StringProperty()
  extra = properties.DictProperty()   # All the rich presence

  # TODO(termie): can't do key_template here yet because we include 
  #               current and history keys :/ class AbuseReport(CachingModel):
  entry = models.StringProperty()  # ref - entry
  actor = models.StringProperty()  # ref - actor for entry
  reports = models.StringListProperty() # the actors who have reported this
  count = models.IntegerProperty() # the count of the number of reports so far
  
  key_template = '%(entry)s' class Task(CachingModel):
  actor = models.StringProperty()     # ref - the owner of this queue item
  action = models.StringProperty()    # api call we are iterating through
  action_id = models.StringProperty() # unique identifier for this queue item
  args = models.StringListProperty()  # *args
  kw = properties.DictProperty()      # *kw
  expire = properties.DateTimeProperty()
                                      # when our lock will expire
  progress = models.StringProperty()  # a string representing the offset to 
                                      # which we've progressed so far
  created_at = properties.DateTimeProperty(auto_now_add=True)
  
  key_template = 'task/%(actor)s/%(action)s/%(action_id)s' PRIVACY_PRIVATE = 1 PRIVACY_CONTACTS = 2 PRIVACY_PUBLIC = 3 def is_allowed_to_send_email_to(email):
def _greeting_name(actor_ref):
def _full_name(actor_ref):
def log_blocked_send(on_behalf, email, subject, message=None):
def send(to_email, subject, message, on_behalf=None, html_message=None):
def filter_out_blocked_addresses(message_tuples):
def mass_send(message_tuples):
def email_comment_notification(actor_to_ref, actor_from_ref, 
                               comment_ref, entry_ref):
def email_confirmation_message(actor, activation_code):
def email_invite(from_actor_ref, invite_code):
def email_new_follower(owner_ref, target_ref):
def email_new_follower_mutual(owner_ref, target_ref):
def email_lost_password(actor, email, code): PRIVACY_PRIVATE = 1
PRIVACY_CONTACTS = 2
PRIVACY_PUBLIC = 3
ACTOR_ALLOWED_EXTRA = ('contact_count', 
                       'follower_count', 
                       'icon', 
                       'description',
                       'member_count', 
                       'admin_count', 
                       'given_name', 
                       'family_name'
                       )
ACTOR_LIMITED_EXTRA = ('icon', 
                       'description',
                       'given_name', 
                       'family_name'
                       )
def _get_actor_type_from_nick(nick):
def _get_actor_urlnick_from_nick(nick):
def _to_api(v):
  def to_api(self):
  def __init__(self, parent=None, key_name=None, _app=None, **kw):
  def key_from(cls, **kw):
  def _remove_from_cache(self):
  def put(self):
  def save(self):
  def delete(self):
  def get_by_key_name(cls, key_names, parent=None):
  def db_get_count(cls):
  def reset_cache(cls):
  def enable_cache(cls, enabled = True):
  def reset_get_count(cls):
  def gql(cls, *args, **kw):
  def Query(cls):
  def mark_as_deleted(self):
  def is_deleted(self):
def actor_url(nick, actor_type, path='', request=None, mobile=False):
  def url(self, path="", request=None, mobile=False):
  def shortnick(self):
  def display_nick(self):
  def to_api(self):
  def to_api_limited(self):
  def is_channel(self):
  def is_public(self):
  def is_restricted(self):
  def __repr__(self):
  def stream_entry_keyname(self):
  def to_string(self):
  def url(self):
  def to_string(self):
  def is_public(self):
  def is_restricted(self):
  def keyname(self):
  def url(self, with_anchor=True, request=None, mobile=False):
  def keyname(self):
  def title(self):
  def is_comment(self):
  def is_channel(self):
  def entry_actor(self):
  def is_subscribed(self):
class ApiMixinModel(aed_models.BaseModel):
class CachingModel(ApiMixinModel):
class DeletedMarkerModel(CachingModel):
class AbuseReport(CachingModel):
class Activation(CachingModel):
class Actor(DeletedMarkerModel):
class Image(CachingModel):
class InboxEntry(CachingModel):
class Invite(CachingModel):
class KeyValue(CachingModel):
class OAuthAccessToken(CachingModel):
class OAuthConsumer(CachingModel):
class OAuthNonce(CachingModel):
class OAuthRequestToken(CachingModel):
class Presence(CachingModel):
class Task(CachingModel):
class Relation(CachingModel):
class Stream(DeletedMarkerModel):
class StreamEntry(DeletedMarkerModel):
class Subscription(CachingModel): class JID(object):
class XmppMessage(object):
class XmppConnection(base.Connection):
  def __init__(self, node, host, resource=None):
  def from_uri(cls, uri):
  def base(self):
  def full(self):
  def __init__(self, sender, target, message):
  def from_request(cls, request):
  def send_message(self, to_jid_list, message, html_message=None, 
                   atom_message=None):
import random
import re
import datetime
import logging
from cleanliness import cleaner
from django import template
from django.conf import settings
from google.appengine.ext import db
from google.appengine.api import urlfetch
from google.appengine.api import images
from common.models import Stream, StreamEntry, InboxEntry, Actor, Relation
from common.models import Subscription, Invite, OAuthConsumer, OAuthRequestToken
from common.models import OAuthAccessToken, Image, Activation
from common.models import KeyValue, Presence
from common.models import AbuseReport
from common.models import Task
from common.models import PRIVACY_PRIVATE, PRIVACY_CONTACTS, PRIVACY_PUBLIC
from common import clean
from common import clock
from common import context_processors
from common import exception
from common import imageutil
from common import mail
from common import memcache
from common import models
from common import normalize
from common import patterns
from common import properties
from common import throttle
from common import util
from common import validate
from common.protocol import sms
from common.protocol import xmpp
    raise exception.ApiNotFound(
        'Could not find task: %s %s %s' % (nick, action, action_id))

  locked = memcache.client.add(key_name, 'owned', time=expire)
  if not locked: 
    raise exception.ApiLocked("Lock could not be acquired: %s" % key_name)

  return q

@owner_required
def task_get_or_create(api_user, nick, action, action_id, args=None, 
                       kw=None, progress=None, expire=DEFAULT_TASK_EXPIRE):
  try:
    task_ref = task_get(api_user, nick, action, action_id, expire)
  except exception.ApiNotFound:
    task_ref = task_create(api_user, 
                           nick, 
                           action, 
                           action_id, 
                           args, 
                           kw, 
                           progress, 
                           expire=expire)
  return task_ref
    

#@throttled(minute=30, hour=1200, day=4000, month=20000)
@owner_required
def task_process_actor(api_user, nick):
  nick = clean.nick(nick)
  return task_process_any(ROOT, nick)
 
@admin_required   
def task_process_any(api_user, nick=None):
  # Basing this code largely off of pubsubhubbub's queueing approach
  # Hard-coded for now, these will get moved up
  work_count = 1
  sample_ratio = 10
  lock_ratio = 4
  lease_period = DEFAULT_TASK_EXPIRE
  sample_size = work_count * sample_ratio

  if nick:
    query = Task.gql('WHERE actor = :1 ORDER BY created_at',
                     nick)
  else:
    query = Task.gql('ORDER BY created_at')
  
  work_to_do = query.fetch(sample_size)
  if not work_to_do:
    raise exception.ApiNoTasks('No tasks')

  # From pubsububhub:
  # Attempt to lock more work than we actually need to do, since there likely
  # will be conflicts if the number of workers is high or the work_count is
  # high. If we've acquired more than we can use, we'll just delete the memcache
  # key and unlock the work. This is much better than an iterative solution,
  # since a single locking API call per worker reduces the locking window.
  possible_work = random.sample(work_to_do,
                                min(len(work_to_do), 
                                    lock_ratio * work_count)
                                )
  work_map = dict([(str(w.key().name()), w) for w in possible_work])
  try_lock_map = dict((k, 'owned') for k in work_map)
  not_set_keys = set(memcache.client.add_multi(try_lock_map, time=lease_period))
  if len(not_set_keys) == len(try_lock_map):
    return
  
  locked_keys = [k for k in work_map if k not in not_set_keys]
  reset_keys = locked_keys[work_count:]
  if reset_keys and not memcache.client.delete_multi(reset_keys):
    logging.warning('Could not reset acquired work for model %s: %s',
                    'Task', reset_keys)

  work = [work_map[k] for k in locked_keys[:work_count]]
  
  for task_ref in work:
    logging.info("Processing task: %s %s %s p=%s", 
                  task_ref.actor,
                  task_ref.action, 
                  task_ref.action_id,
                  task_ref.progress
                  )

    try:
      actor_ref = actor_get(ROOT, task_ref.actor)

      method_ref = PublicApi.get_method(task_ref.action)

      rv = method_ref(actor_ref, 
                      _task_ref = task_ref, 
                      *task_ref.args, 
                      **task_ref.kw)

    except exception.ApiDeleted:
      logging.warning('Owner or target of task has been deleted. Removing task.')
      task_ref.delete()
      return

  return
  

@owner_required
def task_remove(api_user, nick, action, action_id):
  key_name = Task.key_from(actor=nick, action=action, action_id=action_id)
  
  q = Task.get_by_key_name(key_name)
  if not q:
    raise exception.ApiNotFound(
        'Could not find task: %s %s %s' % (nick, action, action_id))

  # clear up any residual locks
  q.delete()
  memcache.client.delete(key_name)

  return True

@owner_required
def task_update(api_user, nick, action, action_id, progress=None, unlock=True):
  """ update the progress for a task and possibly unlock it """

  key_name = Task.key_from(actor=nick, action=action, action_id=action_id)