class IntakeUser(ndb.Expando):
    org = ndb.KeyProperty(kind=Organization)
    developer = ndb.KeyProperty(kind=Developer)
    created = ndb.DateTimeProperty(auto_now=True)
    updated = ndb.DateTimeProperty(auto_now_add=True)
    api_type = ndb.StringProperty()
    
    user_id = ndb.StringProperty()
    user_id_enc = ndb.StringProperty()

    # PII
    name = ndb.StringProperty()
    address = ndb.StringProperty()
    ssn = ndb.StringProperty()
    phone = ndb.StringProperty()
    email = ndb.StringProperty()    
    facebook_id = ndb.StringProperty()
    twitter_id = ndb.StringProperty()
    linkedin_id = ndb.StringProperty()
    drivers_lic = ndb.StringProperty()
    
    name_enc = ndb.StringProperty()
    address_enc = ndb.StringProperty()
    ssn_enc = ndb.StringProperty()
    phone_enc = ndb.StringProperty()
    email_enc = ndb.StringProperty()    
    facebook_id_enc = ndb.StringProperty()
    twitter_id_enc = ndb.StringProperty()
    linkedin_id_enc = ndb.StringProperty()
    drivers_lic_enc = ndb.StringProperty()
    
    # Reputation
    transaction_count = ndb.IntegerProperty()
    review_count = ndb.IntegerProperty()
    positive_review_percentage = ndb.FloatProperty()
    negative_events = ndb.StringProperty() # This should likely be a structured / repeated property
    date_joined = ndb.DateProperty()
    date_banned = ndb.DateProperty()
    reason_banned = ndb.StringProperty()
    
    def pii_dict(self):
        self_dict = self.to_dict()
        for k in self_dict.keys():
            if k not in PII_FIELDS:
                del self_dict[k]

        return self_dict
    
    @classmethod
    def create_or_update(cls, fields, dev_key, org_key, api_type,
                         existing_user=None, pre_hashed=False):
        """
        Creates an IntakeUser model from the given data or updates 
        an existing model.
        
        Note that this method DOES persist the resulting model to the database.
        
        Args:
            fields: dictionary of field -> value
            org_key: entity key for the org this user belongs to
            dev_key: entity key for the dev who uploaded this user
            existing_model: an existing intake user model to be updated
            pre_hashed: bool indicating whether the PII has already been hashed.
            db_key: key of the database entity to use as an ancestor for this 
                    user.
            
        Returns:
            An intake user model that has been persisted to the DB.
        """
        if existing_user:
            intake_user = existing_user
        else:
            intake_user = IntakeUser()

        intake_user.org = org_key
        intake_user.developer = dev_key
        intake_user.api_type = api_type

        values_to_set = {}           
        for field, value in fields.items():
            # Hash and encrypt the PII data     
            if field in (('user_id',) + PII_FIELDS):
                # Only save an encrypted version if we got raw data, it's 
                # a bit silly to save a copy of the raw pre-hashed data.
                
                if ((isinstance(value, str) or isinstance(value, unicode))
                    and not value.strip()):
                    # don't even save pure whitespace PII
                    continue
                    
                if not pre_hashed:
                    values_to_set[field+"_enc"] = encrypt_value(value)
                value = hash_value(value, pre_hashed=pre_hashed)
                         
            values_to_set[field] = value

        for key, value in values_to_set.items():
            setattr(intake_user, key, value)
        
        intake_user.put()
        
        return intake_user class CombinedUser(ndb.Model):
    intake_users = ndb.KeyProperty(kind=IntakeUser, repeated=True)
    orgs = ndb.KeyProperty(kind=Organization, repeated=True)
    
    user_id = ndb.StringProperty(repeated=True)
    name = ndb.StringProperty(repeated=True)
    address = ndb.StringProperty(repeated=True)
    ssn = ndb.StringProperty(repeated=True)
    phone = ndb.StringProperty(repeated=True)
    email = ndb.StringProperty(repeated=True)    
    facebook_id = ndb.StringProperty(repeated=True)
    twitter_id = ndb.StringProperty(repeated=True)
    linkedin_id = ndb.StringProperty(repeated=True)
    drivers_lic = ndb.StringProperty(repeated=True)
    
    legit_report = ndb.JsonProperty()
    
    @classmethod
    def create(cls, intake_users, legit_report):
        cu = cls()
        
        for iu in intake_users:
            org_name = iu.org.get().name
            if iu.key not in cu.intake_users:
                cu.intake_users.append(iu.key)
                for attr in PII_FIELDS:
                    iu_attr = getattr(iu, attr)
                    cu_attrs = getattr(cu, attr) or []
                    if iu_attr and iu_attr not in cu_attrs:                        
                        logging.info("COMBINED USER ATT: %s" % cu_attrs)
                        logging.info("INTAKE USER ATT: %s" % iu_attr)
                        cu_attrs.append(iu_attr)
                        setattr(cu,attr,cu_attrs)
                
                if iu.org not in cu.orgs:
                    cu.orgs.append(iu.org)
                    
                user_id = "%s::%s" % (org_name, iu.user_id)
                if user_id not in cu.user_id:
                    cu.user_id.append(user_id)
       
        cu.legit_report = legit_report        
        
        return cu
    
    # def merge(self, to_merge):
    #     for attr in ('intake_users', ) + PII_FIELDS:
    #         combined_attr = list(set(getattr(self, attr) + getattr(to_merge, attr)))
    #         setattr(self,attr,combined_attrs)
    #         
    # def add_intake(self, intake_user):
    #     if intake_user.key not in self.intake_users:
    #         self.intake_users.append(intake_user.key)
    #     for attr in PII_FIELDS:
    #         value = getattr(intake_user, attr)
    #         if value and value not in getattr(self, attr):
    #             getattr(self, attr).append(value) class Organization(ndb.Model):
    name = ndb.StringProperty()
    full_name = ndb.StringProperty()
    is_intake_analyzed = ndb.BooleanProperty()
    org_type = ndb.StringProperty()
    credit = ndb.FloatProperty(default=0) class Invite(ndb.Model):
    code = ndb.StringProperty()
    org = ndb.KeyProperty(kind=Organization)
    is_used = ndb.BooleanProperty(default=False) class Developer(auth_models.User):
    name = ndb.StringProperty()
    email = ndb.StringProperty()
    org = ndb.KeyProperty(kind=Organization)
    invite_code = ndb.StringProperty()
    consumer_key = ndb.StringProperty()
    consumer_secret = ndb.StringProperty()
    permissions = ndb.StringProperty(repeated=True)    
    stripe_customer_ids = ndb.StringProperty(repeated=True)
    auto_recharge = ndb.BooleanProperty()
    auto_recharge_min = ndb.IntegerProperty()
    auto_recharge_incr = ndb.IntegerProperty()
    auto_recharge_custid = ndb.StringProperty()
    
    is_admin = ndb.BooleanProperty(default=False) class Price(ndb.Model):
    name = ndb.StringProperty()
    price = ndb.FloatProperty(default=0) class RequestLog(ndb.Model):
    created = ndb.DateTimeProperty(auto_now=True)
    request_url = ndb.StringProperty()
    request_resource = ndb.StringProperty()
    request_outh_parameters = ndb.JsonProperty()
    request_parameters = ndb.JsonProperty()  
    request_method = ndb.StringProperty()      
    developer = ndb.KeyProperty(kind=Developer)
    org = ndb.KeyProperty(kind=Organization)    
    price = ndb.FloatProperty()
    actually_charged = ndb.FloatProperty()
    response_code = ndb.IntegerProperty()
    response_text = ndb.TextProperty()
    api_version = ndb.TextProperty()

    @property
    def response_json(self):
        try:
            return json.loads(self.response_text)            
        except:
            return None PII_FIELDS = ("name", "address", "ssn", "phone", "email",
              "facebook_id", "twitter_id", "linkedin_id",
              "drivers_lic") def hash_value(value, salt=None, pre_hashed=False):
    """
    De-identifies a value by one-way hashing it, for storage in our database.
    
    To be compatible with people who pre-hash data for us, if it hasn't been
    done already, we first hash the data using our public hash algo/salt. The
    data is then hashed again for storage/queries, using our internal, secret
    salt.  
    """
    if not pre_hashed:
        value = public_hash_value(value)

    if not salt:
        salt = _get_salt()        
    
    return _hash_value(value, salt) def decrypt_value(encrypted_value, aes_key=None):
    """Decrypts the given value using the provided private key
    
    Args:
        encrypted_value: pickled tuple of encrypted data
        private_key: PEM format private RSA key
    Returns:
        plaintext version of the value
    """
    if not aes_key:
        aes_key = _get_aes_key()
        
    encrypted_value = base64.b64decode(encrypted_value)
    iv = encrypted_value[:AES.block_size]
    encrypted_value = encrypted_value[AES.block_size:]
    aes_decrypt = AES.new(aes_key, AES.MODE_CFB, iv)
    
    return aes_decrypt.decrypt(encrypted_value) class PIIForm(Form):
    def __init__(self, *args, **kwargs):
        self._pre_hashed = kwargs.pop("pre_hashed", False)
        super(PIIForm, self).__init__(*args, **kwargs)
          
    name = PIIField(u'Name', 
                    [prehash_validator(validation.validate_name)])
    address = PIIField(u'Address',
                    [prehash_validator(validation.validate_address)])
    ssn = PIIField(u'Social Security Number',
                    [prehash_validator(validation.validate_ssn)])
    phone = PIIField(u'Phone Number',
                    [prehash_validator(validation.validate_phone)])
    email = PIIField(u'Email',
                    [prehash_validator(validation.validate_address)])
    facebook_id = PIIField(u'Facebook ID',
                    [prehash_validator(validation.validate_facebook_id)])
    twitter_id = PIIField(u'Twitter ID',
                    [prehash_validator(validation.validate_twitter_id)])
    linkedin_id = PIIField(u'Linkedin ID',
                    [prehash_validator(validation.validate_linkedin_id)]) def find_users(pii_fields, api_type, org_key=None, pre_hashed=False):
    """
    Method for finding users matching the given PII. Adds more
    intelligence to the process through the following:
    
        Isolating "user sets", non-intersecting groups of users tied 
        to different fields of the given PII. This could be the innocent 
        result of sparse information, an indicator of fraud, or a sign that
        the given PII belongs to multiple individuals. 
    
        Idenfitying conflicting PII. These are PII values within a set 
        of supposedly coherent users that does not agree. This could be 
        the result of user using different contact information with 
        different organizations, or possibly a sign of fraud.
    
    Args:
        pii_fields: A dictionary of pii_field -> value
        
    Returns:
        A list of user sets. Each user set is a list of lists of 
        MatchingIntakeUsers, each sub-list representing a level of matching
        directness.
    """
    #logging.info("Find Users for: %s" % pii_fields)
    
    hashed_pii = {}
    for key, value in pii_fields.items():
        if value:
            hashed_pii[key] = hash_value(value) if not pre_hashed else value
    
    combined_pii = defaultdict(set)
    user_sets = []
    for field, value in hashed_pii.items():
        if field in combined_pii and value in combined_pii[field]:
            # If we've run across this pii value in a previous user set, 
            # we don't need to search on it again.
            continue
            
        matching_users = find_matching_users({field:[value]}, 
                                              api_type, 
                                              org_key=org_key)
        
        # Get rid of the dictionary part, we don't need it anymore.
        #logging.info("Matching Users: %s" % matching_users)
        user_set = [match_dict.values() for match_dict in matching_users]
        user_sets.append(user_set)        
                
        user_set_pii = aggregate_pii([miu.user for match_level in user_set 
                                                for miu in match_level])
                                                                            
        for k,v in user_set_pii.items():
            combined_pii[k].update(v)
            
    return user_sets def generate_legit_report(components, query_data, api_type, 
                          org_key=None, pre_hashed=False):
    """
    The big mama jama. Pulls together all our sources of data into a
    LegitReport as well as computes a LegitScore
    
    The components piece determines which pieces of data are assembled into the
    report. So far the supported components are:
        blacklist - checkes the given query data against the LRG blacklist.
    """    
    ### LRG Summary ###
    user_sets = find_users(query_data, 
                           api_type, 
                           org_key=org_key, 
                           pre_hashed=pre_hashed)
    
    #logging.info("Generate Legit Report -- User Sets Found: %d" % len(user_sets))
    
    lrg_data = generate_lrg_summary(user_sets)
    intake_users = intake_users_from_user_sets(user_sets)
    agg_pii = aggregate_pii(intake_users)
    
    legit_report = {}
    legit_score = 50

    # Create the specified report components
    if "blacklist" in components:
        blacklist_report = {}
        blacklistings = lrg_data["permanent_removals"] if lrg_data else []
        blacklist_report["blacklistings"] = blacklistings
        blacklist_report["blacklisting_count"] = len(blacklistings)
        
        legit_score -= (40 * blacklist_report["blacklisting_count"])                
        legit_report["blacklist"] = blacklist_report
    
    if "reputation" in components:
        legit_report["lrg_reputation"] = lrg_data
    
    # Limit the legitscore and add it to the report
    legit_score = max([0, min([legit_score, 100])]) # Force 0-100    
    legit_report["legit_score"] = legit_score
    
    return legit_report, intake_users def generate_lrg_summary(user_sets, excluded_orgs=None):
    """
    Compiles the report for a set of intake users who we believe
    to all represent the same user, spread across different systems.
    """
    #logging.info("Generating LRG Summary for: %s" % query_data)    
    #logging.info("Found %d User Sets." % len(user_sets))
    #logging.info("User Sets: %s" % user_sets)
    #logging.info("Excluded Orgs: %s" % excluded_orgs)
    
    intake_users = intake_users_from_user_sets(user_sets, 
                                                excluded_orgs=excluded_orgs)
     
    #logging.info("Non-excluded Users: %s" % intake_users) 
                
    if not intake_users:
        return None
    
    users_by_category = defaultdict(list)
    for iu in intake_users:
        users_by_category[iu.org.get().org_type].append(iu)
    
    overall_summary = lrg_summary(intake_users)
    
    lrg_data = {}
    lrg_data["marketplace_memberships"] = overall_summary.memberships
    lrg_data["total_transactions"] = overall_summary.transactions
    lrg_data["total_reviews"] = overall_summary.reviews
    lrg_data["overall_positive_feedback"] = overall_summary.pos_review_perct
    lrg_data["permanent_removals"] = overall_summary.banned_dates
    lrg_data["categories"] = {}
    
    for k,v in users_by_category.items():
        category_summary = lrg_summary(v)
        for bd in category_summary.banned_dates:
            bd.pop("market_type","")                
        lrg_data["categories"][k] = {
            "marketplace_memberships": category_summary.memberships,
            "transactions": category_summary.transactions,
            "reviews": category_summary.reviews,
            "positive_feedback": category_summary.pos_review_perct,
            "permanent_removals": category_summary.banned_dates,
        }
    
    return lrg_data def aggregate_pii(intake_users):
    """
    Creates a superset of all the PII across all the given intake users.
    
    Args:
        intake_users: A list of IntakeUser objects
        
    Returns:
        A default dictionary of pii_fields -> (set of values)
    """
    aggregate_pii = defaultdict(set)
    for intake_user in intake_users:
        for field in PII_FIELDS:
            if getattr(intake_user, field, None):
                aggregate_pii[field].add(getattr(intake_user, field))
        
    return aggregate_pii def generate_legit_report(components, query_data, api_type, 
                          org_key=None, pre_hashed=False):
    """
    The big mama jama. Pulls together all our sources of data into a
    LegitReport as well as computes a LegitScore
    
    The components piece determines which pieces of data are assembled into the
    report. So far the supported components are:
        blacklist - checkes the given query data against the LRG blacklist.
    """    
    ### LRG Summary ###
    user_sets = find_users(query_data, 
                           api_type, 
                           org_key=org_key, 
                           pre_hashed=pre_hashed)
    
    #logging.info("Generate Legit Report -- User Sets Found: %d" % len(user_sets))
    
    lrg_data = generate_lrg_summary(user_sets)
    intake_users = intake_users_from_user_sets(user_sets)
    agg_pii = aggregate_pii(intake_users)
    
    legit_report = {}
    legit_score = 50

    # Create the specified report components
    if "blacklist" in components:
        blacklist_report = {}
        blacklistings = lrg_data["permanent_removals"] if lrg_data else []
        blacklist_report["blacklistings"] = blacklistings
        blacklist_report["blacklisting_count"] = len(blacklistings)
        
        legit_score -= (40 * blacklist_report["blacklisting_count"])                
        legit_report["blacklist"] = blacklist_report
    
    if "reputation" in components:
        legit_report["lrg_reputation"] = lrg_data
    
    # Limit the legitscore and add it to the report
    legit_score = max([0, min([legit_score, 100])]) # Force 0-100    
    legit_report["legit_score"] = legit_score
    
    return legit_report, intake_users
import logging
import json
import re
import os
    import hashlib
from collections import OrderedDict, defaultdict
from google.appengine.ext import ndb
from flask import Flask, render_template, request, redirect, url_for, flash, jsonify, Response
from wtforms import Form, TextField, BooleanField, FloatField, PasswordField, FileField, validators
from webapp2_extras import security
from google.appengine.api import taskqueue, memcache, files, urlfetch
from models import IntakeUser, CombinedUser, Organization, Invite, Developer, Price, RequestLog, PII_FIELDS
from cryptography import hash_value, decrypt_value
from api import PIIForm, find_users, generate_legit_report, generate_lrg_summary
from mapreduce import base_handler, mapreduce_pipeline
from api import aggregate_pii, generate_legit_report
    return redirect(url_for("invite_management"))

##################
# Databases      #
##################
class CreateDatabaseForm(Form):
    db_id = TextField("Database ID", [validators.DataRequired()])
        
@app.route("/admin/databases/", methods=["GET","POST"])
def databases():
    if request.method == "POST":
        form = CreateDatabaseForm(request.form)
        if form.validate():
            database = Database(id=form.db_id.data)
            database.put()
            
            return redirect(url_for("databases"))
    else:
        form = CreateDatabaseForm()
        
    dbs = Database.query().fetch(100)
    
    return render_template("admin/databases.html",
                            dbs=dbs, form=form)

##################
# Analysis       #
##################
class AnalysisReport(ndb.Model):
    org_name = ndb.StringProperty()
    blob_key = ndb.BlobKeyProperty()

def get_legit_report(intake_user, api_type="api"):
    pii = aggregate_pii([intake_user])
    query_dict = {}
    for k,v in pii.items():
        if v:
            query_dict[k] = v.pop()
                
    return generate_legit_report(query_dict, api_type, pre_hashed=True)
    
@app.template_filter()
def dev_decrypt(value):
    "Jinja2 filter for decrypting values in development"
    
    if DEVELOPMENT:
        return decrypt_value(value, aes_key=hashlib.sha256("dev_key").digest())
    else:
        return value

def analyze_user(intake_user_key):
    #logging.debug("Analyzing User Key: %s" % intake_user_key)
    intake_user = intake_user_key.get()
    counter_key = "analysis::run_count::%s" % intake_user.org.urlsafe()
    
    count = CombinedUser.query(CombinedUser.intake_users == intake_user.key).count(1)
    if not count:  
        # Generate a legit report
        legit_report, intake_users = get_legit_report(intake_user, api_type="api")

        # Create a combined user
        cu = CombinedUser.create(intake_users, legit_report)
        cu.put()
        
    # Check one off the list
    #memcache.decr(counter_key)
    
    return "Great Success"

def generate_csv(org_name):
    org = Organization.query(Organization.name == org_name).get()
    combined_users = CombinedUser.query(CombinedUser.orgs == org.key)
    
    rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
    for cu in combined_users:
        org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
        has_facebook = 1 if cu.facebook_id else 0
        has_twitter = 1 if cu.twitter_id else 0
        has_linkedin = 1 if cu.linkedin_id else 0
        rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" % 
                    (org_user_id,
                     cu.legit_report["legitscore"],
                     cu.legit_report["lrg_reputation"]["marketplace_memberships"],
                     cu.legit_report["lrg_reputation"]["total_transactions"],
                     cu.legit_report["lrg_reputation"]["total_reviews"],
                     cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
                     len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
                     has_facebook,
                     has_twitter,
                     has_linkedin))
    
    #logging.info("CSV FILE")
    #logging.info(rv)
                     
    file_name = files.blobstore.create(mime_type='text/plain',
                                       _blobinfo_uploaded_filename="%s-analysis" % org_name)
    with files.open(file_name, 'a') as f:
        f.write(rv)
        
    files.finalize(file_name) 
    blob_key = files.blobstore.get_blob_key(file_name)
    
    ar = AnalysisReport.query(AnalysisReport.org_name == org_name).get() or AnalysisReport(org_name=org_name)
    ar.blob_key = blob_key
    ar.put()
    
@app.route("/admin/analysis/run_analysis/<org_key>", methods=["POST"])    
def run_analysis(org_key=None):
    """
    Task that deletes all existing CombinedUsers and regenerates them from the
    underlying IntakeUsers. This task can take a long time to run, so it should
    be run on a dedicated instance. 
    """
    DELETE_BATCH = 500
    ANALYZE_BATCH = 50
    
    # Clear out the existing combined users    
    cu_query = CombinedUser.query()
    if org_key:
        org_key = ndb.Key(urlsafe=org_key)

        cu_query = cu_query.filter(CombinedUser.orgs == org_key)