<filename>data/external/repositories/103259/Kaggle_The_Hunt_for_Prohibited_Content-master/Python/generate_bootstrap.py
# -*- coding: UTF-8 -*-

"""
Python version: 2.7.6
Version: 1.0 at Sep 01 2014
Author: <NAME> < <EMAIL> >
Email: <EMAIL>
"""

import argparse
import numpy as np
from collections import defaultdict

def generateBootstrapSamples(in_file, out_file, seed):
    """
    This function reads in a VW format file (in_file) and generates 
    bootstrapping samples which are then wrote to an output file (out_file).
    
    Note that due to the one-line-after-another reading style, all the 
    duplicated samples in the bootstrapping samples (which is due to sampling
    with replacement) are wrote next to each other. Thus to simulate a real
    bootstrapping process, one have to random shuffle the data generated by
    this function, e.g., via "shuf out_file -o out_file_shuffle" in the shell.
    """
    
    # number of samples
    #numSample = len(open(in_file, "rb").readlines())
    numSample = 3995803
    print "     Number of samples: %s" % numSample
    print "     Random seed: %s" % seed
    # generate random index with replacement
    rng = np.random.RandomState(int(seed))
    index = rng.randint(numSample, size=numSample)
    # count the number of each index
    sampleIndex = defaultdict(lambda : 0)
    for i in xrange(numSample):
	sampleIndex[index[i]] += 1

    with open(out_file, "wb") as writer:
        for e,line in enumerate(open(in_file, "rb")):
            if sampleIndex.has_key(e):
                for n in xrange(sampleIndex[e]):
                    writer.write( line )
	    if (e+1)%1000000==0:
		print "     Wrote %s" % (e+1)
                
if __name__ == "__main__":
    
    parser = argparse.ArgumentParser(description="generate bootstrap samples")
    parser.add_argument("input")
    parser.add_argument("output")
    parser.add_argument("seed")
    args = parser.parse_args()
    
    generateBootstrapSamples(args.input, args.output, args.seed)
<filename>data/external/repositories/103259/Kaggle_The_Hunt_for_Prohibited_Content-master/Python/generate_bootstrap.py
# -*- coding: UTF-8 -*-

"""
Python version: 2.7.6
Version: 1.0 at Sep 01 2014
Author: <NAME> < <EMAIL> >
Email: <EMAIL>
"""

import argparse
import numpy as np
from collections import defaultdict

def generateBootstrapSamples(in_file, out_file, seed):
    """
    This function reads in a VW format file (in_file) and generates 
    bootstrapping samples which are then wrote to an output file (out_file).
    
    Note that due to the one-line-after-another reading style, all the 
    duplicated samples in the bootstrapping samples (which is due to sampling
    with replacement) are wrote next to each other. Thus to simulate a real
    bootstrapping process, one have to random shuffle the data generated by
    this function, e.g., via "shuf out_file -o out_file_shuffle" in the shell.
    """
    
    # number of samples
    #numSample = len(open(in_file, "rb").readlines())
    numSample = 3995803
    print "     Number of samples: %s" % numSample
    print "     Random seed: %s" % seed
    # generate random index with replacement
    rng = np.random.RandomState(int(seed))
    index = rng.randint(numSample, size=numSample)
    # count the number of each index
    sampleIndex = defaultdict(lambda : 0)
    for i in xrange(numSample):
	sampleIndex[index[i]] += 1

    with open(out_file, "wb") as writer:
        for e,line in enumerate(open(in_file, "rb")):
            if sampleIndex.has_key(e):
                for n in xrange(sampleIndex[e]):
                    writer.write( line )
	    if (e+1)%1000000==0:
		print "     Wrote %s" % (e+1)
                
if __name__ == "__main__":
    
    parser = argparse.ArgumentParser(description="generate bootstrap samples")
    parser.add_argument("input")
    parser.add_argument("output")
    parser.add_argument("seed")
    args = parser.parse_args()
    
    generateBootstrapSamples(args.input, args.output, args.seed)
# -*- coding: UTF-8 -*- Python version: 2.7.6 Version: 1.0 at Sep 01 2014 Author: <NAME> < <EMAIL> > Email: <EMAIL> This function reads in a VW format file (in_file) and generates      bootstrapping samples which are then wrote to an output file (out_file).          Note that due to the one-line-after-another reading style, all the      duplicated samples in the bootstrapping samples (which is due to sampling     with replacement) are wrote next to each other. Thus to simulate a real     bootstrapping process, one have to random shuffle the data generated by     this function, e.g., via "shuf out_file -o out_file_shuffle" in the shell. # number of samples #numSample = len(open(in_file, "rb").readlines()) # generate random index with replacement # count the number of each index