package com.twitter.elephantbird.mapreduce.input;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;

/**
 * An input format for LZO-encoded W3C-style log files.  The W3C style has lines that read
 * hash value1 value2 value3 ... valueN
 * with a corresponding lookup file
 * hash field1 field2 field3 ... fieldN
 * where fieldK is the name of the metric valueK represents.  The hash is a CRC of all the
 * field names, and adding or removing metrics results in a new hash at the beginning of the
 * associated line.  This class expects there to be a filename (in HDFS) that contains all
 * the hash/field-name lines used in the course of writing the data, and uses those as a look-aside
 * to parse W3C log lines into a hash map.  To use the class, derive from LzoW3CLogInputFormat
 * and override createRecordReader to return a LzoW3CLogReaderReader-derived object.
 */

public class LzoW3CLogInputFormat extends LzoInputFormat<LongWritable, MapWritable> {

  @Override
  public RecordReader<LongWritable, MapWritable> createRecordReader(InputSplit arg0,
      TaskAttemptContext arg1) throws IOException, InterruptedException {
    throw new IllegalArgumentException("LzoW3CLogInputFormat must be initialized by calling newInstance(fieldFile)");
  }

  /**
   * Use this method to create valid instances of LzoW3CLogInputFormat
   * @param fieldDefinitionFile path to file in HDFS that contains the CRC hash to column mappings.
   * @return
   */
  public static LzoW3CLogInputFormat newInstance(final String fieldDefinitionFile) {
    return new LzoW3CLogInputFormat() {
      @Override
      public RecordReader<LongWritable, MapWritable> createRecordReader(InputSplit split,
          TaskAttemptContext context) throws IOException, InterruptedException {
        RecordReader<LongWritable, MapWritable> reader = new LzoW3CLogRecordReader() {
          @Override
          protected String getFieldDefinitionFile() {
            return fieldDefinitionFile;
          }
        };
        reader.initialize(split, context);
        return reader;
      }
    };
  }
}


package com.twitter.elephantbird.crunch;

import com.google.protobuf.Message;
import com.twitter.elephantbird.mapreduce.input.LzoProtobufBlockInputFormat;
import org.apache.crunch.ReadableData;
import org.apache.crunch.io.FormatBundle;
import org.apache.crunch.io.ReadableSource;
import org.apache.crunch.io.impl.FileSourceImpl;
import org.apache.crunch.types.PType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.util.List;

/**
 * A Crunch {@code Source} for reading protocol buffers from an {@link LzoProtobufBlockInputFormat} file. The
 * format implements the {@code ReadableSource} interface, so records from these files can be read for
 * in-memory joins.
 */
public class LzoProtobufSource<T extends Message> extends FileSourceImpl<T> implements ReadableSource<T> {

  /**
   * Factory method for creating a new {@code LzoProtobufSource} from a given path and protocol buffer
   * message class.
   *
   * @param path path to the data
   * @param protoClass the Message class to read
   * @return a new {@code LzoProtobufSource}
   */
  public static <S extends Message> LzoProtobufSource<S> at(Path path, Class<S> protoClass) {
    return new LzoProtobufSource<S>(path, EBTypes.protos(protoClass));
  }

  /**
   * Factory method for creating a new {@code LzoProtobufSource} from the given paths and protocol buffer
   * message class.
   *
   * @param paths paths to the data
   * @param protoClass the Message class to read
   * @return a new {@code LzoProtobufSource}
   */
  public static <S extends Message> LzoProtobufSource<S> at(List<Path> paths, Class<S> protoClass) {
    return new LzoProtobufSource<S>(paths, EBTypes.protos(protoClass));
  }

  private static <T> FormatBundle<LzoProtobufBlockInputFormat> getBundle(PType<T> ptype) {
    return FormatBundle.forInput(LzoProtobufBlockInputFormat.class)
        .set("elephantbird.class.for.MultiInputFormat", ptype.getTypeClass().getName());
  }

  public LzoProtobufSource(Path path, PType<T> ptype) {
    super(path, ptype, getBundle(ptype));
  }

  public LzoProtobufSource(List<Path> paths, PType<T> ptype) {
    super(paths, ptype, getBundle(ptype));
  }

  @Override
  public Iterable<T> read(Configuration conf) throws IOException {
    return read(conf, new ProtobufFileReaderFactory<T>(ptype));
  }

  @Override
  public ReadableData<T> asReadable() {
    return new ProtobufReadableData<T>(paths, ptype);
  }
}


package com.twitter.elephantbird.mapreduce.input;

import java.io.IOException;
import java.io.InputStream;

import com.google.protobuf.InvalidProtocolBufferException;
import com.twitter.elephantbird.mapreduce.input.MapredInputFormatCompatible;
import com.twitter.elephantbird.mapreduce.io.BinaryBlockReader;
import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
import com.twitter.elephantbird.mapreduce.io.BinaryWritable;
import com.twitter.elephantbird.mapreduce.io.DecodeException;
import com.twitter.elephantbird.util.HadoopCompat;
import com.twitter.elephantbird.util.HadoopUtils;
import com.twitter.elephantbird.util.TypeRef;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A reader for LZO-encoded protobuf blocks, generally written by
 * a ProtobufBlockWriter or similar.
 *
 *

 * A small fraction of bad records are tolerated. See {@link LzoRecordReader}
 * for more information on error handling.
 */
public class LzoBinaryBlockRecordReader<M, W extends BinaryWritable<M>>
    extends LzoRecordReader<LongWritable, W> implements MapredInputFormatCompatible<LongWritable, W> {
  private static final Logger LOG = LoggerFactory.getLogger(LzoBinaryBlockRecordReader.class);

  private LongWritable key_;
  private W value_;
  private final TypeRef<M> typeRef_;
  boolean updatePosition = false;
  /* make LzoBinaryBlockRecordReader return lzoblock offset the same way as
   * LzoBinaryBlockRecordReader for indexing purposes.
   * For the the first record returned, pos_ should be 0
   * if the recordreader is reading the first split,
   * otherwise it should be end of the current lzo block.
   * This makes pos_ consistent with LzoBinaryB64LineRecordReader.
   */

  private final BinaryBlockReader<M> reader_;

  private Counter recordsReadCounter;
  private Counter recordErrorsCounter;
  private Counter recordsSkippedCounter;

  private final BinaryConverter<M> deserializer_;

  public LzoBinaryBlockRecordReader(TypeRef<M> typeRef, BinaryBlockReader<M> reader, W binaryWritable) {
    key_ = new LongWritable();
    value_ = binaryWritable;
    reader_ = reader;
    typeRef_ = typeRef;
    deserializer_ = reader.getConverter();
  }

  @Override
  public synchronized void close() throws IOException {
    super.close();
    if (reader_ != null) {
      reader_.close();
    }
  }

  @Override
  public LongWritable getCurrentKey() throws IOException, InterruptedException {
    return key_;
  }

  @Override
  public W getCurrentValue() throws IOException, InterruptedException {
    return value_;
  }

  @Override
  protected void createInputReader(InputStream input, Configuration conf) throws IOException {
    reader_.setInputStream(input);
  }

  @Override
  public void initialize(InputSplit genericSplit, TaskAttemptContext context)
                                     throws IOException, InterruptedException {
    String group = "LzoBlocks of " + typeRef_.getRawClass().getName();
    recordsReadCounter = HadoopUtils.getCounter(context, group, "Records Read");
    recordsSkippedCounter = HadoopUtils.getCounter(context, group, "Records Skipped");
    recordErrorsCounter = HadoopUtils.getCounter(context, group, "Errors");

    super.initialize(genericSplit, context);
  }

  @Override
  protected void skipToNextSyncPoint(boolean atFirstRecord) throws IOException {
    // No need to skip to the sync point here; the block reader will do it for us.
    LOG.debug("LzoProtobufBlockRecordReader.skipToNextSyncPoint called with atFirstRecord = " + atFirstRecord);
    updatePosition = !atFirstRecord;
    // except for the first split, skip a protobuf block if it starts exactly at the split boundary
    // because such a block would be read by the previous split (note comment about 'pos_ > end_'
    // in nextKeyValue() below)
    reader_.parseNextBlock(!atFirstRecord);
  }

  @Override
  public void setKeyValue(LongWritable key, W value) {
    key_ = key;
    value_ = value;
  }

  /**
   * Read the next key, value pair.
   *

   * A small fraction of bad records in input are tolerated.
   * See  {@link LzoRecordReader} for more information on error handling.
   *
   * @return true if a key/value pair was read
   * @throws IOException
   * @throws InterruptedException
   */
  @Override
  public boolean nextKeyValue() throws IOException, InterruptedException {
    // If we are past the end of the file split, tell the reader not to read any more new blocks.
    // Then continue reading until the last of the reader's already-parsed values are used up.
    while (true) { // loop to skip over bad records
      if (pos_ > end_) {
        reader_.markNoMoreNewBlocks();
        // Why not pos_ >= end_, stop when we just reach the end?
        // we don't know if we have read all the bytes uncompressed in the current lzo block,
        // only way to make sure that we have read all of the split is to read till the
        // first record that has at least one byte in the next split.
        // As a consequence of this, next split reader skips at least one byte
        // (i.e. skips either partial or full record at the beginning).
      }
      byte[] byteArray = reader_.readNextProtoBytes();

      if(byteArray == null) {
        return false;
      }

      errorTracker.incRecords();
      M decoded = null;
      try {
        decoded = deserializer_.fromBytes(byteArray);
      } catch (DecodeException e) {
        errorTracker.incErrors(e);
        HadoopCompat.incrementCounter(recordErrorsCounter, 1);
        continue;
      }

      if (updatePosition) {
        pos_ = getLzoFilePos();
        updatePosition = false;
      }

      if (decoded != null) {
        key_.set(pos_);
        value_.set(decoded);
        pos_ = getLzoFilePos();

        HadoopCompat.incrementCounter(recordsReadCounter, 1);
        return true;
      } else {
        HadoopCompat.incrementCounter(recordsSkippedCounter, 1);
      }
    }
  }
}


package com.twitter.elephantbird.mapred.output;

import java.io.IOException;

import com.twitter.elephantbird.util.HadoopCompat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.StatusReporter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;

import com.twitter.elephantbird.mapred.input.DeprecatedInputFormatWrapper;
import com.twitter.elephantbird.mapred.input.DeprecatedInputFormatWrapper.ReporterWrapper;
import com.twitter.elephantbird.mapreduce.output.WorkFileOverride;
import com.twitter.elephantbird.util.HadoopUtils;

/**
 * The wrapper enables an {@link OutputFormat} written for new
 * mapreduce interface to be used in contexts where
 * a {@link org.apache.hadoop.mapred.OutputFormat} old mapred
 * interface is required.

 *
 * Usage:

 *    jobConf.setInputFormat(anInputFormat.class);
 *    //set OutputFormat class using a mapreduce OutputFormat
 *    DeprecatedOutputFormatWrapper.setOutputFormat(org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class, jobConf);
 *

 *
 * @see DeprecatedInputFormatWrapper
 *
 * @author Raghu Angadi
 */
@SuppressWarnings("deprecation")
public class DeprecatedOutputFormatWrapper<K, V>
                      implements org.apache.hadoop.mapred.OutputFormat<K, V> {

  protected OutputFormat<K, V> realOutputFormat;

  private static final String CLASS_CONF_KEY = "elephantbird.class.for.DeprecatedOutputFormatWrapper";

  public DeprecatedOutputFormatWrapper() {}

  public DeprecatedOutputFormatWrapper(OutputFormat<K, V> mapreduceOutputFormat) {
    realOutputFormat = mapreduceOutputFormat;
  }

  /**
   * Sets jobs output format to {@link DeprecatedOutputFormatWrapper} and stores
   * supplied real {@link OutputFormat} class name in job configuration.
   * This configuration is read on the remote tasks to instantiate actual
   * OutputFormat correctly.
   */
  public static void setOutputFormat(Class<?> realOutputFormatClass, JobConf jobConf) {
    jobConf.setOutputFormat(DeprecatedOutputFormatWrapper.class);
    HadoopUtils.setClassConf(jobConf, CLASS_CONF_KEY, realOutputFormatClass);
  }

  /**
   * For cases where we need to set hadoop1 output format in a hadoop2 Configuration object.
   */
  public static void setOutputFormat(Class<?> realOutputFormatClass, Configuration conf) {
    conf.setClass("mapred.output.format.class", DeprecatedOutputFormatWrapper.class, org.apache.hadoop.mapred.OutputFormat.class);
    HadoopUtils.setClassConf(conf, CLASS_CONF_KEY, realOutputFormatClass);
  }

  @SuppressWarnings("unchecked")
  private void initOutputFormat(JobConf conf) {
    if (realOutputFormat == null) {
      realOutputFormat = ReflectionUtils.newInstance(
                           conf.getClass(CLASS_CONF_KEY, null, OutputFormat.class),
                           conf);
    }
  }

  @Override
  public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
    initOutputFormat(job);
    try {
      realOutputFormat.checkOutputSpecs(HadoopCompat.newJobContext(job, null));
    } catch (InterruptedException e) {
      throw new IOException(e);
    }
  }

  @Override
  public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job,
      String name, Progressable progress) throws IOException {
    initOutputFormat(job);
    if (realOutputFormat instanceof WorkFileOverride) {
      ((WorkFileOverride) realOutputFormat).setName(name);
    }
    return new RecordWriterWrapper<K, V>(realOutputFormat, job, name, progress);
  }

  private static class RecordWriterWrapper<K, V> implements RecordWriter<K, V> {

    private org.apache.hadoop.mapreduce.RecordWriter<K, V> realWriter;
    private TaskAttemptContext taskContext;

    @SuppressWarnings("unchecked")
    RecordWriterWrapper(OutputFormat<K, V> realOutputFormat,
                        JobConf jobConf, String name, Progressable progress)
                        throws IOException {
      try {
        // create a MapContext to provide access to the reporter (for counters)
        taskContext = HadoopCompat.newMapContext(
            jobConf, TaskAttemptID.forName(jobConf.get("mapred.task.id")),
            null, null, null, new ReporterWrapper((Reporter) progress), null);

        realWriter = realOutputFormat.getRecordWriter(taskContext);
      } catch (InterruptedException e) {
        throw new IOException(e);
      }
    }

    @Override
    public void close(Reporter reporter) throws IOException {
      try {
        // create a context just to pass reporter
        realWriter.close(taskContext);
      } catch (InterruptedException e) {
        throw new IOException(e);
      }
    }

    @Override
    public void write(K key, V value) throws IOException {
      try {
        realWriter.write(key, value);
      } catch (InterruptedException e) {
        throw new IOException(e);
      }
    }
  }
}


package com.twitter.elephantbird.util;

import java.util.List;
import java.util.ArrayList;
import java.util.regex.Pattern;

/**
 * Handles singularization and pluralization.
 *
 * Largely taken from the defunct rogueweb project at
 * <link>http://code.google.com/p/rogueweb/</link>
 * Original author Anthony Eden.
 */

public class Inflection {
  private static final List<Inflection> plurals_ = new ArrayList<Inflection>();
  private static final List<Inflection> singulars_ = new ArrayList<Inflection>();
  private static final List<String> uncountables_ = new ArrayList<String>();

  static {
    // plural is "singular to plural form"
    // singular is "plural to singular form"
    plural("$", "s");
    plural("s$", "s");
    plural("(ax|test)is$", "$1es");
    plural("(octop|vir)us$", "$1i");
    plural("(alias|status)$", "$1es");
    plural("(bu)s$", "$1ses");
    plural("(buffal|tomat)o$", "$1oes");
    plural("([ti])um$", "$1a");
    plural("sis$", "ses");
    plural("(?:([^f])fe|([lr])f)$", "$1$2ves");
    plural("(hive)$", "$1s");
    plural("([^aeiouy]|qu)y$", "$1ies");
    // plural("([^aeiouy]|qu)ies$", "$1y");
    plural("(x|ch|ss|sh)$", "$1es");
    plural("(matr|vert|ind)ix|ex$", "$1ices");
    plural("([m|l])ouse$", "$1ice");
    plural("^(ox)$", "$1en");
    plural("(quiz)$", "$1zes");

    singular("s$", "");
    singular("(n)ews$", "$1ews");
    singular("([ti])a$", "$1um");
    singular("((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)ses$", "$1$2sis");
    singular("(^analy)ses$", "$1sis");
    singular("([^f])ves$", "$1fe");
    singular("(hive)s$", "$1");
    singular("(tive)s$", "$1");
    singular("([lr])ves$", "$1f");
    singular("([^aeiouy]|qu)ies$", "$1y");
    singular("(s)eries$", "$1eries");
    singular("(m)ovies$", "$1ovie");
    singular("(x|ch|ss|sh)es$", "$1");
    singular("([m|l])ice$", "$1ouse");
    singular("(bus)es$", "$1");
    singular("(o)es$", "$1");
    singular("(shoe)s$", "$1");
    singular("(cris|ax|test)es$", "$1is");
    singular("(octop|vir)i$", "$1us");
    singular("(alias|status)es$", "$1");
    singular("^(ox)en", "$1");
    singular("(vert|ind)ices$", "$1ex");
    singular("(matr)ices$", "$1ix");
    singular("(quiz)zes$", "$1");

    irregular("person", "people");
    irregular("man", "men");
    irregular("child", "children");
    irregular("sex", "sexes");
    irregular("move", "moves");

    uncountable("equipment");
    uncountable("information");
    uncountable("rice");
    uncountable("money");
    uncountable("species");
    uncountable("series");
    uncountable("fish");
    uncountable("sheep");
  }

  private String pattern_;
  private String replacement_;
  private boolean ignoreCase_;

  public Inflection(String pattern) {
    this(pattern, null, true);
  }

  public Inflection(String pattern, String replacement) {
    this(pattern, replacement, true);
  }

  public Inflection(String pattern, String replacement, boolean ignoreCase) {
    pattern_ = pattern;
    replacement_ = replacement;
    ignoreCase_ = ignoreCase;
  }

  private static void plural(String pattern, String replacement) {
    plurals_.add(0, new Inflection(pattern, replacement));
  }

  private static void singular(String pattern, String replacement) {
    singulars_.add(0, new Inflection(pattern, replacement));
  }

  private static void irregular(String s, String p) {
    plural("(" + s.substring(0, 1) + ")" + s.substring(1) + "$", "$1" + p.substring(1));
    singular("(" + p.substring(0, 1) + ")" + p.substring(1) + "$", "$1" + s.substring(1));
  }

  private static void uncountable(String word) {
    uncountables_.add(word);
  }

  /**
   * Does the given word match?
   * @param word The word
   * @return True if it matches the inflection pattern
   */
  public boolean match(String word) {
    int flags = ignoreCase_ ? Pattern.CASE_INSENSITIVE : 0;
    return Pattern.compile(pattern_, flags).matcher(word).find();
  }

  /**
   * Replace the word with its pattern.
   * @param word The word
   * @return The result
   */
  public String replace(String word) {
    int flags = ignoreCase_ ? Pattern.CASE_INSENSITIVE : 0;
    return Pattern.compile(pattern_, flags).matcher(word).replaceAll(replacement_);
  }

  /**
   * Return the pluralized version of a word.
   * @param word The word
   * @return The pluralized word
   */
  public static String pluralize(String word) {
    if (isUncountable(word)) {
      return word;
    } else {
      for (Inflection inflection : plurals_) {
        if (inflection.match(word)) {
          return inflection.replace(word);
        }
      }
      return word;
    }
  }

  /**
   * Return the singularized version of a word.
   * @param word The word
   * @return The singularized word
   */
  public static String singularize(String word) {
    if (isUncountable(word)) {
      return word;
    } else {
      for (Inflection inflection : singulars_) {
        if (inflection.match(word)) {
          return inflection.replace(word);
        }
      }
    }
    return word;
  }

  /**
   * Return true if the word is uncountable.
   * @param word The word
   * @return True if it is uncountable
   */
  public static boolean isUncountable(String word) {
    for (String w : uncountables_) {
      if (w.equalsIgnoreCase(word)) {
        return true;
      }
    }
    return false;
  }

}


package com.twitter.elephantbird.mapreduce.output;

import java.io.DataOutputStream;
import java.io.IOException;

import com.twitter.elephantbird.util.HadoopCompat;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.hadoop.compression.lzo.LzopCodec;
import com.twitter.elephantbird.util.LzoUtils;

/**
 * Base class for Lzo outputformats.
 * provides an helper method to create lzo output stream.
 */
public abstract class LzoOutputFormat<K, V> extends WorkFileOverride.FileOutputFormat<K, V> {

  public static final Logger LOG = LoggerFactory.getLogger(LzoOutputFormat.class);

  /**
   * Helper method to create lzo output file needed to create RecordWriter
   */
  protected DataOutputStream getOutputStream(TaskAttemptContext job)
                  throws IOException, InterruptedException {

    return LzoUtils.getIndexedLzoOutputStream(
                      HadoopCompat.getConfiguration(job),
                      getDefaultWorkFile(job, LzopCodec.DEFAULT_LZO_EXTENSION));
  }
}


package com.twitter.elephantbird.thrift;

import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.TMemoryInputTransport;

/**
 * Extends TDeserializer to improve handling of corrupt records in a few ways:
 *
 * <ul>
 * <li> sets read-limit for TBinaryProtocol before each deserialization.
 *      Reduces OutOfMemoryError exceptions.
 *
 * <li> {@link ThriftBinaryProtocol} to avoid excessive cpu consumed while
 *      skipping some corrupt records.
 *
 * <li> {@code deserialize(buf, offset, len)} method can avoid buffer copies.
 *      Serialized struct need not span a entire byte array.
 * </ul>
 *
 * To obtain an instance of ThriftBinaryDeserializer use {@link ThriftCompat#createBinaryDeserializer()}.
 * It will take care of cross version compatibility between thrift 0.7 and 0.9+ code.
 *
 * @see ThriftCompat
 */
public class ThriftBinaryDeserializer extends AbstractThriftBinaryDeserializer {

  public ThriftBinaryDeserializer() {
    super(new ThriftBinaryProtocol.Factory());
  }
  // set the default limit of 10M to check against corruption. In reality no single record
  // or container should hit this limit.
  private long lengthLimit = 10*1024*1024;

  // use protocol and transport directly instead of using ones in TDeserializer
  private final TMemoryInputTransport trans = new TMemoryInputTransport();
  private TBinaryProtocol protocol = new ThriftBinaryProtocol(trans, lengthLimit, lengthLimit);

  @Override
  public void deserialize(TBase base, byte[] bytes) throws TException {
    deserialize(base, bytes, 0, bytes.length);
  }

  /**
   * Same as {@link #deserialize(TBase, byte[])}, but much more buffer copy friendly.
   */
  public void deserialize(TBase base, byte[] bytes, int offset, int len) throws TException {

    // If incoming payload is larger than threshold then adjust the threshold by making it 10% more
    // than incoming payload-length and re-initialize object. This generally should not happen.
    if (len > lengthLimit) {
      lengthLimit = len + Double.valueOf(0.1 * len).longValue();
      protocol = new ThriftBinaryProtocol(trans, lengthLimit, lengthLimit);
    }

    resetAndInitialize(protocol, len);
    trans.reset(bytes, offset, len);
    base.read(protocol);
  }
}


package com.twitter.elephantbird.pig.store;

import java.io.IOException;

import com.twitter.elephantbird.util.HadoopCompat;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.pig.LoadFunc;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.UDFContext;

import com.google.common.base.Preconditions;
import com.twitter.elephantbird.pig.load.SequenceFileLoader;
import com.twitter.elephantbird.pig.util.GenericWritableConverter;
import com.twitter.elephantbird.pig.util.PigCounterHelper;
import com.twitter.elephantbird.pig.util.SequenceFileConfig;
import com.twitter.elephantbird.pig.util.WritableConverter;

/**
 * Pig StoreFunc supporting conversion from Pig tuples to arbitrary key-value pairs stored within
 * {@link SequenceFile}s. Example usage:
 *
 *

 * pairs = LOAD '$INPUT' AS (key: int, value: chararray);
 *
 * STORE pairs INTO '$OUTPUT' USING com.twitter.elephantbird.pig.store.SequenceFileStorage (
 *   '-c com.twitter.elephantbird.pig.util.IntWritableConverter',
 *   '-c com.twitter.elephantbird.pig.util.TextConverter'
 * );
 *

 *
 * @author Andy Schlaikjer
 * @see SequenceFileConfig
 * @see WritableConverter
 * @see SequenceFileLoader
 */
public class SequenceFileStorage<K extends Writable, V extends Writable> extends BaseStoreFunc {
  /**
   * Failure modes for use with {@link PigCounterHelper} to keep track of runtime error counts.
   */
  public static enum Error {
    /**
     * Null tuple was supplied to {@link SequenceFileStorage#putNext(Tuple)}.
     */
    NULL_TUPLE,
    /**
     * Tuple supplied to {@link SequenceFileStorage#putNext(Tuple)} whose length is not 2.
     */
    TUPLE_SIZE,
    /**
     * Null key was supplied to {@link SequenceFileStorage#putNext(Tuple)} and key type is not
     * {@link NullWritable}.
     */
    NULL_KEY,
    /**
     * Null value was supplied to {@link SequenceFileStorage#putNext(Tuple)} and value type is not
     * {@link NullWritable}.
     */
    NULL_VALUE;
  }

  /**
   * Refinement of {@link SequenceFileConfig} which adds key/value "type" option.
   */
  private class Config extends SequenceFileConfig<K, V> {
    public static final String TYPE_PARAM = "type";
    public Class<K> keyClass;
    public Class<V> valueClass;

    public Config(String keyArgs, String valueArgs, String otherArgs)
        throws ParseException, IOException {
      super(keyArgs, valueArgs, otherArgs);
    }

    @Override
    protected Options getKeyValueOptions() {
      @SuppressWarnings("static-access")
      Option typeOption =
          OptionBuilder
              .withLongOpt(TYPE_PARAM)
              .hasArg()
              .withArgName("cls")
              .withDescription(
                  "Writable type of data. Defaults to type returned by getWritableClass()"
                      + " method of configured WritableConverter.").create("t");
      return super.getKeyValueOptions().addOption(typeOption);
    }

    @Override
    protected void initialize() throws IOException {
      /*
       * Attempt to initialize key, value classes using arguments. If user doesn't specify '--type'
       * arg, then class will be null.
       */
      keyClass = getWritableClass(keyArguments.getOptionValue(TYPE_PARAM));
      valueClass = getWritableClass(valueArguments.getOptionValue(TYPE_PARAM));

      // initialize key, value converters
      keyConverter.initialize(keyClass);
      valueConverter.initialize(valueClass);

      // allow converters to define writable classes if not already defined
      if (keyClass == null) {
        keyClass = keyConverter.getWritableClass();
      }
      if (valueClass == null) {
        valueClass = valueConverter.getWritableClass();
      }
    }

    /**
     * @param writableClassName
     * @return {@code null} if writableClassName is {@code null}, otherwise the Class instance named
     * by writableClassName.
     * @throws IOException
     */
    @SuppressWarnings("unchecked")
    private <W extends Writable> Class<W> getWritableClass(String writableClassName)
        throws IOException {
      if (writableClassName == null) {
        return null;
      }
      try {
        return PigContext.resolveClassName(writableClassName);
      } catch (Exception e) {
        throw new IOException(
            String.format("Failed to load Writable class '%s'", writableClassName),
            e);
      }
    }
  }

  private final Config config;

  /**
   * Parses key and value options from argument strings. Available options for both key and value
   * argument strings match those supported by
   * {@link SequenceFileLoader#SequenceFileLoader(String, String)}, as well as:
   *

   *
-t|--type cls

   *
{@link Writable} implementation class of data. If Writable class reported by
   * {@link WritableConverter#getWritableClass()} is null (e.g. when using
   * {@link GenericWritableConverter}), this option must be specified.

   *

   *
   * @param keyArgs
   * @param valueArgs
   * @param otherArgs
   * @throws ParseException
   * @throws IOException
   * @throws ClassNotFoundException
   */
  public SequenceFileStorage(String keyArgs, String valueArgs, String otherArgs)
      throws ParseException, IOException,
      ClassNotFoundException {
    super();
    this.config = new Config(keyArgs, valueArgs, otherArgs);
  }

  /**
   * Delegates to {@link #SequenceFileStorage(String, String, String)}, passing empty string for
   * {@code otherArgs} param.
   *
   * @param keyArgs
   * @param valueArgs
   * @throws ParseException
   * @throws IOException
   * @throws ClassNotFoundException
   */
  public SequenceFileStorage(String keyArgs, String valueArgs) throws ParseException, IOException,
      ClassNotFoundException {
    super();
    this.config = new Config(keyArgs, valueArgs, "");
  }

  /**
   * Default constructor which uses default options for key and value.
   *
   * @throws ClassNotFoundException
   * @throws IOException
   * @throws ParseException
   * @see #SequenceFileStorage(String, String)
   */
  public SequenceFileStorage() throws ParseException, IOException, ClassNotFoundException {
    this("", "", "");
  }

  @Override
  public void checkSchema(ResourceSchema schema) throws IOException {
    Preconditions.checkNotNull(schema, "Schema is null");
    ResourceFieldSchema[] fields = schema.getFields();
    Preconditions.checkNotNull(fields, "Schema fields are undefined");
    Preconditions.checkArgument(2 == fields.length,
        "Expecting 2 schema fields but found %s", fields.length);
    config.keyConverter.checkStoreSchema(fields[0]);
    config.valueConverter.checkStoreSchema(fields[1]);
  }

  @Override
  public String relToAbsPathForStoreLocation(String location, Path cwd) throws IOException {
    return LoadFunc.getAbsolutePath(location, cwd);
  }

  @SuppressWarnings("unchecked")
  @Override
  public void setStoreLocation(String location, Job job) throws IOException {
    Configuration conf = HadoopCompat.getConfiguration(job);
    ensureUDFContext(conf);
    verifyWritableClass(config.keyClass, true, config.keyConverter);
    verifyWritableClass(config.valueClass, false, config.valueConverter);
    job.setOutputKeyClass(config.keyClass);
    job.setOutputValueClass(config.valueClass);
    super.setStoreLocation(location, job);
    if ("true".equals(conf.get("output.compression.enabled"))) {
      FileOutputFormat.setCompressOutput(job, true);
      String codec = conf.get("output.compression.codec");
      FileOutputFormat.setOutputCompressorClass(job,
          PigContext.resolveClassName(codec).asSubclass(CompressionCodec.class));
    } else {
      // This makes it so that storing to a directory ending with ".gz" or ".bz2" works.
      setCompression(new Path(location), job);
    }
  }

  private void ensureUDFContext(Configuration conf) throws IOException {
    if (UDFContext.getUDFContext().isUDFConfEmpty()
        && conf.get("pig.udf.context") != null) {
      MapRedUtil.setupUDFContext(conf);
    }
  }

  /**
   * Tests validity of Writable class, ensures consistent error message for both key and value
   * tests.
   *
   * @param writableClass class being tested.
   * @param isKeyClass {@code true} if testing keyClass, {@code false} otherwise.
   * @param writableConverter associated WritableConverter instance.
   */
  private static <W extends Writable> void verifyWritableClass(Class<W> writableClass,
      boolean isKeyClass, WritableConverter<W> writableConverter) {
    Preconditions.checkNotNull(writableClass, "%s Writable class is undefined;"
        + " WritableConverter of type '%s' does not define default Writable type,"
        + " and no type was specified by user", isKeyClass ? "Key" : "Value", writableConverter
        .getClass().getName());
  }

  /**
   * @param path
   * @param job
   */
  private void setCompression(Path path, Job job) {
    CompressionCodecFactory codecFactory =
        new CompressionCodecFactory(HadoopCompat.getConfiguration(job));
    CompressionCodec codec = codecFactory.getCodec(path);
    if (codec != null) {
      FileOutputFormat.setCompressOutput(job, true);
      FileOutputFormat.setOutputCompressorClass(job, codec.getClass());
    } else {
      FileOutputFormat.setCompressOutput(job, false);
    }
  }

  @Override
  public OutputFormat<K, V> getOutputFormat() {
    return new SequenceFileOutputFormat<K, V>();
  }

  @Override
  public void putNext(Tuple t) throws IOException {
    // validate input tuple
    if (t == null) {
      incrCounter(Error.NULL_TUPLE, 1);
      return;
    }
    if (t.size() != 2) {
      incrCounter(Error.TUPLE_SIZE, 1);
      return;
    }

    // convert key from pig to writable
    K key = config.keyConverter.toWritable(t.get(0));
    if (key == null) {
      incrCounter(Error.NULL_KEY, 1);
      return;
    }

    // convert value from pig to writable
    V value = config.valueConverter.toWritable(t.get(1));
    if (value == null) {
      incrCounter(Error.NULL_VALUE, 1);
      return;
    }

    // write key-value pair
    writeRecord(key, value);
  }
}


package com.twitter.elephantbird.mapreduce.io;

import org.apache.thrift.TBase;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.twitter.elephantbird.thrift.ThriftBinaryDeserializer;
import com.twitter.elephantbird.util.TypeRef;

public class ThriftConverter<M extends TBase<?, ?>> implements BinaryConverter<M> {

  public static final Logger LOG = LoggerFactory.getLogger(ThriftConverter.class);

  private TypeRef<M> typeRef;
  private TSerializer serializer;
  private TDeserializer deserializer;

  // limit the number of warnings in case of serialization errors.
  private static final int MAX_WARNINGS = 100;
  private static int numWarningsLogged = 0;

  private static void logWarning(String message, Throwable t) {
    // does not need to be thread safe
    if ( numWarningsLogged < MAX_WARNINGS ) {
      LOG.info(message, t);
      numWarningsLogged++;
    }
  }

  /**
   * Returns a ThriftConverter for a given Thrift class.
   */
  public static <M extends TBase<?, ?>> ThriftConverter<M> newInstance(Class<M> tClass) {
    return new ThriftConverter<M>(new TypeRef<M>(tClass){});
  }

  public static <M extends TBase<?, ?>> ThriftConverter<M> newInstance(TypeRef<M> typeRef) {
    return new ThriftConverter<M>(typeRef);
  }

  public ThriftConverter(TypeRef<M> typeRef) {
    this.typeRef = typeRef;
  }

  @Override
  public M fromBytes(byte[] messageBuffer) throws DecodeException {
    try {
      if (deserializer == null)
        deserializer = new ThriftBinaryDeserializer();
      M message = typeRef.safeNewInstance();
      deserializer.deserialize(message, messageBuffer);
      return message;
    } catch (Exception e) {
      // normally a TException. but some corrupt records can cause
      // other runtime exceptions (e.g. IndexOutOfBoundsException).
      throw new DecodeException(e);
    }
  }

  @Override
  public byte[] toBytes(M message) {
    if (serializer == null)
      serializer = new TSerializer();
    try {
      return serializer.serialize(message);
    } catch (TException e) {
      logWarning("failed to serialize", e);
      return null;
    }
  }

  @Override
  public boolean equals(Object obj) {
    if (this == obj)
      return true;
    try {
      return typeRef.getType().equals(((ThriftConverter<?>)obj).typeRef.getType());
    } catch (ClassCastException e) {
      return false;
    }
  }
}


package com.twitter.elephantbird.mapreduce.input;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.Counters.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.util.ReflectionUtils;

import com.twitter.elephantbird.util.HadoopCompat;
import com.twitter.elephantbird.util.HadoopUtils;

/**
 * The wrapper enables an {@link InputFormat} written for old deprecated
 * mapred interface to be used unmodified in contexts where
 * a InputFormat with new mapreduce interface is required.

 *
 * Usage:

 *    // set InputFormat class using a mapred InputFormat
 *    MapReduceInputFormatWrapper.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class, job);
 *    job.setOutputFormatClass(org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.class);
 *    // ...
 *

 *
 * This is a mirror image of {@link DeprecatedInputFormatWrapper}
 *
 * @author Raghu Angadi
 *
 */
@SuppressWarnings("deprecation")
public class MapReduceInputFormatWrapper<K, V> extends org.apache.hadoop.mapreduce.InputFormat<K, V> {

  public static final String CLASS_CONF_KEY = "elephantbird.class.for.MapReduceInputFormatWrapper";

  protected InputFormat<K, V> realInputFormat;

  /**
   * Sets jobs input format to {@link MapReduceInputFormatWrapper} and stores
   * supplied real {@link InputFormat} class name in job configuration.
   * This configuration is read on the remote tasks to instantiate actual
   * InputFormat correctly.
   */
  public static void setInputFormat(Class<?> realInputFormatClass, Job job) {
    job.setInputFormatClass(MapReduceInputFormatWrapper.class);
    setWrappedInputFormat(realInputFormatClass, HadoopCompat.getConfiguration(job));
  }

  public static void setWrappedInputFormat(Class<?> realInputFormatClass, Configuration conf) {
    HadoopUtils.setClassConf(conf, CLASS_CONF_KEY, realInputFormatClass);
  }

  @SuppressWarnings("unchecked")
  private void initInputFormat(Configuration conf) {
    if (realInputFormat == null) {
      realInputFormat = ReflectionUtils.newInstance(
                          conf.getClass(CLASS_CONF_KEY, null, InputFormat.class),
                          conf);
    }
  }

  public MapReduceInputFormatWrapper() {
    // real inputFormat is initialized based on conf.
  }

  public MapReduceInputFormatWrapper(InputFormat<K, V> realInputFormat) {
    this.realInputFormat = realInputFormat;
  }

  @Override
  public RecordReader<K, V> createRecordReader(InputSplit split,
                                               TaskAttemptContext context)
                                               throws IOException, InterruptedException {

    initInputFormat(HadoopCompat.getConfiguration(context));
    return new RecordReaderWrapper<K, V>(realInputFormat);
  }

  @Override
  public List<InputSplit> getSplits(JobContext context)
                                    throws IOException, InterruptedException {

    JobConf jobConf = (JobConf) HadoopCompat.getConfiguration(context);

    initInputFormat(jobConf);

    org.apache.hadoop.mapred.InputSplit[] splits =
      realInputFormat.getSplits(jobConf, jobConf.getNumMapTasks());

    if (splits == null) {
      return null;
    }

    List<InputSplit> resultSplits = new ArrayList<InputSplit>(splits.length);

    for (org.apache.hadoop.mapred.InputSplit split : splits) {
      if (split.getClass() == org.apache.hadoop.mapred.FileSplit.class) {
        org.apache.hadoop.mapred.FileSplit mapredFileSplit =
          ((org.apache.hadoop.mapred.FileSplit)split);
        resultSplits.add(new FileSplit(mapredFileSplit.getPath(),
                                       mapredFileSplit.getStart(),
                                       mapredFileSplit.getLength(),
                                       mapredFileSplit.getLocations()));
      } else {
        resultSplits.add(new InputSplitWrapper(split));
      }
    }

    return resultSplits;
  }

  private static class RecordReaderWrapper<K, V>
      extends RecordReader<K, V> implements MapredInputFormatCompatible<K, V> {

    private org.apache.hadoop.mapred.RecordReader<K, V> realReader;
    private InputFormat<K, V> realInputFormat;

    private K keyObj = null;
    private V valueObj = null;

    public RecordReaderWrapper(InputFormat<K, V> realInptuFormat) throws IOException {
      this.realInputFormat = realInptuFormat;
    }

    @Override
    public void close() throws IOException {
      realReader.close();
    }

    @Override
    public float getProgress() throws IOException {
      return realReader.getProgress();
    }

    @Override
    public void initialize(InputSplit split, final TaskAttemptContext context)
        throws IOException, InterruptedException {

      org.apache.hadoop.mapred.InputSplit oldSplit;

      if (split.getClass() == FileSplit.class) {
        oldSplit = new org.apache.hadoop.mapred.FileSplit(
                        ((FileSplit)split).getPath(),
                        ((FileSplit)split).getStart(),
                        ((FileSplit)split).getLength(),
                        split.getLocations());
      } else {
        oldSplit = ((InputSplitWrapper)split).realSplit;
      }

      @SuppressWarnings("unchecked")
      Reporter reporter = new Reporter() { // Reporter interface over ctx

        final TaskInputOutputContext ioCtx =
                context instanceof TaskInputOutputContext ?
                       (TaskInputOutputContext) context : null;

        public void progress() { HadoopCompat.progress(context); }

        // @Override
        public float getProgress() {
          return (ioCtx != null) ? ioCtx.getProgress() : 0;
        }

        public void setStatus(String status) {
          if (ioCtx != null)
            HadoopCompat.setStatus(ioCtx, status);
        }

        public void incrCounter(String group, String counter, long amount) {
          if (ioCtx != null)
            HadoopCompat.incrementCounter(ioCtx.getCounter(group, counter), amount);
        }

        @SuppressWarnings("unchecked")
        public void incrCounter(Enum<?> key, long amount) {
          if (ioCtx != null)
            HadoopCompat.incrementCounter(ioCtx.getCounter(key), amount);
        }

        public org.apache.hadoop.mapred.InputSplit getInputSplit()
            throws UnsupportedOperationException {
          throw new UnsupportedOperationException();
        }

        public Counter getCounter(String group, String name) {
          return ioCtx != null ?
            (Counter) HadoopCompat.getCounter(ioCtx, group, name) : null;
        }

        @SuppressWarnings("unchecked")
        public Counter getCounter(Enum<?> name) {
          return ioCtx != null ?
             (Counter)ioCtx.getCounter(name) : null;
        }
      };

      realReader = realInputFormat.getRecordReader(
                      oldSplit,
                      (JobConf) HadoopCompat.getConfiguration(context),
                      reporter);

      keyObj = realReader.createKey();
      valueObj = realReader.createValue();
    }

    @Override
    public K getCurrentKey() throws IOException, InterruptedException {
      return keyObj;
    }

    @Override
    public V getCurrentValue() throws IOException, InterruptedException {
      return valueObj;
    }

    @Override
    public boolean nextKeyValue() throws IOException, InterruptedException {
      return realReader.next(keyObj, valueObj);
    }

    @Override
    public void setKeyValue(K key, V value) {
      keyObj = key;
      valueObj = value;
    }
  }

  private static class InputSplitWrapper extends InputSplit implements Writable {

    org.apache.hadoop.mapred.InputSplit realSplit;

    @SuppressWarnings("unused") // MapReduce instantiates this.
    public InputSplitWrapper() {}

    public InputSplitWrapper(org.apache.hadoop.mapred.InputSplit realSplit) {
      this.realSplit = realSplit;
    }

    @Override
    public long getLength() throws IOException {
      return realSplit.getLength();
    }

    @Override
    public String[] getLocations() throws IOException {
      return realSplit.getLocations();
    }

    @Override
    public void readFields(DataInput in) throws IOException {
      String className = WritableUtils.readString(in);
      Class<?> splitClass;

      try {
        splitClass = Class.forName(className);
      } catch (ClassNotFoundException e) {
        throw new IOException(e);
      }

      realSplit = (org.apache.hadoop.mapred.InputSplit)
                  ReflectionUtils.newInstance(splitClass, null);
      ((Writable)realSplit).readFields(in);
    }

    @Override
    public void write(DataOutput out) throws IOException {
      WritableUtils.writeString(out, realSplit.getClass().getName());
      ((Writable)realSplit).write(out);
    }
  }
}


package com.twitter.elephantbird.pig.util;

import java.io.IOException;

import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.thrift.TBase;

import com.google.protobuf.Descriptors;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Message;
import com.twitter.elephantbird.util.Protobufs;
import com.twitter.elephantbird.util.ThriftUtils;
import com.twitter.elephantbird.util.TypeRef;

public class PigUtil {
  /**
   *  Returns class using Pig's class loader.
   */
  public static Class<?> getClass(String className) {
    try {
      return PigContext.resolveClassName(className);
    } catch (IOException e) {
      throw new RuntimeException("Could not instantiate " + className, e);
    }
  }

  /**
   * Temporary hack to check if PIG version is 0.9.0 or newer
   */
  public static final boolean Pig9orNewer;
  static {
    boolean methodFound = false;
    try {
      // check for a new method in a class common to both Pig 0.8 and 0.9
      Class<?> cls = Class.forName("org.apache.pig.EvalFunc");
      methodFound = cls.getMethod("getCacheFiles") != null;
    } catch (Exception e) {
    }
    Pig9orNewer = methodFound;
  }

  /**
   * Returns class using Pig's class loader.
   * If that fails tries {@link Protobufs#getProtobufClass(String)} to
   * resolve the class
   *
   */
  public static Class<? extends Message> getProtobufClass(String protoClassName) {
    try {
      // try Pig's loader
      Class<?> protoClass = getClass(protoClassName);
      return protoClass.asSubclass(Message.class);
    } catch (RuntimeException e) {
      // try normal loader first (handles inner class).
      return Protobufs.getProtobufClass(protoClassName);
    }
  }

  /**
   * Invokes the static {@code getDescriptor} method of the given {@link Message} class.
   *
   * @param protoClass the {@link Message} class whose {@link Descriptor} should be retrieved.
   * @return the Descriptor instance for the given {@link Message} class.
   * @throws RuntimeException on any exception encountered during introspection of protoClass or
   * invocation of its static {@code getDescriptor} method.
   * @see Descriptors
   */
  public static Descriptor getProtobufDescriptor(Class<? extends Message> protoClass) {
    try {
      return (Descriptor) protoClass.getMethod("getDescriptor").invoke(null);
    } catch (Exception e) {
      throw new RuntimeException(String.format("Failed to get Descriptor for Message type '%s'", protoClass.getName()), e);
    }
  }

  /**
   * @param protoClassName name of the {@link Message} class whose {@link Descriptor} should be
   * retrieved.
   * @return the Descriptor instance for the given {@link Message} class.
   * @throws RuntimeException on any exception encountered during class load or introspection of
   * protoClassName or invocation of its static {@code getDescriptor} method.
   * @see #getProtobufDescriptor(Class)
   */
  public static Descriptor getProtobufDescriptor(String protoClassName) {
    return getProtobufDescriptor(getProtobufClass(protoClassName));
  }

  public static<M extends Message> TypeRef<M> getProtobufTypeRef(String protoClassName) {
    return new TypeRef<M>(getProtobufClass(protoClassName)){};
  }

  /** Returns TypeRef using Pig class loader. */
  public static<T extends TBase<?,?>> TypeRef<T> getThriftTypeRef(String thriftClassName) {
    return ThriftUtils.getTypeRef(getClass(thriftClassName));
  }

  public static Schema outputSchemaForProtobuf(ProtobufToPig protoToPig, TypeRef<? extends Message> typeRef) {
    Schema outSchema;
    try {
      outSchema = protoToPig.toSchema(Protobufs.getMessageDescriptor(typeRef.getRawClass()));
      // wrap the schema if size > 1
      if(outSchema.size() > 1) {
        outSchema = new Schema(new Schema.FieldSchema(typeRef.getRawClass().getSimpleName(), outSchema, DataType.TUPLE));
      }
    } catch (FrontendException e) {
      throw new RuntimeException(e);
    }
    return outSchema;
  }

  public static Schema outputSchemaForThrift(TypeRef<? extends TBase<?,?>> typeRef) {
    Schema outSchema;
    try {
      outSchema = ThriftToPig.toSchema(typeRef.getRawClass());
      // wrap the schema if size > 1
      if(outSchema.size() > 1) {
        outSchema = new Schema(new Schema.FieldSchema(typeRef.getRawClass().getSimpleName(), outSchema, DataType.TUPLE));
      }
    } catch (FrontendException e) {
      throw new RuntimeException(e);
    }
    return outSchema;
  }
}


package com.twitter.elephantbird.util;

import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.RCFile;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.pig.LoadPushDown.RequiredField;
import org.apache.pig.LoadPushDown.RequiredFieldList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Joiner;
import com.google.common.collect.Lists;

public class RCFileUtil {

  private static final Logger LOG = LoggerFactory.getLogger(RCFileUtil.class);

  /**
   * comma separated list of indices of the fields. This is not a list of field
   * numbers (as in a Protobuf or a Thrift class).
   *
   * If this configuration is not set or is empty, all the fields
   * are read ("unknown fields" in Protobufs are not carried over).
   */
  public static String REQUIRED_FIELD_INDICES_CONF =
                           "elephantbird.rcfile.input.required.field.indices";

  public static String COLUMN_METADATA_PROTOBUF_KEY = "elephantbird.rcfile.column.info.protobuf";

  /**
   * reads {@link ColumnarMetadata} stored in an RCFile.
   * @throws IOException if metadata is not stored or in case of any other error.
   */
  public static ColumnarMetadata readMetadata(Configuration conf, Path rcfile)
                                              throws IOException {

    Metadata metadata = null;

    Configuration confCopy = new Configuration(conf);
    // set up conf to read all the columns
    ColumnProjectionUtils.setFullyReadColumns(confCopy);

    RCFile.Reader reader = new RCFile.Reader(rcfile.getFileSystem(confCopy),
                                             rcfile,
                                             confCopy);

    //ugly hack to get metadata. RCFile has to provide access to metata
    try {
      Field f = RCFile.Reader.class.getDeclaredField("metadata");
      f.setAccessible(true);
      metadata = (Metadata)f.get(reader);
    } catch (Throwable t) {
      throw new IOException("Could not access metadata field in RCFile reader", t);
    }

    reader.close();

    Text metadataKey = new Text(COLUMN_METADATA_PROTOBUF_KEY);

    if (metadata == null || metadata.get(metadataKey) == null) {
      throw new IOException("could not find ColumnarMetadata in " + rcfile);
    }

    return ColumnarMetadata.parseFrom(metadata.get(metadataKey).getBytes());
  }

  /**
   * Returns list of columns that need to be read from the RCFile.
   * These columns are the intersection of currently required columns and
   * columns stored in the file.
   * If any required column does not exist in the file, we need to read
   * the "unknown fields" column, which is usually the last last one.
   */
  public static ArrayList<Integer> findColumnsToRead(
                                         Configuration      conf,
                                         List<Integer>      currFieldIds,
                                         ColumnarMetadata   storedInfo)
                                         throws IOException {

    ArrayList<Integer> columnsToRead = Lists.newArrayList();

    // first find the required fields
    ArrayList<Integer> requiredFieldIds = Lists.newArrayList();
    String reqFieldStr = conf.get(RCFileUtil.REQUIRED_FIELD_INDICES_CONF, "");

    int numKnownFields = currFieldIds.size();

    if (reqFieldStr == null || reqFieldStr.equals("")) {
      for(int i=0; i<numKnownFields; i++) {
        requiredFieldIds.add(currFieldIds.get(i));
      }
    } else {
      for (String str : reqFieldStr.split(",")) {
        int idx = Integer.valueOf(str);
        if (idx < 0 || idx >= numKnownFields) {
          throw new IOException("idx " + idx + " is out of range for known fields");
        }
        requiredFieldIds.add(currFieldIds.get(idx));
      }
    }

    List<Integer> storedFieldIds = storedInfo.getFieldIdList();

    for(int i=0; i < storedFieldIds.size(); i++) {
      int sid = storedFieldIds.get(i);
      if (sid > 0 && requiredFieldIds.contains(sid)) {
        columnsToRead.add(i);
      }
    }

    // unknown fields : the required fields that are not listed in storedFieldIds
    String unknownFields = "";
    for(int rid : requiredFieldIds) {
      if (!storedFieldIds.contains(rid)) {
        unknownFields += " " + rid;
      }
    }

    if (unknownFields.length() > 0) {
      int last = storedFieldIds.size() - 1;
      LOG.info("unknown fields among required fileds :" + unknownFields);
      if (storedFieldIds.get(last) != -1) { // not expected
        throw new IOException("No unknowns column in in input");
      }
      columnsToRead.add(last);
    }

    LOG.info(String.format(
        "reading %d%s out of %d stored columns for %d required columns",
        columnsToRead.size(),
        (unknownFields.length() > 0 ? " (including unknowns column)" : ""),
        storedInfo.getFieldIdList().size(),
        requiredFieldIds.size()));

    return columnsToRead;
  }

  /**
   * Sets {@link #REQUIRED_FIELD_INDICES_CONF} to list of indices
   * if requiredFieldList is not null.
   */
  public static void setRequiredFieldConf(Configuration conf,
                                          RequiredFieldList requiredFieldList) {

    // set required fields conf for RCFile[Protobuf|Thrift] input format
    String value = "";

    if (requiredFieldList != null) {
      List<Integer> indices = Lists.newArrayList();
      for(RequiredField f : requiredFieldList.getFields()) {
        indices.add(f.getIndex());
      }

      value = Joiner.on(",").join(indices);
    }

    conf.set(RCFileUtil.REQUIRED_FIELD_INDICES_CONF, value);
  }
}


package com.twitter.elephantbird.mapreduce.output;

import java.io.IOException;

import com.twitter.elephantbird.mapreduce.io.ThriftConverter;
import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
import com.twitter.elephantbird.util.HadoopCompat;
import com.twitter.elephantbird.util.ThriftUtils;
import com.twitter.elephantbird.util.TypeRef;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.thrift.TBase;

/**
 * Data is written as one base64 encoded serialized thrift per line.

 *
 * Do not forget to set Thrift class using setClassConf().
 */
public class LzoThriftB64LineOutputFormat<M extends TBase<?, ?>>
    extends LzoOutputFormat<M, ThriftWritable<M>> {

  protected TypeRef<M> typeRef_;

  public LzoThriftB64LineOutputFormat() {}

  public LzoThriftB64LineOutputFormat(TypeRef<M> typeRef) {
    typeRef_ = typeRef;
  }

  /**
   * Sets an internal configuration in jobConf so that remote Tasks
   * instantiate appropriate object for this generic class based on thriftClass
   */
  public static <M extends TBase<?, ?>>
  void setClassConf(Class<M> thriftClass, Configuration jobConf) {
    ThriftUtils.setClassConf(jobConf,
                             LzoThriftB64LineOutputFormat.class,
                             thriftClass);
  }

  @Override
  public RecordWriter<M, ThriftWritable<M>> getRecordWriter(TaskAttemptContext job)
      throws IOException, InterruptedException {
    if (typeRef_ == null) {
      typeRef_ = ThriftUtils.getTypeRef(HadoopCompat.getConfiguration(job), LzoThriftB64LineOutputFormat.class);
    }
    return new LzoBinaryB64LineRecordWriter<M, ThriftWritable<M>>(new ThriftConverter<M>(typeRef_), getOutputStream(job));
  }
}


package com.twitter.elephantbird.mapreduce.output;

import java.io.DataOutputStream;
import java.io.IOException;

import com.google.protobuf.Message;
import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
import com.twitter.elephantbird.mapreduce.io.BinaryWritable;
import com.twitter.elephantbird.mapreduce.io.ProtobufConverter;
import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
import com.twitter.elephantbird.mapreduce.io.ThriftConverter;
import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
import com.twitter.elephantbird.util.Codecs;
import com.twitter.elephantbird.util.Protobufs;

import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.thrift.TBase;

/**
 * A RecordWriter-derived class for use with the LzoProtobufB64LineOutputFormat.
 * Writes data as base64 encoded serialized protocol buffers, one per line.
 */

public class LzoBinaryB64LineRecordWriter<M, W extends BinaryWritable<M>>
    extends RecordWriter<M, W> {

  private final BinaryConverter<M> converter;
  private final DataOutputStream out;
  private final Base64 base64;

  public LzoBinaryB64LineRecordWriter(BinaryConverter<M> converter, DataOutputStream out) {
    this.converter = converter;
    this.out = out;
    this.base64 = Codecs.createStandardBase64();
  }

  @Override
  public void write(M nullWritable, W writable) throws IOException {
    byte[] b64Bytes = base64.encode(converter.toBytes(writable.get()));
    out.write(b64Bytes);
    out.write(Protobufs.NEWLINE_UTF8_BYTE);
  }

  @Override
  public void close(TaskAttemptContext taskAttemptContext)
      throws IOException, InterruptedException {
    out.close();
  }

  // for convenience
  public static <M extends TBase<?, ?>> LzoBinaryB64LineRecordWriter<M, ThriftWritable<M>>
               newThriftWriter(Class<M> tClass, DataOutputStream out) {
    return new LzoBinaryB64LineRecordWriter<M, ThriftWritable<M>>
                            (ThriftConverter.newInstance(tClass), out);
  }

  // for convenience
  public static <M extends Message> LzoBinaryB64LineRecordWriter<M, ProtobufWritable<M>>
               newProtobufWriter(Class<M> tClass, DataOutputStream out) {
    return new LzoBinaryB64LineRecordWriter<M, ProtobufWritable<M>>
                            (ProtobufConverter.newInstance(tClass), out);
  }
}


package com.github.jreddit.parser.listing;

import java.util.LinkedList;
import java.util.List;

import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.jreddit.parser.entity.Submission;
import com.github.jreddit.parser.entity.Thing;
import com.github.jreddit.parser.exception.RedditParseException;

public class SubmissionsListingParser extends RedditListingParser {

    private static final Logger LOGGER = LoggerFactory.getLogger(SubmissionsListingParser.class);

    /**
     * Parse JSON received from reddit into a list of submissions.
     * This parser expects the JSON to be of a listing of submissions ('links').
     *
     * @param jsonText JSON Text
     * @return Parsed list of submissions
     *
     * @throws ParseException
     */
    public List<Submission> parse(String jsonText) throws RedditParseException {

        // Parse to a list of things
        List<Thing> things = this.parseGeneric(jsonText);

        // List of comment and submission mixed elements
        List<Submission> submissions = new LinkedList<Submission>();

        // Iterate over things
        for (Thing t : things) {

            if (t instanceof Submission) {
                submissions.add((Submission) t);
            } else {
                LOGGER.warn("Encountered an unexpected reddit thing (" + t.getKind().value() + "), skipping it.");
            }

        }

        // Return resulting comments list
        return submissions;

    }

}


package com.github.jreddit.oauth.client;

import java.io.IOException;
import java.io.UnsupportedEncodingException;

import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.jreddit.oauth.RedditToken;
import com.github.jreddit.request.RedditGetRequest;
import com.github.jreddit.request.RedditPostRequest;

/**
 * HTTP client implementation for a RedditClient.
 *
 * @author Simon Kassing
 *
 * @see RedditClient
 */
public class RedditHttpClient extends RedditClient {

    private static final Logger LOGGER = LoggerFactory.getLogger(RedditHttpClient.class);

    private final String userAgent;

    private final HttpClient httpClient;

    /**
     * @param userAgent User agent of your application
     * @param httpClient HTTP client to use for the requests
     */
    public RedditHttpClient(String userAgent, HttpClient httpClient) {
        this.userAgent = userAgent;
        this.httpClient = httpClient;
    }

    @Override
    public String post(RedditToken rToken, RedditPostRequest redditRequest) {

        try {

            // Create post request
            HttpPost request = new HttpPost(OAUTH_API_DOMAIN + redditRequest.generateRedditURI());

            // Add parameters to body
            request.setEntity(new StringEntity(redditRequest.generateBody()));

            // Add authorization
            addAuthorization(request, rToken);

            // Add user agent
            addUserAgent(request);

            // Add content type
            request.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");

            return executeHttpRequest(request);

        } catch (UnsupportedEncodingException uee) {
            LOGGER.warn("Unsupported Encoding Exception thrown in POST request when encoding body", uee);
        }

        return null;

    }

    @Override
    public String get(RedditToken rToken, RedditGetRequest redditRequest) {

        // Create get request
        HttpGet request = new HttpGet(OAUTH_API_DOMAIN + redditRequest.generateRedditURI());

        // Add authorization
        addAuthorization(request, rToken);

        // Add user agent
        addUserAgent(request);

        return executeHttpRequest(request);

    }

    /**
     * Execute the given HTTP request.
     *
     * @param request HTTP request
     *
     * @return Result, null if failed
     */
    private String executeHttpRequest(HttpUriRequest request) {
        try {

            // Attempt to do execute request
            HttpResponse response = httpClient.execute(request);

            // Return response if successful
            if (response != null) {
                return EntityUtils.toString(response.getEntity(), "UTF-8");
            }

        } catch (UnsupportedEncodingException uee) {
            LOGGER.warn("Unsupported Encoding Exception thrown in request", uee);
        } catch (ClientProtocolException cpe) {
            LOGGER.warn("Client Protocol Exception thrown in request", cpe);
        } catch (IOException ioe) {
            LOGGER.warn("I/O Exception thrown in request", ioe);
        }

        return null;

    }

    /**
     * Add authorization to the HTTP request.
     *
     * @param request HTTP request
     * @param rToken Reddit token (generally of the "bearer" type)
     */
    private void addAuthorization(HttpRequest request, RedditToken rToken) {
        request.addHeader("Authorization", rToken.getTokenType() + " " + rToken.getAccessToken());
    }

    /**
     * Add user agent to the HTTP request.
     *
     * @param request HTTP request
     */
    private void addUserAgent(HttpRequest request) {
        request.addHeader("User-Agent", userAgent);
    }

}


package com.github.jreddit.parser.listing;

import static com.github.jreddit.parser.util.JsonUtils.safeJsonToString;

import java.util.LinkedList;
import java.util.List;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.jreddit.parser.entity.Comment;
import com.github.jreddit.parser.entity.Kind;
import com.github.jreddit.parser.entity.More;
import com.github.jreddit.parser.entity.Submission;
import com.github.jreddit.parser.entity.Subreddit;
import com.github.jreddit.parser.entity.Thing;
import com.github.jreddit.parser.exception.RedditParseException;
import com.github.jreddit.parser.util.JsonUtils;

public class RedditListingParser {

    private static final Logger LOGGER = LoggerFactory.getLogger(RedditListingParser.class);

    protected static final JSONParser JSON_PARSER = new JSONParser();

    /**
     * Validate that it is indeed the starting of a listing of reddit things.
     *
     * @param response Object returned by JSON parser
     *
     * @throws RedditRequestException If the response is not valid listing of reddit things
     */
    public void validate(Object response) throws RedditParseException {

        // Check for null
        if (response == null) {
            throw new RedditParseException();
        }

        // Check it is a JSON response
        if (!(response instanceof JSONObject)) {
            throw new RedditParseException("not a JSON response");
        }

        // Cast to JSON object
        JSONObject jsonResponse = ((JSONObject) response);

        // Check for error
        if (jsonResponse.get("error") != null) {
            throw new RedditParseException(JsonUtils.safeJsonToInteger(jsonResponse.get("error")));
        }

        // Check that data exists
        if (jsonResponse.get("data") == null && jsonResponse.get("json") == null) {
            throw new RedditParseException("data is missing from listing");
        }

    }

    /**
     * Parse JSON received from reddit into a list of things.
     * This parser expects the JSON to be of a listing of things, and supports
     * the following things: More, Comment, Submission, and Subreddit.
     *
     * @param jsonText JSON Text
     * @return Parsed list of things
     *
     * @throws ParseException
     */
    public List<Thing> parseGeneric(String jsonText) throws RedditParseException {
        return parseGeneric(jsonText, "children");
    }

    /**
     * Parse JSON received from reddit into a list of things.
     * This parser expects the JSON to be of a listing of things, and supports
     * the following things: More, Comment, Submission, and Subreddit.

     *

     * Note: if it encounters an invalid element (e.g. missing kind or data), it will
     * log a warning using SLF4J and would return null.
     *
     * @param jsonText JSON Text
     * @param listingName Name of the listing name within the data
     *
     * @return Parsed list of things
     *
     * @throws ParseException
     */
    public List<Thing> parseGeneric(String jsonText, String listingName) throws RedditParseException {

        try {

            // List of submissions
            List<Thing> things = new LinkedList<Thing>();

            // Send request to reddit server via REST client
            Object response = JSON_PARSER.parse(jsonText);

            // Check for reddit error, can throw a RedditError
            validate(response);

            // Cast to a JSON object
            JSONObject object = (JSONObject) response;

            // Get the array of children
            JSONArray array = (JSONArray) ((JSONObject) object.get("data")).get(listingName);

            // Iterate over array of children
            for (Object element : array) {

                // Get the element
                JSONObject data = (JSONObject) element;

                // Make sure it is of the correct kind
                String kindData = safeJsonToString(data.get("kind"));
                Object objData = data.get("data");

                // If no kind is given
                if (kindData == null) {
                    LOGGER.warn("Kind data missing, skipping it.");

                // If no data is given
                } else if (objData == null || !(objData instanceof JSONObject)) {
                    LOGGER.warn("Object data missing, skipping it.");

                } else {

                    // Attempt to match
                    Kind kind = Kind.match(kindData);

                    // Parse the thing
                    Thing thing = parseThing(kind, ((JSONObject) data.get("data")));

                    // Show warning if failed
                    if (thing == null) {
                        LOGGER.warn("Encountered invalid kind for a listing (" + kindData + "), skipping it.");

                    } else {
                        things.add(thing);
                    }

                }

            }

            // Finally return list of submissions
            return things;

        } catch (ParseException pe) {
            throw new RedditParseException(pe);
        }

    }

    /**
     * Parse the data into a thing if possible.
     *
     * @param kind Kind of data
     * @param data Data for the thing
     * @return The thing generated from the data, if failed null
     */
    private Thing parseThing(Kind kind, JSONObject data) {

        // For a comment
        if (kind == Kind.COMMENT) {
            return new Comment(data);

        // For a submission
        } else if (kind == Kind.LINK) {
            return new Submission(data);

        // For a subreddit
        } else if (kind == Kind.SUBREDDIT) {
            return new Subreddit(data);

        // For a more
        } else if (kind == Kind.MORE) {
            return new More(data);

        // In all other cases (null, or of a different type)
        } else {
            return null;
        }

    }

}


package com.github.jreddit.oauth.param;

import java.util.HashSet;
import java.util.Set;

/**
 * Builder for the scopes of a request.
 *
 * @author Simon Kassing
 *
 */
public class RedditScopeBuilder {

    /** Set of reddit scopes. */
    private Set<RedditScope> scopes;

    public RedditScopeBuilder() {
        scopes = new HashSet<RedditScope>();
    }

    /**
     * Build a string for the request. Called upon by
     * <i>RedditToken</i> when requesting a token that
     * allows scope definition.
     *
     * @return String list of the scopes
     */
    public String build() {

        // Add each scope to the scope list
        String s = "";
        for (RedditScope scope : scopes) {
            s += scope.value() + RedditScope.SEPARATOR;
        }

        // Remove final separator
        if (s.length() > 0) {
            s = s.substring(0, s.length() - RedditScope.SEPARATOR.length());
        }

        // Return scope list
        return s;

    }

    /**
     * Add a scope to the builder. If the scope has already
     * been added, it will not be added double.
     *
     * @param scope Reddit scope
     *
     * @return This builder
     */
    public RedditScopeBuilder addScope(RedditScope scope) {
        scopes.add(scope);
        return this;
    }

    /**
     * Add multiple scopes to the builder. If a scope has already
     * been added, it will not be added double.
     *
     * @param scopes Multiple scopes
     *
     * @return This builder
     */
    public RedditScopeBuilder addScopes(RedditScope... scopes) {
        for (RedditScope scope : scopes) {
            addScope(scope);
        }
        return this;
    }

    /**
     * Remove a scope from the builder.
     *
     * @param scope Reddit scope
     *
     * @return This builder
     */
    public RedditScopeBuilder removeScope(RedditScope scope) {
        scopes.remove(scope);
        return this;
    }

    /**
     * Remove multiple scopes from the builder.
     *
     * @param scopes Multiple scopes
     *
     * @return This builder
     */
    public RedditScopeBuilder removeScopes(RedditScope... scopes) {
        for (RedditScope scope : scopes) {
            removeScope(scope);
        }
        return this;
    }

}


package com.github.jreddit.parser.listing;

import java.util.LinkedList;
import java.util.List;

import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.jreddit.parser.entity.Comment;
import com.github.jreddit.parser.entity.Submission;
import com.github.jreddit.parser.entity.Thing;
import com.github.jreddit.parser.entity.imaginary.MixedListingElement;
import com.github.jreddit.parser.exception.RedditParseException;

/**
 * Parser for a listing that has both submissions and comments mixed together.
 *
 * @author Simon Kassing
 *
 * @see MixedListingElement
 */
public class MixedListingParser extends RedditListingParser {

    private static final Logger LOGGER = LoggerFactory.getLogger(MixedListingParser.class);

    /**
     * Parse JSON received from reddit into a list of submissions and comments.
     * This parser expects the JSON to be of a listing of submissions and comments.
     *
     * @param jsonText JSON Text
     * @return Parsed list of submissions
     *
     * @throws ParseException
     */
    public List<MixedListingElement> parse(String jsonText) throws RedditParseException {

        // Parse to a list of things
        List<Thing> things = this.parseGeneric(jsonText);

        // List of comment and submission mixed elements
        List<MixedListingElement> mixedElements = new LinkedList<MixedListingElement>();

        // Iterate over things
        for (Thing t : things) {

            if (t instanceof Comment) {
                mixedElements.add((Comment) t);
            } else if (t instanceof Submission) {
                mixedElements.add((Submission) t);
            } else {
                LOGGER.warn("Encountered an unexpected reddit thing (" + t.getKind().value() + "), skipping it.");
            }

        }

        // Return result
        return mixedElements;

    }

}


package com.github.jreddit.parser.listing;

import java.util.LinkedList;
import java.util.List;

import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.jreddit.parser.entity.Comment;
import com.github.jreddit.parser.entity.More;
import com.github.jreddit.parser.entity.Thing;
import com.github.jreddit.parser.entity.imaginary.CommentTreeElement;
import com.github.jreddit.parser.exception.RedditParseException;

public class CommentsMoreParser extends RedditListingParser {

    private static final Logger LOGGER = LoggerFactory.getLogger(CommentsMoreParser.class);

    private static final JSONParser JSON_PARSER = new JSONParser();

    /**
     * Parse JSON received from reddit into a list of new additional comment tree elements.
     * This parser expects the JSON to be of a listing of comments and more's.

     *

     * Note: this parsing can only be performed on listings of comments and more's, not on
     * a comment tree of a submission.
     *
     * @param jsonText JSON Text
     * @return Parsed list of comments
     *
     * @throws ParseException
     * @throws RedditRequestException
     */
    public List<CommentTreeElement> parse(String jsonText) throws RedditParseException {

        try {

            // Parse JSON response
            Object response = JSON_PARSER.parse(jsonText);

            // Validate main
            this.validate(response);

            // Move to the main object
            JSONObject main = (JSONObject) ((JSONObject) response).get("json");

            // List of comment and more mixed elements
            List<CommentTreeElement> elements = new LinkedList<CommentTreeElement>();

            // If the main has data (it can happen that it does not, when no comments identifiers were passed along)
            if (main.get("data") != null) {

                // Parse to a list of things
                List<Thing> things = this.parseGeneric(main.toJSONString(), "things");

                // Iterate over things
                for (Thing t : things) {

                    if (t instanceof Comment) {
                        elements.add((Comment) t);
                    } else if (t instanceof More) {
                        elements.add((More) t);
                    } else {
                        LOGGER.warn("Encountered an unexpected reddit thing (" + t.getKind().value() + "), skipping it.");
                    }

                }

            }

            // Return resulting element list
            return elements;

        } catch (ParseException pe) {
            throw new RedditParseException(pe);
        }

    }

}


package com.github.jreddit.request.util;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class KeyValueFormatter {

    private static final Logger LOGGER = LoggerFactory.getLogger(KeyValueFormatter.class);

    private KeyValueFormatter() {
        // Left empty to prevent instantiation of this utility class
    }

    /**
     * Format a mapping of key-value pairs to a string, separating key from value using an
     * equals (=) sign, and separating pairs using an ampersand (&) sign.
     *
     * @param keyValueParameters Mapping of key-value pairs
     * @param encodeUTF8 Whether or not the values should be encoded in UTF-8
     *
     * @return Formatted string of key-value pairs (e.g. "a=1&b=something")
     */
    public static String format(Map<String, String> keyValueParameters, boolean encodeUTF8) {

        // Key set
        Set<String> keys = keyValueParameters.keySet();

        // Iterate over keys
        String paramsString = "";
        boolean start = true;
        for (String key : keys) {

            // Add separation ampersand
            if (!start) {
                paramsString = paramsString.concat("&");
            } else {
                start = false;
            }

            // Retrieve value
            String value = keyValueParameters.get(key);

            // Encode key
            if (encodeUTF8) {
                try {
                    value = URLEncoder.encode(value, "UTF-8");
                } catch (UnsupportedEncodingException e) {
                    LOGGER.warn("Unsupported Encoding Exception thrown when encoding value", e);
                }
            }

            // Add key-value pair
            paramsString = paramsString.concat(key + "=" + value);

        }

        // Return final parameter string
        return paramsString;

    }

    /**
     * Format a comma separated list of the given list.
     *
     * @param list List of strings
     *
     * @return Comma-separated list string (e.g. "a,b,c,d")
     */
    public static String formatCommaSeparatedList(List<String> list) {
        StringBuilder builder = new StringBuilder();
        for (int i = 0; i < list.size(); i++) {
            if (i != 0) {
                builder.append(",");
            }
            builder.append(list.get(i));
        }
        return builder.toString();
    }

}


package com.github.jreddit.oauth.client;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.jreddit.oauth.RedditToken;
import com.github.jreddit.request.RedditGetRequest;
import com.github.jreddit.request.RedditPostRequest;

/**
 * Wrapper for any reddit client, which makes it <i>polite</i>.
 * Polite means that it will only send requests in an interval,
 * and does not overload reddit. It will also mean you will get
 * less denial messages from reddit *hint* *hint*.
 *
 * @author Simon Kassing
 */
public class RedditPoliteClient extends RedditClient {

    /** Logger for this class. */
    private static final Logger LOGGER = LoggerFactory.getLogger(RedditPoliteClient.class);

    /** Wrapped reddit client. */
    private RedditClient redditClient;

    /** Default interval in milliseconds (1 per second). */
    private static final long DEFAULT_INTERVAL = 1000L;

    /** Default interval in milliseconds. */
    private long interval;

    /** Last time a request was made. */
    private long lastReqTime = 0;

    /**
     * Polite wrapper around the reddit client.
     *
     * @param redditClient Reddit client to wrap
     */
    public RedditPoliteClient(RedditClient redditClient) {
        this(redditClient, DEFAULT_INTERVAL);
    }

    /**
     * Polite wrapper around the reddit client with configurable time.
     *
     * @param redditClient Reddit client to wrap
     * @param interval Interval in milliseconds
     */
    public RedditPoliteClient(RedditClient redditClient, long interval) {
        this.redditClient = redditClient;
        this.interval = interval;
    }

    @Override
    public String post(RedditToken rToken, RedditPostRequest request) {
        waitIfNeeded();
        String result = redditClient.post(rToken, request);
        noteTime();
        return result;
    }

    @Override
    public String get(RedditToken rToken, RedditGetRequest request) {
        waitIfNeeded();
        String result = redditClient.get(rToken, request);
        noteTime();
        return result;
    }

    /**
     * Note the current time.
     */
    private void noteTime() {
        lastReqTime = System.currentTimeMillis();
    }

    /**
     * Wait if required.
     */
    private void waitIfNeeded() {

        // Calculate elapsed milliseconds
        long elapsed = System.currentTimeMillis() - lastReqTime;

        // If enough time has elapsed, no need to wait
        if (elapsed >= interval) {
            return;
        }

        // If not enough time was elapsed, wait the remainder
        long toWait = interval - elapsed;
        try {
            Thread.sleep(toWait);
        } catch (InterruptedException ie) {
            LOGGER.warn("Interrupted Exception thrown while politely waiting for remainder of interval", ie);
        }

    }

}