package com.twitter.elephantbird.mapreduce.input.combine;

import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.*;

import com.twitter.elephantbird.util.Pair;
import com.twitter.elephantbird.util.SplitUtil;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.serializer.*;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This InputSplit wraps a number of child InputSplits. Any InputSplit inserted
 * into this collection must have a public default constructor. This is based on
 * code in Hadoop, but is not available in past versions so we copy it here.
 * Also, the version in Hadoop had some bugs.
 */
public class CompositeInputSplit extends InputSplit implements Writable, Configurable {
  private static final Logger LOG = LoggerFactory.getLogger(CompositeInputSplit.class);

  private long totalSplitSizes = 0L;
  private List<InputSplit> splits = new ArrayList<InputSplit>();
  private Configuration conf = null;
  private String[] locations;

  public CompositeInputSplit() {
    // Writable
  }

  public CompositeInputSplit(List<InputSplit> splits) throws IOException, InterruptedException {
    for (InputSplit split : splits) {
      add(split);
    }
  }

  /**
   * Add an InputSplit to this collection.
   */
  public void add(InputSplit split) throws IOException, InterruptedException {
    splits.add(split);
    totalSplitSizes += split.getLength();
    locations = null;
  }

  public List<InputSplit> getSplits() {
    return splits;
  }

  /**
   * Get ith child InputSplit.
   */
  public InputSplit get(int i) {
    return splits.get(i);
  }

  /**
   * Return the aggregate length of all child InputSplits currently added.
   */
  public long getLength() {
    return totalSplitSizes;
  }

  /**
   * Get the length of ith child InputSplit.
   */
  public long getLength(int i) throws IOException, InterruptedException {
    return splits.get(i).getLength();
  }

  /**
   * Collect a set of hosts from all child InputSplits. Note that this is just a hint to the MapReduce framework
   * for where the task should go, so we return the top 5, as returning too many can be expensive on the MR side
   * of things.
   */
  public String[] getLocations() throws IOException, InterruptedException {
    if (locations == null) {
      Map<String, Integer> hosts = new HashMap<String, Integer>();
      for (InputSplit s : splits) {
        String[] hints = s.getLocations();
        if (hints != null) {
          for (String host : hints) {
            Integer value = hosts.get(host);
            if (value == null) {
              value = 0;
            }
            value++;
            hosts.put(host, value);
          }
        }
      }
      if (hosts.size() < 5) {
        locations = hosts.keySet().toArray(new String[hosts.size()]);
      } else {
        Queue<Pair<String, Integer>> queue = new PriorityQueue<Pair<String, Integer>>(hosts.size(), new Comparator<Pair<String, Integer>>() {
          public int compare(Pair<String, Integer> o1, Pair<String, Integer> o2) {
            return -o1.getSecond().compareTo(o2.getSecond());
          }
        });
        for (Map.Entry<String, Integer> entry : hosts.entrySet()) {
          queue.add(new Pair<String, Integer>(entry.getKey(), entry.getValue()));
        }
        locations = new String[] {
          queue.remove().getFirst(),
          queue.remove().getFirst(),
          queue.remove().getFirst(),
          queue.remove().getFirst(),
          queue.remove().getFirst()
        };
      }
    }
    return locations;
  }

  /**
   * getLocations from ith InputSplit.
   */
  public String[] getLocation(int i) throws IOException, InterruptedException {
    return splits.get(i).getLocations();
  }

  /**
   * Write splits in the following format.
   * {@code
   * <count><class1><split1><class2><split2>...<classn><splitn>
   * }
   */
  @SuppressWarnings("unchecked")
  public void write(DataOutput out) throws IOException {
    if (conf == null) {
      throw new IOException("Hadoop Configuration not set via setConf");
    }
    WritableUtils.writeVInt(out, splits.size());
    for (InputSplit split : splits) {
      SplitUtil.serializeInputSplit(conf, (DataOutputStream) out, split);
    }
  }

  /**
   * {@inheritDoc}
   * @throws IOException If the child InputSplit cannot be read, typically
   *                     for failing access checks.
   */
  @SuppressWarnings("unchecked")  // Generic array assignment
  public void readFields(DataInput in) throws IOException {
    if (conf == null) {
      throw new IOException("Hadoop Configuration not set via setConf");
    }
    int card = WritableUtils.readVInt(in);
    splits = new ArrayList<InputSplit>();
    for (int i = 0; i < card; ++i) {
      splits.add(SplitUtil.deserializeInputSplit(conf, (DataInputStream) in));
    }
  }

  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
  }

  @Override
  public Configuration getConf() {
    return conf;
  }

  @Override
  public String toString() {
    return "CompositeInputSplit(totalSplitSizes=" + totalSplitSizes + ", splits=" + splits + ")";
  }
}


package com.twitter.elephantbird.pig.util;

/**
 * A helper class to deal with standard Pig tokens and delimiters.
 */
public class PigTokenHelper {
  public static final byte DEFAULT_RECORD_DELIMITER = '\n';
  public static final byte DEFAULT_FIELD_DELIMITER = '\t';
  public static final String DEFAULT_FIELD_DELIMITER_STRING = "\\t";

  // String constants for each delimiter
  public static final String TUPLE_BEGIN = "(";
  public static final String TUPLE_END = ")";
  public static final String BAG_BEGIN = "{";
  public static final String BAG_END = "}";
  public static final String MAP_BEGIN = "[";
  public static final String MAP_END = "]";
  public static final String MAP_KV = "#";

  /**
   * Parse an input delimiter string, as with PigStorage, and return the byte it represents.
   * @param inputDelimiter the string passed in from the pig script.
   * @return the corresponding byte that will serve as the field separator.
   */
  public static byte evaluateDelimiter(String inputDelimiter) {
    if (inputDelimiter.length() == 1) {
      return inputDelimiter.getBytes()[0];
    } else if (inputDelimiter.length() > 1 && inputDelimiter.charAt(0) == '\\') {
      switch (inputDelimiter.charAt(1)) {
      case 't':
        return (byte)'\t';

      case 'x':
      case 'u':
        return Integer.valueOf(inputDelimiter.substring(2)).byteValue();

      default:
        throw new IllegalArgumentException("Unknown delimiter " + inputDelimiter);
      }
    } else {
      throw new IllegalArgumentException("LzoTokenizedStorage delimeter must be a single character");
    }
  }
}


package com.twitter.elephantbird.pig.util;

import java.io.IOException;
import java.util.List;

import org.apache.pig.LoadPushDown.RequiredField;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;

/**
 * Utilities for {@link ResourceSchema} and friends.
 *
 * @author Andy Schlaikjer
 */
public final class ResourceSchemaUtil {
  /**
   * Creates a new ResourceFieldSchema which reflects data from an input RequiredField.
   *
   * @param field
   * @return new ResourceFieldSchema which reflects {@code field}.
   * @throws IOException
   */
  public static ResourceFieldSchema createResourceFieldSchema(RequiredField field)
      throws IOException {
    ResourceFieldSchema schema =
        new ResourceFieldSchema().setName(field.getAlias()).setType(field.getType());
    List<RequiredField> subFields = field.getSubFields();
    if (subFields != null && !subFields.isEmpty()) {
      ResourceFieldSchema[] subSchemaFields = new ResourceFieldSchema[subFields.size()];
      int i = 0;
      for (RequiredField subField : subFields) {
        subSchemaFields[i++] = createResourceFieldSchema(subField);
      }
      ResourceSchema subSchema = new ResourceSchema();
      subSchema.setFields(subSchemaFields);
      schema.setSchema(subSchema);
    }
    return schema;
  }

  private ResourceSchemaUtil() {
    // hide ctor
  }
}


package com.twitter.elephantbird.cascading.protobuf;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Comparator;

import com.google.protobuf.Message;

import cascading.tuple.StreamComparator;
import cascading.tuple.hadoop.io.BufferedInputStream;

/**
 * Comparator for protobufs
 * @author Ning Liang
 */
public class ProtobufComparator implements StreamComparator<BufferedInputStream>,
    Comparator<Message> {

  private static final int BUFFER_SIZE = 1024;

  private byte[] buffer = new byte[BUFFER_SIZE];
  private NoCopyByteArrayOutputStream leftBos = new NoCopyByteArrayOutputStream();
  private NoCopyByteArrayOutputStream rightBos = new NoCopyByteArrayOutputStream();

  @Override
  public int compare(BufferedInputStream leftStream, BufferedInputStream rightStream) {
    leftBos.reset();
    rightBos.reset();

    readFully(leftStream, leftBos, buffer);
    readFully(rightStream, rightBos, buffer);

    return compareByteArrays(leftBos, rightBos);
  }

  @Override
  public int compare(Message left, Message right) {
    try {
      leftBos.reset();
      rightBos.reset();
      left.writeTo(leftBos);
      right.writeTo(rightBos);
      return compareByteArrays(leftBos, rightBos);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  /**
   * Method used to read a protobuf input stream into a byte array
   * @param in the protobuf input stream
   * @param out the byte output stream
   * @param buffer the array out output bytes
   */
  public static void readFully(InputStream in, ByteArrayOutputStream out, byte[] buffer) {
    try {
      int numRead = 0;
      while ((numRead = in.read(buffer, 0, buffer.length)) != -1) {
        out.write(buffer, 0, numRead);
      }
      out.flush();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  private static int compareByteArrays(NoCopyByteArrayOutputStream leftBos,
      NoCopyByteArrayOutputStream rightBos) {
    byte[] left = leftBos.getByteArray();
    byte[] right = rightBos.getByteArray();

    int leftCount = leftBos.getCount();
    int rightCount = rightBos.getCount();

    if (leftCount != rightCount) {
      return leftCount - rightCount;
    } else {
      for (int i = 0; i < leftCount; i++) {
        if (left[i] != right[i]) {
          return left[i] - right[i];
        }
      }
      return 0;
    }
  }

  /**
   * Private class that holds a no copy version of a ByteArrayOutputStream
   */
  private static class NoCopyByteArrayOutputStream extends ByteArrayOutputStream {
    public byte[] getByteArray() {
      return buf;
    }

    public int getCount() {
      return count;
    }
  }

}


package com.twitter.elephantbird.mapreduce.input;

import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;

import com.twitter.elephantbird.mapreduce.input.MapredInputFormatCompatible;
import com.twitter.elephantbird.mapreduce.io.BinaryConverter;
import com.twitter.elephantbird.mapreduce.io.BinaryWritable;
import com.twitter.elephantbird.mapreduce.io.DecodeException;
import com.twitter.elephantbird.util.Codecs;
import com.twitter.elephantbird.util.HadoopCompat;
import com.twitter.elephantbird.util.HadoopUtils;
import com.twitter.elephantbird.util.TypeRef;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.LineReader;

/**
 * Reads line from an lzo compressed text file, base64 decodes it, and then
 * deserializes that into the templatized object.
 *
 *

 * A small fraction of bad records are tolerated. See {@link LzoRecordReader}
 * for more information on error handling.
 */
public class  LzoBinaryB64LineRecordReader<M, W extends BinaryWritable<M>>
    extends LzoRecordReader<LongWritable, W> implements MapredInputFormatCompatible<LongWritable, W> {

  private LineReader lineReader_;

  private final Text line_ = new Text();
  private LongWritable key_ = new LongWritable();
  private W value_;
  private TypeRef<M> typeRef_;
  private int maxLineLen = Integer.MAX_VALUE;

  private final BinaryConverter<M> converter_;

  private Counter linesReadCounter;
  private Counter emptyLinesCounter;
  private Counter recordsReadCounter;
  private Counter recordErrorsCounter;
  private Counter truncatedLinesCounter;
  private Counter recordsSkippedCounter;

  protected LzoBinaryB64LineRecordReader(TypeRef<M> typeRef, W protobufWritable, BinaryConverter<M> protoConverter) {
    typeRef_ = typeRef;
    converter_ = protoConverter;
    value_ = protobufWritable;
  }

  @Override
  public synchronized void close() throws IOException {
    super.close();
    if (lineReader_ != null) {
      lineReader_.close();
    }
  }

  @Override
  public LongWritable getCurrentKey() throws IOException, InterruptedException {
    return key_;
  }

  @Override
  public W getCurrentValue() throws IOException, InterruptedException {
    return value_;
  }

  @Override
  protected void createInputReader(InputStream input, Configuration conf) throws IOException {
    maxLineLen = conf.getInt(LzoLineRecordReader.MAX_LINE_LEN_CONF, Integer.MAX_VALUE);
    lineReader_ = new LineReader(input, conf);
  }

  @Override
  public void initialize(InputSplit genericSplit, TaskAttemptContext context)
                                      throws IOException, InterruptedException {
    String group = "LzoB64Lines of " + typeRef_.getRawClass().getName();
    linesReadCounter = HadoopUtils.getCounter(context, group, "Lines Read");
    recordsReadCounter = HadoopUtils.getCounter(context, group, "Records Read");
    recordErrorsCounter = HadoopUtils.getCounter(context, group, "Errors");
    emptyLinesCounter = HadoopUtils.getCounter(context, group, "Empty Lines");
    truncatedLinesCounter = HadoopUtils.getCounter(context, group, "Truncated Lines");
    recordsSkippedCounter = HadoopUtils.getCounter(context, group, "Records Skipped by Converter");
    super.initialize(genericSplit, context);
  }

  @Override
  protected void skipToNextSyncPoint(boolean atFirstRecord) throws IOException {
    if (!atFirstRecord) {
      lineReader_.readLine(new Text(), maxLineLen);
    }
  }

  @Override
  public void setKeyValue(LongWritable key, W value) {
    key_ = key;
    value_ = value;
  }

  /**
   * Read the next key, value pair.
   *

   * A small fraction of bad records in input are tolerated.
   * See  {@link LzoRecordReader} for more information on error handling.
   *
   * @return true if a key/value pair was read
   * @throws IOException
   * @throws InterruptedException
   */
  @Override
  public boolean nextKeyValue() throws IOException, InterruptedException {
    // Since the lzop codec reads everything in lzo blocks, we can't stop if pos == end.
    // Instead we wait for the next block to be read in, when pos will be > end.
    while (pos_ <= end_) {
      key_.set(pos_);

      int newSize = lineReader_.readLine(line_, maxLineLen);
      if (newSize == 0) {
        return false;
      }
      HadoopCompat.incrementCounter(linesReadCounter, 1);
      pos_ = getLzoFilePos();
      if (line_.getLength() == 0 || line_.charAt(0) == '\n') {
        HadoopCompat.incrementCounter(emptyLinesCounter, 1);
        continue;
      }
      if (line_.getLength() >= maxLineLen) {
        HadoopCompat.incrementCounter(truncatedLinesCounter, 1);
      }

      M protoValue = null;

      errorTracker.incRecords();

      try {
        protoValue = converter_.fromBytes(Base64Codec.decodeFast(line_.getBytes(), line_.getLength()));
      } catch(DecodeException t1) {
        HadoopCompat.incrementCounter(recordErrorsCounter, 1);
        errorTracker.incErrors(t1);
      }

      if (protoValue == null) {
        HadoopCompat.incrementCounter(recordsSkippedCounter, 1);
        continue;
      }

      HadoopCompat.incrementCounter(recordsReadCounter, 1);
      value_.set(protoValue);
      return true;
    }

    return false;
  }
}


package com.twitter.elephantbird.pig.load;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.twitter.elephantbird.mapreduce.input.LzoTextInputFormat;

/**
 * Serves as the base class for all regex-based lzo loading functions.  Each deriving
 * class just needs to implement getPattern in some way.
 */

public abstract class LzoBaseRegexLoader extends LzoBaseLoadFunc {
  private static final Logger LOG = LoggerFactory.getLogger(LzoBaseRegexLoader.class);

  protected static final TupleFactory tupleFactory_ = TupleFactory.getInstance();
  protected static final Charset UTF8 = Charset.forName("UTF-8");
  protected static final byte RECORD_DELIMITER = (byte)'\n';
  // This loader tracks the number of matched and unmatched lines as Hadoop counters.
  protected enum LzoBaseRegexLoaderCounters { MatchedRegexLines, UnmatchedRegexLines }

  public LzoBaseRegexLoader() {
    LOG.info("LzoBaseRegexLoader created.");
  }

  /**
   * The regex pattern must be filled out by the inheritor.
   */
  public abstract Pattern getPattern();

  /**
   * Read the file line by line, returning lines the match the regex in Tuples
   * based on the regex match groups.
   */
  @Override
  public Tuple getNext() throws IOException {
	  if (reader == null) {
		  return null;
	  }

	  Pattern pattern = getPattern();
	  Matcher matcher = pattern.matcher("");
	  Object lineObj;
	  String line;
	  Tuple t = null;
	  // Read lines until a match is found, making sure there's no reading past the
	  // end of the assigned byte range.
	  try {
		  while (reader.nextKeyValue()) {

			  lineObj = reader.getCurrentValue();

			  if (lineObj == null) {
				  break;
			  }
			  line = lineObj.toString();
			  matcher = matcher.reset(line);
			  // Increment counters for the number of matched and unmatched lines.
			  if (matcher.find()) {

				  incrCounter(LzoBaseRegexLoaderCounters.MatchedRegexLines, 1L);
				  t = tupleFactory_.newTuple(matcher.groupCount());
				  for (int i = 1; i <= matcher.groupCount(); i++) {
					  if(matcher.group(i) != null) {
						  t.set(i - 1, matcher.group(i));
					  } else {
						  t.set(i - 1, "");
					  }
				  }
				  break;
			  } else {
				  incrCounter(LzoBaseRegexLoaderCounters.UnmatchedRegexLines, 1L);
				  // TODO: stop doing this, as it can slow down the job.
				  LOG.debug("No match for line " + line);
			  }

			  // If the read has walked beyond the end of the split, move on.

		  }
	  } catch (InterruptedException e) {
		  int errCode = 6018;
		  String errMsg = "Error while reading input";
		  throw new ExecException(errMsg, errCode,
				  PigException.REMOTE_ENVIRONMENT, e);
	  }

	  return t;
  }

  @Override
  public void setLocation(String location, Job job)
  throws IOException {
	  FileInputFormat.setInputPaths(job, location);
  }

  @Override
  public InputFormat getInputFormat() {
      return new LzoTextInputFormat();
  }

}


package com.twitter.elephantbird.util;

import java.lang.reflect.InvocationTargetException;

import org.apache.commons.codec.binary.Base64;

/**
 * Various Codecs specific utilities.
 */
public final class Codecs {
    private Codecs() {

    }

    /**
     * Get a instance of standard base64 implementation from apache
     * commons-codec library
     * @return standard base64 instance
     */
    public static Base64 createStandardBase64() {
      /* with constructor Base64() in commons-codec-1.4
       * encode() inserts a newline after every 76 characters.
       * Base64(0) disables that incompatibility.
       */
        try {
            return Base64.class.getConstructor(int.class).newInstance(0);
        } catch (SecurityException e) {
        } catch (NoSuchMethodException e) {
        } catch (IllegalArgumentException e) {
        } catch (InstantiationException e) {
        } catch (IllegalAccessException e) {
        } catch (InvocationTargetException e) {
        }
        return new Base64();
    }

}


package com.twitter.elephantbird.mapreduce.input;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import com.twitter.elephantbird.mapreduce.io.ProtobufWritable;
import com.twitter.elephantbird.util.HadoopCompat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.Message;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.FieldDescriptor;
import com.google.protobuf.Message.Builder;
import com.twitter.elephantbird.util.ColumnarMetadata;
import com.twitter.elephantbird.util.RCFileUtil;
import com.twitter.elephantbird.util.Protobufs;
import com.twitter.elephantbird.util.TypeRef;

public class RCFileProtobufInputFormat extends RCFileBaseInputFormat {

  private static final Logger LOG = LoggerFactory.getLogger(RCFileProtobufInputFormat.class);

  private TypeRef<Message> typeRef;

  // for MR
  public  RCFileProtobufInputFormat() {}

  public RCFileProtobufInputFormat(TypeRef<Message> typeRef) {
    this.typeRef = typeRef;
  }

  /**
   * Stores supplied class name in configuration. This configuration is
   * read on the remote tasks to initialize the input format correctly.
   */
  public static void setClassConf(Class<? extends Message> protoClass, Configuration conf) {
    Protobufs.setClassConf(conf, RCFileProtobufInputFormat.class, protoClass);
  }

  @Override
  public RecordReader<LongWritable, Writable>
  createRecordReader(InputSplit split, TaskAttemptContext taskAttempt)
          throws IOException, InterruptedException {
    if (typeRef == null) {
      typeRef = Protobufs.getTypeRef(HadoopCompat.getConfiguration(taskAttempt),
                                     RCFileProtobufInputFormat.class);
    }
    return new ProtobufReader(createUnwrappedRecordReader(split, taskAttempt));
  }

  public class ProtobufReader extends FilterRecordReader<LongWritable, Writable> {

    protected Message               msgInstance;
    protected Builder               msgBuilder;
    protected boolean               readUnknownsColumn = false;
    protected List<FieldDescriptor> knownRequiredFields = Lists.newArrayList();
    protected ArrayList<Integer>    columnsBeingRead = Lists.newArrayList();

    protected ProtobufWritable<Message> protoWritable;

    /**
     * The reader is expected to be a
     * RecordReader< LongWritable, BytesRefArrayWritable >.
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    public ProtobufReader(RecordReader reader) {
      super(reader);
    }

    /** is valid only after initialize() is called */
    public boolean isReadingUnknonwsColumn() {
      return readUnknownsColumn;
    }

    @Override
    public void initialize(InputSplit split, TaskAttemptContext ctx)
                           throws IOException, InterruptedException {
      /* set up columns that needs to be read from the RCFile.
       * if any of the required fields is not among stored columns,
       * read the the "unknowns" column (the last column).
      */
      msgBuilder = Protobufs.getMessageBuilder(typeRef.getRawClass());
      msgInstance = msgBuilder.getDefaultInstanceForType();

      protoWritable = ProtobufWritable.newInstance(typeRef.getRawClass());

      Descriptor msgDesc = msgBuilder.getDescriptorForType();
      final List<FieldDescriptor> msgFields = msgDesc.getFields();

      // set up conf to read all the columns
      Configuration conf = new Configuration(HadoopCompat.getConfiguration(ctx));
      ColumnProjectionUtils.setFullyReadColumns(conf);

      FileSplit fsplit = (FileSplit)split;
      Path file = fsplit.getPath();

      LOG.info(String.format("reading %s from %s:%d:%d"
          , typeRef.getRawClass().getName()
          , file.toString()
          , fsplit.getStart()
          , fsplit.getStart() + fsplit.getLength()));

      ColumnarMetadata storedInfo = RCFileUtil.readMetadata(conf, file);

      // list of field numbers
      List<Integer> msgFieldIds = Lists.transform(msgFields,
                                    new Function<FieldDescriptor, Integer>() {
                                       public Integer apply(FieldDescriptor fd) {
                                         return fd.getNumber();
                                       }
                                    });

      columnsBeingRead = RCFileUtil.findColumnsToRead(conf, msgFieldIds, storedInfo);

      for(int idx : columnsBeingRead) {
        int fid = storedInfo.getFieldId(idx);
        if (fid >= 0) {
          knownRequiredFields.add(msgFields.get(msgFieldIds.indexOf(fid)));
        } else {
          readUnknownsColumn = true;
        }
      }

      ColumnProjectionUtils.setReadColumnIDs(conf, columnsBeingRead);

      // finally!
      super.initialize(split, ctx);
    }

    @Override
    public Writable getCurrentValue() throws IOException, InterruptedException {
      protoWritable.set(getCurrentProtobufValue());
      return protoWritable;
    }

    /** returns super.getCurrentValue() */
    public BytesRefArrayWritable getCurrentBytesRefArrayWritable() throws IOException, InterruptedException {
      return (BytesRefArrayWritable) super.getCurrentValue();
    }

    /**
     * Builds protobuf message from the raw bytes returned by RCFile reader.
     */
    public Message getCurrentProtobufValue() throws IOException, InterruptedException {
      BytesRefArrayWritable byteRefs = getCurrentBytesRefArrayWritable();
      if (byteRefs == null) {
        return null;
      }

      Builder builder = msgInstance.newBuilderForType();

      for (int i=0; i < knownRequiredFields.size(); i++) {
        BytesRefWritable buf = byteRefs.get(columnsBeingRead.get(i));
        if (buf.getLength() > 0) {
          Protobufs.setFieldValue(
              CodedInputStream.newInstance(buf.getData(), buf.getStart(), buf.getLength()),
              knownRequiredFields.get(i),
              builder);
        }
      }

      // parse unknowns column if required
      if (readUnknownsColumn) {
        int last = columnsBeingRead.get(columnsBeingRead.size() - 1);
        BytesRefWritable buf = byteRefs.get(last);
        if (buf.getLength() > 0) {
          builder.mergeFrom(buf.getData(), buf.getStart(), buf.getLength());
        }
      }

      return builder.build();
    }
  }

 }


package com.twitter.elephantbird.mapreduce.output;

import java.io.IOException;
import java.util.List;
import java.util.Map;

import com.google.common.collect.Lists;
import com.twitter.elephantbird.util.HadoopCompat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.protocol.TProtocolUtil;
import org.apache.thrift.protocol.TType;
import org.apache.thrift.transport.TIOStreamTransport;
import org.apache.thrift.transport.TMemoryInputTransport;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
import com.twitter.elephantbird.thrift.TStructDescriptor;
import com.twitter.elephantbird.thrift.TStructDescriptor.Field;
import com.twitter.elephantbird.util.ColumnarMetadata;
import com.twitter.elephantbird.util.Protobufs;
import com.twitter.elephantbird.util.ThriftUtils;
import com.twitter.elephantbird.util.TypeRef;

/**
 * OutputFormat for storing Thrift objects in RCFile.

 *
 * Each of the top level fields is stored in a separate column.
 * Thrift field ids are stored in RCFile metadata.

 *
 * The user can write either a {@link ThriftWritable} with the Thrift object
 * or a {@link BytesWritable} with serialized Thrift bytes. The latter
 * ensures that all the fields are preserved even if the current Thrift
 * definition does not match the definition represented in the serialized bytes.
 * Any fields not recognized by current Thrift class are stored in the last
 * column.
 */
public class RCFileThriftOutputFormat extends RCFileOutputFormat {

  // typeRef is only required for setting metadata for the RCFile
  private TypeRef<? extends TBase<?, ?>> typeRef;
  private TStructDescriptor tDesc;
  private List<Field> tFields;
  private int numColumns;

  private BytesRefArrayWritable rowWritable = new BytesRefArrayWritable();
  private BytesRefWritable[] colValRefs;

  /** internal, for MR use only. */
  public RCFileThriftOutputFormat() {
  }

  public RCFileThriftOutputFormat(TypeRef<? extends TBase<?, ?>> typeRef) { // for PigLoader etc.
    this.typeRef = typeRef;
    init();
  }

  private void init() {
    tDesc = TStructDescriptor.getInstance(typeRef.getRawClass());
    tFields = tDesc.getFields();
    numColumns = tFields.size() + 1; // known fields + 1 for unknown fields
    colValRefs = new BytesRefWritable[numColumns];

    for (int i = 0; i < numColumns; i++) {
      colValRefs[i] = new BytesRefWritable();
      rowWritable.set(i, colValRefs[i]);
    }
  }

  protected ColumnarMetadata makeColumnarMetadata() {

    List<Integer> fieldIds = Lists.newArrayList();
    for(Field fd : tDesc.getFields()) {
      fieldIds.add((int)fd.getFieldId());
    }
    fieldIds.add(-1); // -1 for unknown fields

    return ColumnarMetadata.newInstance(typeRef.getRawClass().getName(), fieldIds);
  }

  private class ThriftWriter extends RCFileOutputFormat.Writer {

    private ByteStream.Output byteStream = new ByteStream.Output();
    private TBinaryProtocol tProto = new TBinaryProtocol(
                                        new TIOStreamTransport(byteStream));

    // used when deserializing thrift bytes
    private Map<Short, Integer> idMap;
    private TMemoryInputTransport mTransport;
    private TBinaryProtocol skipProto;

    ThriftWriter(TaskAttemptContext job) throws IOException {
      super(RCFileThriftOutputFormat.this, job,
          Protobufs.toText(makeColumnarMetadata().getMessage()));
    }

    @Override @SuppressWarnings("unchecked")
    public void write(NullWritable key, Writable value) throws IOException, InterruptedException {
      try {
        if (value instanceof BytesWritable) {
          // TODO: handle errors
          fromBytes((BytesWritable)value);
        } else {
          fromObject((TBase<?, ?>)((ThriftWritable)value).get());
        }
      } catch (TException e) {
        // might need to tolerate a few errors.
        throw new IOException(e);
      }

      super.write(null, rowWritable);
    }

    @SuppressWarnings("unchecked")
    private void fromObject(TBase tObj)
                    throws IOException, InterruptedException, TException {

      byteStream.reset(); // reinitialize the byteStream if buffer is too large?
      int startPos = 0;

      // top level fields are split across the columns.
      for (int i=0; i < numColumns; i++) {

        if (i < (numColumns - 1)) {

          Field fd = tFields.get(i);
          ThriftUtils.writeFieldNoTag(tProto, fd, tDesc.getFieldValue(i, tObj));

        } // else { }  : no 'unknown fields' in thrift object

        colValRefs[i].set(byteStream.getData(),
                          startPos,
                          byteStream.getCount() - startPos);
        startPos = byteStream.getCount();
      }
    }

    /**
     * extract serialized bytes for each field, including unknown fields and
     * store those byes in columns.
     */
    private void fromBytes(BytesWritable bytesWritable)
                       throws IOException, InterruptedException, TException {

      if (mTransport == null) {
        initIdMap();
        mTransport = new TMemoryInputTransport();
        skipProto = new TBinaryProtocol(mTransport);
      }

      byte[] bytes = bytesWritable.getBytes();
      mTransport.reset(bytes, 0, bytesWritable.getLength());
      byteStream.reset();

      // set all the fields to null
      for(BytesRefWritable ref : colValRefs) {
        ref.set(bytes, 0, 0);
      }

      skipProto.readStructBegin();

      while (true) {
        int start = mTransport.getBufferPosition();

        TField field = skipProto.readFieldBegin();
        if (field.type == TType.STOP) {
          break;
        }

        int fieldStart = mTransport.getBufferPosition();

        // skip still creates and copies primitive objects (String, buffer, etc)
        // skipProto could override readString() and readBuffer() to avoid that.
        TProtocolUtil.skip(skipProto, field.type);

        int end = mTransport.getBufferPosition();

        Integer idx = idMap.get(field.id);

        if (idx != null && field.type == tFields.get(idx).getType()) {
          // known field
          colValRefs[idx].set(bytes, fieldStart, end-fieldStart);
        } else {
          // unknown field, copy the bytes to last column (with field id)
          byteStream.write(bytes, start, end-start);
        }
      }

      if (byteStream.getCount() > 0) {
        byteStream.write(TType.STOP);
        colValRefs[colValRefs.length-1].set(byteStream.getData(),
                                            0,
                                            byteStream.getCount());
      }
    }

    private void initIdMap() {
      idMap = Maps.newHashMap();
      for(int i=0; i<tFields.size(); i++) {
        idMap.put(tFields.get(i).getFieldId(), i);
      }
      idMap = ImmutableMap.copyOf(idMap);
    }
  }

  /**
   * Stores supplied class name in configuration. This configuration is
   * read on the remote tasks to initialize the output format correctly.
   */
  public static void setClassConf(Class<? extends TBase<?, ?> > thriftClass, Configuration conf) {
    ThriftUtils.setClassConf(conf, RCFileThriftOutputFormat.class, thriftClass);
  }

  @Override
  public RecordWriter<NullWritable, Writable>
    getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {

    if (typeRef == null) {
      typeRef = ThriftUtils.getTypeRef(HadoopCompat.getConfiguration(job), RCFileThriftOutputFormat.class);
      init();
    }

    RCFileOutputFormat.setColumnNumber(HadoopCompat.getConfiguration(job), numColumns);
    return new ThriftWriter(job);
  }
}


package com.twitter.elephantbird.mapreduce.io;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
import com.google.protobuf.UninitializedMessageException;
import com.twitter.elephantbird.util.Protobufs;
import com.twitter.elephantbird.util.TypeRef;

/**
 * {@link BinaryConverter} for Protobufs
 */
public class ProtobufConverter<M extends Message> implements BinaryConverter<M> {
  private static final Logger LOG = LoggerFactory.getLogger(ProtobufConverter.class);

  private Message defaultInstance;
  private TypeRef<M> typeRef;

  // limit the number of warnings in case of serialization errors.
  private static final int MAX_WARNINGS = 100;
  private static int numWarningsLogged = 0;

  private static void logWarning(String message, Throwable t) {
    // does not need to be thread safe
    if ( numWarningsLogged < MAX_WARNINGS ) {
      LOG.info(message, t);
      numWarningsLogged++;
    }
  }

  /**
   * Returns a ProtobufConverter for a given Protobuf class.
   */
  public static <M extends Message> ProtobufConverter<M> newInstance(Class<M> protoClass) {
    return new ProtobufConverter<M>(new TypeRef<M>(protoClass){});
  }

  public static <M extends Message> ProtobufConverter<M> newInstance(TypeRef<M> typeRef) {
    return new ProtobufConverter<M>(typeRef);
  }

  public ProtobufConverter(TypeRef<M> typeRef) {
    this.typeRef = typeRef;
  }

  @Override
  public M fromBytes(byte[] messageBuffer) throws DecodeException {
    try {
      return fromBytes(messageBuffer, 0, messageBuffer.length);
    } catch (InvalidProtocolBufferException ipbe) {
      throw new DecodeException(ipbe);
    } catch (UninitializedMessageException ume) {
      throw new DecodeException(ume);
    }
  }

  @SuppressWarnings("unchecked")
  public M fromBytes(byte[] messageBuffer, int offset, int len)
      throws InvalidProtocolBufferException, UninitializedMessageException {
    if (defaultInstance == null) {
      defaultInstance = Protobufs.getMessageBuilder(typeRef.getRawClass())
                                 .getDefaultInstanceForType();
    }
    return (M) defaultInstance.newBuilderForType()
                              .mergeFrom(messageBuffer, offset, len)
                              .build();
  }

  @Override
  public byte[] toBytes(M message) {
    return message.toByteArray();
  }

  @Override
  public boolean equals(Object obj) {
    if (this == obj)
      return true;
    try {
      return typeRef.getType().equals(((ProtobufConverter<?>)obj).typeRef.getType());
    } catch (ClassCastException e) {
      return false;
    }
  }
}


package com.twitter.elephantbird.crunch;

import com.twitter.elephantbird.mapreduce.input.LzoThriftBlockInputFormat;
import org.apache.crunch.ReadableData;
import org.apache.crunch.io.FormatBundle;
import org.apache.crunch.io.ReadableSource;
import org.apache.crunch.io.impl.FileSourceImpl;
import org.apache.crunch.types.PType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.thrift.TBase;

import java.io.IOException;
import java.util.List;

/**
 * A Crunch {@code Source} for reading Thrift records from an {@link LzoThriftBlockInputFormat} file. The
 * format implements the {@code ReadableSource} interface, so records from these files can be read for
 * in-memory joins.
 */
public class LzoThriftSource<T extends TBase<?, ?>> extends FileSourceImpl<T> implements ReadableSource<T> {

  /**
   * Factory method for creating a new {@code LzoThriftSource} from a given path and Thrift
   * record class.
   *
   * @param path path to the data
   * @param thriftClass the Thrift class to read
   * @return a new {@code LzoThriftSource}
   */
  public static <S extends TBase<?, ?>> LzoThriftSource<S> at(Path path, Class<S> thriftClass) {
    return new LzoThriftSource<S>(path, EBTypes.thrifts(thriftClass));
  }

  /**
   * Factory method for creating a new {@code LzoThriftSource} from the given paths and Thrift
   * record class.
   *
   * @param paths list of paths to data
   * @param thriftClass the Thrift class to read
   * @return a new {@code LzoThriftSource}
   */
  public static <S extends TBase<?, ?>> LzoThriftSource<S> at(List<Path> paths, Class<S> thriftClass) {
    return new LzoThriftSource<S>(paths, EBTypes.thrifts(thriftClass));
  }

  private static <T> FormatBundle<LzoThriftBlockInputFormat> getBundle(PType<T> ptype) {
    return FormatBundle.forInput(LzoThriftBlockInputFormat.class)
        .set("elephantbird.class.for.MultiInputFormat", ptype.getTypeClass().getName());
  }

  public LzoThriftSource(Path path, PType<T> ptype) {
    super(path, ptype, getBundle(ptype));
  }

  public LzoThriftSource(List<Path> paths, PType<T> ptype) {
    super(paths, ptype, getBundle(ptype));
  }

  @Override
  public Iterable<T> read(Configuration conf) throws IOException {
    return read(conf, new ThriftFileReaderFactory<T>(ptype));
  }

  @Override
  public ReadableData<T> asReadable() {
    return new ThriftReadableData<T>(paths, ptype);
  }
}


package com.twitter.elephantbird.mapreduce.io;

import java.io.IOException;
import java.io.InputStream;
import java.util.List;

import com.google.protobuf.ByteString;
import com.twitter.elephantbird.util.Protobufs;
import com.twitter.elephantbird.util.StreamSearcher;

import org.apache.commons.io.input.BoundedInputStream;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A class to read blocks of binary objects like protobufs.
 */
public class BinaryBlockReader<M> {
  private static final Logger LOG = LoggerFactory.getLogger(BinaryBlockReader.class);

  // though any type of objects can be stored, each block itself is
  // stored as a protocolbuf (SerializedBlock).

  private InputStream in_;
  private final StreamSearcher searcher_;
  private final BinaryConverter<M> protoConverter_;
  private List<ByteString> curBlobs_;
  private int numLeftToReadThisBlock_ = 0;
  private boolean readNewBlocks_ = true;
  private boolean skipEmptyRecords = true; // skip any records of length zero

  public BinaryBlockReader(InputStream in, BinaryConverter<M> protoConverter) {
    this(in, protoConverter, true);
  }

  public BinaryBlockReader(InputStream in,
                              BinaryConverter<M> protoConverter,
                              boolean skipEmptyRecords) {
    in_ = in;
    protoConverter_ = protoConverter;
    searcher_ = new StreamSearcher(Protobufs.KNOWN_GOOD_POSITION_MARKER);
    this.skipEmptyRecords = skipEmptyRecords;
  }

  public BinaryConverter<M> getConverter() {
    return protoConverter_;
  }

  public void close() throws IOException {
    if (in_ != null)
      in_.close();
  }

  /**
   * Sets input stream. Sometimes the actual input stream might be created
   * away from the constructor.
   */
  public void setInputStream(InputStream in) {
    in_ = in; // not closing existing in_, normally it is null
  }

  /**
   * Returns next deserialized object. null indicates end of stream or a
   * deserialization error. Use {@link #readNext(BinaryWritable)} to
   * distinguish betwen end of stream or deserialization error.
   */
  public M readNext() throws IOException {
    byte[] blob = readNextProtoBytes();
    return blob == null ?
        null : protoConverter_.fromBytes(blob);
  }

  /**
   * Returns true if new proto object was read into writable, false other wise.
   */
  public boolean readNext(BinaryWritable<M> writable) throws IOException {
    byte[] blob = readNextProtoBytes();
    if (blob != null) {
      writable.set(protoConverter_.fromBytes(blob));
      return true;
    }
    return false;
  }

  /**
   * Return byte blob for the next proto object. null indicates end of stream;
   */
  public byte[] readNextProtoBytes() throws IOException {
    while (true) {
      if (!setupNewBlockIfNeeded()) {
        return null;
      }

      int blobIndex = curBlobs_.size() - numLeftToReadThisBlock_;
      numLeftToReadThisBlock_--;
      byte[] blob = curBlobs_.get(blobIndex).toByteArray();
      if (blob.length == 0 && skipEmptyRecords) {
        continue;
      }
      return blob;
    }
  }

  /**
   * returns true if bytes for next object are written to writable, false
   * other wise.
   */
  public boolean readNextProtoBytes(BytesWritable writable) throws IOException {
    byte[] blob = readNextProtoBytes();
    if (blob != null) {
      writable.set(blob, 0, blob.length);
      return true;
    }
    return false;
  }

  public void markNoMoreNewBlocks() {
    readNewBlocks_ = false;
  }

  public long skipToNextSyncPoint() throws IOException {
    return searcher_.search(in_);
  }

  /**
   * Finds next block marker and reads the block. If skipIfStartingOnBoundary is set
   * skips the the first block if the marker starts exactly at the current position.
   * (i.e. there were no bytes from previous block before the start of the marker).
   */
  public List<ByteString> parseNextBlock(boolean skipIfStartingOnBoundary) throws IOException {
    LOG.debug("BlockReader: none left to read, skipping to sync point");
    long skipped = skipToNextSyncPoint();
    if (skipped <= -1) {
      LOG.debug("BlockReader: SYNC point eof");
      // EOF if there are no more sync markers.
      return null;
    }

    int blockSize = readInt();
    LOG.debug("BlockReader: found sync point, next block has size " + blockSize);
    if (blockSize < 0) {
      LOG.debug("ProtobufReader: reading size after sync point eof");
      // EOF if the size cannot be read.
      return null;
    }

    if (skipIfStartingOnBoundary && skipped == Protobufs.KNOWN_GOOD_POSITION_MARKER.length) {
      // skip the current current block
      in_.skip(blockSize);
      return parseNextBlock(false);
    }

    SerializedBlock block = SerializedBlock.parseFrom(
      new BoundedInputStream(in_, blockSize),
      blockSize);

    curBlobs_ = block.getProtoBlobs();
    numLeftToReadThisBlock_ = curBlobs_.size();
    LOG.debug("ProtobufReader: number in next block is " + numLeftToReadThisBlock_);
    return curBlobs_;
  }

  private boolean setupNewBlockIfNeeded() throws IOException {
    if (numLeftToReadThisBlock_ == 0) {
      if (!readNewBlocks_) {
        // If the reader has been told not to read more blocks, stop.
        // This happens when a map boundary has been crossed in a map job, for example.
        // The goal then is to finsh reading what has been parsed, but let the next split
        // handle everything starting at the next sync point.
        return false;
      }
      curBlobs_ = parseNextBlock(false);
      if (curBlobs_ == null) {
        // If there is nothing, it likely means EOF. Signal that processing is done.
        return false;
      }
    }

    return true;
  }

  private int readInt() throws IOException {
    int b = in_.read();
    if (b == -1) {
      return -1;
    }

    return b | (in_.read() << 8) | (in_.read() << 16) | (in_.read() << 24);
  }
}


package com.twitter.elephantbird.thrift;

import org.apache.thrift.TException;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TMap;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.protocol.TSet;
import org.apache.thrift.protocol.TType;
import org.apache.thrift.transport.TTransport;

/**
 * Functionally exactly same as {@link TBinaryProtocol}. <p>
 *
 * Overwrites a few methods so that some malformed messages don't end up
 * taking excessively large amounts of cpu inside TProtocolUtil.skip().
 *
 * To obtain an instance of ThriftBinaryProtocol use {@link ThriftCompat#createBinaryProtocol(TTransport)}.
 * It will take care of cross version compatibility between thrift 0.7 and 0.9+ code.
 *
 * @see ThriftCompat
 */
public class ThriftBinaryProtocol extends AbstractThriftBinaryProtocol {

  ThriftBinaryProtocol(TTransport trans) {
    super(trans);
  }

  ThriftBinaryProtocol(TTransport trans, long stringLengthLimit, long containerLengthLimit) {
    super(trans, stringLengthLimit, containerLengthLimit);
  }

  /**
   * Ensures that an element type in a for container (List, Set, Map) is
   * a valid container.
   *
   * @param type
   * @throws TException if the type is not one of the expected type.
   */
  public static void checkContainerElemType(byte type) throws TException {
    // only valid types for an element in a container (List, Map, Set)
    // are the ones that are considered in TProtocolUtil.skip()
    switch (type) {
      case TType.BOOL:
      case TType.BYTE:
      case TType.I16:
      case TType.I32:
      case TType.I64:
      case TType.DOUBLE:
      case TType.STRING:
      case TType.STRUCT:
      case TType.MAP:
      case TType.SET:
      case TType.LIST:
        break;

      // list other known types, but not expected
      case TType.STOP:
      case TType.VOID:
      case TType.ENUM: // would be I32 on the wire
      default:
        throw new TException("Unexpected type " + type + " in a container");
    }
  }

  @Override
  public TMap readMapBegin() throws TException {
    TMap map = super.readMapBegin();
    checkContainerSize(map.size);
    checkContainerElemType(map.keyType);
    checkContainerElemType(map.valueType);
    return map;
  }

  @Override
  public TList readListBegin() throws TException {
    TList list = super.readListBegin();
    checkContainerSize(list.size);
    checkContainerElemType(list.elemType);
    return list;
  }

  @Override
  public TSet readSetBegin() throws TException {
    TSet set = super.readSetBegin();
    checkContainerSize(set.size);
    checkContainerElemType(set.elemType);
    return set;
  }

  @SuppressWarnings("serial")
  public static class Factory implements TProtocolFactory {

    public TProtocol getProtocol(TTransport trans) {
      return new ThriftBinaryProtocol(trans);
    }
  }
}


/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.twitter.elephantbird.util;

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MapContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.StatusReporter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.util.Progressable;

/*
 * This is based on ContextFactory.java from hadoop-2.0.x sources.
 */

/**
 * Utility methods to allow applications to deal with inconsistencies between
 * MapReduce Context Objects API between Hadoop 1.x and 2.x.
 */
public class HadoopCompat {

  private static final boolean useV21;

  private static final Constructor<?> JOB_CONTEXT_CONSTRUCTOR;
  private static final Constructor<?> TASK_CONTEXT_CONSTRUCTOR;
  private static final Constructor<?> MAP_CONTEXT_CONSTRUCTOR;
  private static final Constructor<?> GENERIC_COUNTER_CONSTRUCTOR;

  private static final Field READER_FIELD;
  private static final Field WRITER_FIELD;

  private static final Method GET_CONFIGURATION_METHOD;
  private static final Method SET_STATUS_METHOD;
  private static final Method GET_COUNTER_METHOD;
  private static final Method GET_COUNTER_ENUM_METHOD;
  private static final Method INCREMENT_COUNTER_METHOD;
  private static final Method GET_COUNTER_VALUE_METHOD;
  private static final Method GET_TASK_ATTEMPT_ID;

  private static final Method GET_JOB_ID_METHOD;
  private static final Method GET_JOB_NAME_METHOD;

  private static final Method GET_INPUT_SPLIT_METHOD;
  private static final Method GET_DEFAULT_BLOCK_SIZE_METHOD;
  private static final Method GET_DEFAULT_REPLICATION_METHOD;

  //TODO : private static final Method H2_IS_FILE_CLOSED_METHOD;

  static {
    boolean v21 = true;
    final String PACKAGE = "org.apache.hadoop.mapreduce";
    try {
      Class.forName(PACKAGE + ".task.JobContextImpl");
    } catch (ClassNotFoundException cnfe) {
      v21 = false;
    }
    useV21 = v21;
    Class<?> jobContextCls;
    Class<?> taskContextCls;
    Class<?> taskIOContextCls;
    Class<?> mapContextCls;
    Class<?> genericCounterCls;
    try {
      if (v21) {
        jobContextCls =
          Class.forName(PACKAGE+".task.JobContextImpl");
        taskContextCls =
          Class.forName(PACKAGE+".task.TaskAttemptContextImpl");
        taskIOContextCls =
          Class.forName(PACKAGE+".task.TaskInputOutputContextImpl");
        mapContextCls = Class.forName(PACKAGE + ".task.MapContextImpl");
        genericCounterCls = Class.forName(PACKAGE+".counters.GenericCounter");
      } else {
        jobContextCls =
          Class.forName(PACKAGE+".JobContext");
        taskContextCls =
          Class.forName(PACKAGE+".TaskAttemptContext");
        taskIOContextCls =
          Class.forName(PACKAGE+".TaskInputOutputContext");
        mapContextCls = Class.forName(PACKAGE + ".MapContext");
        genericCounterCls =
            Class.forName("org.apache.hadoop.mapred.Counters$Counter");
      }
    } catch (ClassNotFoundException e) {
      throw new IllegalArgumentException("Can't find class", e);
    }
    try {
      JOB_CONTEXT_CONSTRUCTOR =
        jobContextCls.getConstructor(Configuration.class, JobID.class);
      JOB_CONTEXT_CONSTRUCTOR.setAccessible(true);
      TASK_CONTEXT_CONSTRUCTOR =
        taskContextCls.getConstructor(Configuration.class,
                                      TaskAttemptID.class);
      TASK_CONTEXT_CONSTRUCTOR.setAccessible(true);
      GENERIC_COUNTER_CONSTRUCTOR =
          genericCounterCls.getDeclaredConstructor(String.class,
                                                   String.class,
                                                   Long.TYPE);
      GENERIC_COUNTER_CONSTRUCTOR.setAccessible(true);

      if (useV21) {
        MAP_CONTEXT_CONSTRUCTOR =
          mapContextCls.getDeclaredConstructor(Configuration.class,
                                               TaskAttemptID.class,
                                               RecordReader.class,
                                               RecordWriter.class,
                                               OutputCommitter.class,
                                               StatusReporter.class,
                                               InputSplit.class);
        Method get_counter;
        try {
          get_counter = TaskAttemptContext.class.getMethod("getCounter", String.class, String.class);
        } catch (Exception e) {
          get_counter = TaskInputOutputContext.class.getMethod("getCounter", String.class, String.class);
        }

        GET_COUNTER_METHOD = get_counter;
        GET_COUNTER_ENUM_METHOD         = TaskAttemptContext.class.getMethod("getCounter", Enum.class);
        GET_DEFAULT_BLOCK_SIZE_METHOD   = FileSystem.class.getMethod("getDefaultBlockSize", Path.class);
        GET_DEFAULT_REPLICATION_METHOD  = FileSystem.class.getMethod("getDefaultReplication", Path.class);

      } else {
        MAP_CONTEXT_CONSTRUCTOR =
               mapContextCls.getConstructor(Configuration.class,
                                            TaskAttemptID.class,
                                            RecordReader.class,
                                            RecordWriter.class,
                                            OutputCommitter.class,
                                            StatusReporter.class,
                                            InputSplit.class);

        GET_COUNTER_METHOD      = TaskInputOutputContext.class.getMethod("getCounter",
                                                                  String.class, String.class);
        GET_COUNTER_ENUM_METHOD = TaskInputOutputContext.class.getMethod("getCounter", Enum.class);
        GET_DEFAULT_BLOCK_SIZE_METHOD   = FileSystem.class.getMethod("getDefaultBlockSize");
        GET_DEFAULT_REPLICATION_METHOD  = FileSystem.class.getMethod("getDefaultReplication");
      }

      MAP_CONTEXT_CONSTRUCTOR.setAccessible(true);
      READER_FIELD = mapContextCls.getDeclaredField("reader");
      READER_FIELD.setAccessible(true);
      WRITER_FIELD = taskIOContextCls.getDeclaredField("output");
      WRITER_FIELD.setAccessible(true);

      GET_CONFIGURATION_METHOD  = JobContext.class        .getMethod("getConfiguration");
      SET_STATUS_METHOD         = TaskAttemptContext.class.getMethod("setStatus", String.class);
      GET_TASK_ATTEMPT_ID       = TaskAttemptContext.class.getMethod("getTaskAttemptID");
      INCREMENT_COUNTER_METHOD  = Counter.class           .getMethod("increment", Long.TYPE);
      GET_COUNTER_VALUE_METHOD  = Counter.class           .getMethod("getValue");
      GET_JOB_ID_METHOD         = JobContext.class        .getMethod("getJobID");
      GET_JOB_NAME_METHOD       = JobContext.class        .getMethod("getJobName");
      GET_INPUT_SPLIT_METHOD    = MapContext.class        .getMethod("getInputSplit");

    } catch (SecurityException e) {
      throw new IllegalArgumentException("Can't run constructor ", e);
    } catch (NoSuchMethodException e) {
      throw new IllegalArgumentException("Can't find constructor ", e);
    } catch (NoSuchFieldException e) {
      throw new IllegalArgumentException("Can't find field ", e);
    }
  }

  /**
   * True if runtime Hadoop version is 2.x, false otherwise.
   */
  public static boolean isVersion2x() {
    return useV21;
  }

  private static Object newInstance(Constructor<?> constructor, Object...args) {
    try {
      return constructor.newInstance(args);
    } catch (InstantiationException e) {
      throw new IllegalArgumentException("Can't instantiate " + constructor, e);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("Can't instantiate " + constructor, e);
    } catch (InvocationTargetException e) {
      throw new IllegalArgumentException("Can't instantiate " + constructor, e);
    }
  }

  /**
   * Creates JobContext from a JobConf and jobId using the correct constructor
   * for based on Hadoop version. jobId could be null.
   */
  public static JobContext newJobContext(Configuration conf, JobID jobId) {
    return (JobContext) newInstance(JOB_CONTEXT_CONSTRUCTOR, conf, jobId);
  }

  /**
   * Creates TaskAttempContext from a JobConf and jobId using the correct
   * constructor for based on Hadoop version.
   */
  public static TaskAttemptContext newTaskAttemptContext(
                            Configuration conf, TaskAttemptID taskAttemptId) {
    return (TaskAttemptContext)
        newInstance(TASK_CONTEXT_CONSTRUCTOR, conf, taskAttemptId);
  }

  /**
   * Instantiates MapContext under Hadoop 1 and MapContextImpl under Hadoop 2.
   */
  public static MapContext newMapContext(Configuration conf,
                                         TaskAttemptID taskAttemptID,
                                         RecordReader recordReader,
                                         RecordWriter recordWriter,
                                         OutputCommitter outputCommitter,
                                         StatusReporter statusReporter,
                                         InputSplit inputSplit) {
    return (MapContext) newInstance(MAP_CONTEXT_CONSTRUCTOR,
        conf, taskAttemptID, recordReader, recordWriter, outputCommitter,
        statusReporter, inputSplit);
  }

  /**
   * @return with Hadoop 2 : new GenericCounter(args),

   *         with Hadoop 1 : new Counter(args)
   */
  public static Counter newGenericCounter(String name, String displayName, long value) {
    try {
      return (Counter)
          GENERIC_COUNTER_CONSTRUCTOR.newInstance(name, displayName, value);
    } catch (InstantiationException e) {
      throw new IllegalArgumentException("Can't instantiate Counter", e);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("Can't instantiate Counter", e);
    } catch (InvocationTargetException e) {
      throw new IllegalArgumentException("Can't instantiate Counter", e);
    }
  }

  /**
   * Invokes a method and rethrows any exception as runtime excetpions.
   */
  private static Object invoke(Method method, Object obj, Object... args) {
    try {
      return method.invoke(obj, args);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException("Can't invoke method " + method.getName(), e);
    } catch (InvocationTargetException e) {
      throw new IllegalArgumentException("Can't invoke method " + method.getName(), e);
    }
  }

  /**
   * Invoke getConfiguration() on JobContext. Works with both
   * Hadoop 1 and 2.
   */
  public static Configuration getConfiguration(JobContext context) {
    return (Configuration) invoke(GET_CONFIGURATION_METHOD, context);
  }

  /**
   * Invoke setStatus() on TaskAttemptContext. Works with both
   * Hadoop 1 and 2.
   */
  public static void setStatus(TaskAttemptContext context, String status) {
    invoke(SET_STATUS_METHOD, context, status);
  }

  /**
   * returns TaskAttemptContext.getTaskAttemptID(). Works with both
   * Hadoop 1 and 2.
   */
  public static TaskAttemptID getTaskAttemptID(TaskAttemptContext taskContext) {
    return (TaskAttemptID) invoke(GET_TASK_ATTEMPT_ID, taskContext);
  }

  /**
   * Invoke getCounter() on TaskInputOutputContext. Works with both
   * Hadoop 1 and 2.
   */
  public static Counter getCounter(TaskInputOutputContext context,
                                   String groupName, String counterName) {
    return (Counter) invoke(GET_COUNTER_METHOD, context, groupName, counterName);
  }

    /**
     * Invoke TaskAttemptContext.progress(). Works with both
     * Hadoop 1 and 2.
     */
  public static void progress(TaskAttemptContext context) {
    ((Progressable)context).progress();
  }

  /**
   * Hadoop 1 & 2 compatible context.getCounter()
   * @return {@link TaskInputOutputContext#getCounter(Enum key)}
   */
  public static Counter getCounter(TaskInputOutputContext context, Enum<?> key) {
    return (Counter) invoke(GET_COUNTER_ENUM_METHOD, context, key);
  }

  /**
   * Increment the counter. Works with both Hadoop 1 and 2
   */
  public static void incrementCounter(Counter counter, long increment) {
    // incrementing a count might be called often. Might be affected by
    // cost of invoke(). might be good candidate to handle in a shim.
    // (TODO Raghu) figure out how achieve such a build with maven
    invoke(INCREMENT_COUNTER_METHOD, counter, increment);
  }

  /**
   * Hadoop 1 & 2 compatible counter.getValue()
   * @return {@link Counter#getValue()}
   */
  public static long getCounterValue(Counter counter) {
    return (Long) invoke(GET_COUNTER_VALUE_METHOD, counter);
  }

  /**
   * Hadoop 1 & 2 compatible JobContext.getJobID()
   * @return {@link JobContext#getJobID()}
   */
  public static JobID getJobID(JobContext jobContext) {
    return (JobID) invoke(GET_JOB_ID_METHOD, jobContext);
  }

  /**
   * Hadoop 1 & 2 compatible JobContext.getJobName()
   * @return {@link JobContext#getJobName()}
   */
  public static String getJobName(JobContext jobContext) {
    return (String) invoke(GET_JOB_NAME_METHOD, jobContext);
  }

  /**
   * Hadoop 1 & 2 compatible MapContext.getInputSplit();
   * @return {@link MapContext#getInputSplit()}
   */
  public static InputSplit getInputSplit(MapContext mapContext) {
    return (InputSplit) invoke(GET_INPUT_SPLIT_METHOD, mapContext);
  }

  /**
   * This method invokes getDefaultBlocksize() without path on Hadoop 1 and
   * with Path on Hadoop 2.
   *
   * Older versions of Hadoop 1 do not have
   * {@link FileSystem#getDefaultBlocksize(Path)}. This api exists in Hadoop 2
   * and recent versions of Hadoop 1. Path argument is required for viewfs filesystem
   * in Hadoop 2.

   *
   */
  public static long getDefaultBlockSize(FileSystem fs, Path path) {
    return (Long)
        (HadoopCompat.isVersion2x()
            ? invoke(GET_DEFAULT_BLOCK_SIZE_METHOD, fs, path)
            : invoke(GET_DEFAULT_BLOCK_SIZE_METHOD, fs));
  }

  /**
   * This method invokes getDefaultReplication() without path on Hadoop 1 and
   * with Path on Hadoop 2.
   *
   * Older versions of Hadoop 1 do not have
   * {@link FileSystem#getDefaultReplication(Path)}. This api exists in Hadoop 2
   * and recent versions of Hadoop 1. Path argument is required for viewfs filesystem
   * in Hadoop 2.

   *
   */
  public static short getDefaultReplication(FileSystem fs, Path path) {
    return (Short)
        (HadoopCompat.isVersion2x()
            ? invoke(GET_DEFAULT_REPLICATION_METHOD, fs, path)
            : invoke(GET_DEFAULT_REPLICATION_METHOD, fs));
  }

}


package com.twitter.elephantbird.mapreduce.io;

import org.apache.thrift.TBase;

import com.twitter.elephantbird.util.TypeRef;

/**
 * {@link BinaryWritable} for Thrift
 */
public class ThriftWritable<M extends TBase<?, ?>> extends BinaryWritable<M> {
  /**
   * Returns a ThriftWritable for a given Thrift class.
   */
  public static <M extends TBase<?, ?>> ThriftWritable<M> newInstance(Class<M> tClass) {
    return new ThriftWritable<M>(new TypeRef<M>(tClass){});
  }

  public ThriftWritable() {
    super(null, null);
  }

  public ThriftWritable(TypeRef<M> typeRef) {
    this(null, typeRef);
  }

  public ThriftWritable(M message, TypeRef<M> typeRef) {
    super(message, new ThriftConverter<M>(typeRef));
  }

  @Override
  protected BinaryConverter<M> getConverterFor(Class<M> clazz) {
    return ThriftConverter.newInstance(clazz);
  }
}


package com.twitter.elephantbird.pig.util;

import java.io.IOException;
import java.util.Map;

import com.google.common.base.Preconditions;

import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Writable;
import org.apache.pig.LoadCaster;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.joda.time.DateTime;

/**
 * Base LoadCaster implementation which simplifies conversion specification from Writable type to
 * Pig types.
 *
 * @author Andy Schlaikjer
 */
public abstract class WritableLoadCaster<W extends Writable> implements LoadCaster {
  private final DataInputBuffer buf = new DataInputBuffer();
  protected W writable;

  public WritableLoadCaster(W writable) {
    super();
    this.writable = writable;
  }

  public WritableLoadCaster() {
  }

  /**
   * Deserializes raw bytes into a Writable instance, returning that instance. This implementation
   * relies on derived classes to initialize {@link #writable} before this method is called.
   *
   * @param bytes serialized Writable data.
   * @param writable Writable instance into which data should be stored.
   * @return {@code writable}.
   * @throws IllegalArgumentException if either argument is {@code null}.
   * @throws IOException
   */
  protected <T extends Writable> T readFields(byte[] bytes, T writable) throws IOException {
    Preconditions.checkNotNull(bytes, "Input bytes are null");
    Preconditions.checkNotNull(writable, "Writable is null");
    buf.reset(bytes, bytes.length);
    writable.readFields(buf);
    return writable;
  }

  @Override
  public String bytesToCharArray(byte[] bytes) throws IOException {
    return toCharArray(writable = readFields(bytes, writable));
  }

  @Override
  public Boolean bytesToBoolean(byte[] bytes) throws IOException {
    return toBoolean(writable = readFields(bytes, writable));
  }

  @Override
  public Integer bytesToInteger(byte[] bytes) throws IOException {
    return toInteger(writable = readFields(bytes, writable));
  }

  @Override
  public Long bytesToLong(byte[] bytes) throws IOException {
    return toLong(writable = readFields(bytes, writable));
  }

  @Override
  public Float bytesToFloat(byte[] bytes) throws IOException {
    return toFloat(writable = readFields(bytes, writable));
  }

  @Override
  public Double bytesToDouble(byte[] bytes) throws IOException {
    return toDouble(writable = readFields(bytes, writable));
  }

  @Override
  public Map<String, Object> bytesToMap(byte[] bytes) throws IOException {
    return toMap(writable = readFields(bytes, writable));
  }

  public Map<String, Object> bytesToMap(byte[] bytes, ResourceFieldSchema schema) throws IOException {
    return toMap(writable = readFields(bytes, writable), schema);
  }

  @Override
  public Tuple bytesToTuple(byte[] bytes, ResourceFieldSchema schema) throws IOException {
    return toTuple(writable = readFields(bytes, writable), schema);
  }

  @Override
  public DataBag bytesToBag(byte[] bytes, ResourceFieldSchema schema) throws IOException {
    return toBag(writable = readFields(bytes, writable), schema);
  }

  @Override
  public DateTime bytesToDateTime(byte[] bytes) throws IOException {
   return toDateTime(writable = readFields(bytes, writable));
  }

  protected String toCharArray(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Boolean toBoolean(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Integer toInteger(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Long toLong(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Float toFloat(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Double toDouble(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Map<String, Object> toMap(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Map<String, Object> toMap(W writable, ResourceFieldSchema schema) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected Tuple toTuple(W writable, ResourceFieldSchema schema) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected DataBag toBag(W writable, ResourceFieldSchema schema) throws IOException {
    throw new UnsupportedOperationException();
  }

  protected DateTime toDateTime(W writable) throws IOException {
    throw new UnsupportedOperationException();
  }
}


package com.twitter.elephantbird.util;

import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;

/**
 * Represents a generic type {@code T}.
 *
 * Neil Gafter's "Super Type Token" implementation. Taken from
 * here and
 * here.
 *
 * The class is abstract so that the user is required to create a subclass, typically an anonymous one via
 * TypeRef<MyClass> typeRef = new TypeRef<MyClass>(){};
 * Note the {} at the end, which makes this an anonymous subclass.  Then calling getGenericSuperclass
 * in the constructor returns the class in this file, from which we can inspect the type parameter.
 * See here for
 * more about type erasure in Java.
 */
public abstract class TypeRef<T> {
  private final Type type_;
  private volatile Class class_;
  private volatile Constructor<?> constructor_;

  /**
   * Constructs a new generic type, deriving the generic type and class from
   * type parameter. Note that this constructor is protected, users should create
   * a (usually anonymous) subclass as shown above.
   *
   */
  protected TypeRef() {
    Type superclass = getClass().getGenericSuperclass();
    if (!(superclass instanceof ParameterizedType)) {
      throw new RuntimeException("Missing type parameter.");
    }
    ParameterizedType parameterized = (ParameterizedType) superclass;

    type_ = parameterized.getActualTypeArguments()[0];
  }

  /**
   * Constructs a new generic type, supplying the generic type
   * information and derving the class.
   *
   * @param genericType the generic type.
   * @throws IllegalArgumentException if genericType
   * is null or is neither an instance of Class or ParameterizedType whose raw
   * type is not an instance of Class.
   */
  public TypeRef(Type genericType) {
    if (genericType == null) {
      throw new IllegalArgumentException("Type must not be null");
    }

    type_ = genericType;
  }

  private static Class getClass(Type type) {
    if (type instanceof Class) {
      return (Class)type;
    } else if (type instanceof ParameterizedType) {
      ParameterizedType parameterizedType = (ParameterizedType)type;
      if (parameterizedType.getRawType() instanceof Class) {
        return (Class)parameterizedType.getRawType();
      }
    }
    throw new IllegalArgumentException("Type parameter [" + type.toString() + "] not a class or " +
              "parameterized type whose raw type is a class");
  }

  /**
   * Use the typeRef's parameter to create a new instance of the TypeRef's template parameter.
   * @return a new instance of type T
   * @throws NoSuchMethodException
   * @throws IllegalAccessException
   * @throws InvocationTargetException
   * @throws InstantiationException
   */
  @SuppressWarnings("unchecked")
  public T newInstance()
      throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
    if (constructor_ == null) {
      constructor_ = getRawClass().getConstructor();
    }

    return (T)constructor_.newInstance();
  }

  /**
   * The cheap, ugly version of the above, for when you don't want to catch 900 exceptions at
   * the calling site.
   * @return a new instance of type T.
   */
  public T safeNewInstance() {
    try {
      return newInstance();
    } catch (NoSuchMethodException e) {
      throw new IllegalArgumentException(e);
    } catch (IllegalAccessException e) {
      throw new IllegalArgumentException(e);
    } catch (InvocationTargetException e) {
      throw new IllegalArgumentException(e);
    } catch (InstantiationException e) {
      throw new IllegalArgumentException(e);
    }
  }

  /**
   * Gets underlying {@code Type} instance derived from the
   * type.
   * @return the type.
   */
  public final Type getType() {
    return type_;
  }

  /**
   * Gets underlying raw class instance derived from the
   * type.
   * @return the class.
   */
  public final Class<T> getRawClass() {
    if (class_ == null) {
      class_ = getClass(type_);
    }
    return class_;
  }
}


package com.twitter.elephantbird.pig.load;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.ResourceSchema;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.protobuf.Message;
import com.twitter.elephantbird.mapreduce.input.LzoRecordReader;
import com.twitter.elephantbird.mapreduce.input.MultiInputFormat;
import com.twitter.elephantbird.mapreduce.io.BinaryWritable;
import com.twitter.elephantbird.pig.util.PigUtil;
import com.twitter.elephantbird.pig.util.ProjectedProtobufTupleFactory;
import com.twitter.elephantbird.pig.util.ProtobufToPig;
import com.twitter.elephantbird.util.Protobufs;
import com.twitter.elephantbird.util.TypeRef;

/**
 * Loader for Protobuf objects stored as base64 lines or as binary
 * blocks. This may support more file formats in future.
 * Initialize with a String argument that represents the full classpath of the protocol buffer class to be loaded.

 */
public class ProtobufPigLoader<M extends Message> extends LzoBaseLoadFunc {
  static final Logger LOG = LoggerFactory.getLogger(ProtobufPigLoader.class);

  protected TypeRef<M> typeRef = null;
  private final ProtobufToPig protoToPig = new ProtobufToPig();
  private ProjectedProtobufTupleFactory<M> tupleTemplate = null;

  /**
  *
  * @param protoClassName full classpath to the generated Protocol Buffer to be loaded.
  */
  public ProtobufPigLoader(String protoClassName) {
    typeRef = PigUtil.getProtobufTypeRef(protoClassName);
  }

  @Override
  public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList)
                                              throws FrontendException {
    return pushProjectionHelper(requiredFieldList);
  }

  /**
   * Return every non-null line as a single-element tuple to Pig.
   *

   * A small fraction of bad records in input are tolerated.
   * See  {@link LzoRecordReader} for more information on error handling.
   */
  @Override
  public Tuple getNext() throws IOException {
    if (tupleTemplate == null) {
      tupleTemplate = new ProjectedProtobufTupleFactory<M>(typeRef, requiredFieldList);
    }

    M value = getNextBinaryValue(typeRef);
    return value != null ?
        tupleTemplate.newTuple(value) : null;
  }

  @Override
  public ResourceSchema getSchema(String filename, Job job) throws IOException {
    return new ResourceSchema(protoToPig.toSchema(Protobufs.getMessageDescriptor(typeRef.getRawClass())));
  }

  @Override
  public InputFormat<LongWritable, BinaryWritable<M>> getInputFormat() throws IOException {
    if (typeRef == null) {
      LOG.error("Protobuf class must be specified before an InputFormat can be created. Do not use the no-argument constructor.");
      throw new IllegalArgumentException("Protobuf class must be specified before an InputFormat can be created. Do not use the no-argument constructor.");
    }
    return new MultiInputFormat<M>(typeRef);
  }
}