core/src/main/java/org/apache/druid/data/input/InputEntity.java
/**
   * Fetches the input entity into the local storage.
   * This method might be preferred instead of {@link #open()}, for example
   *
   * - {@link InputFormat} requires expensive random access on remote storage.
   * - Holding a connection until you consume the entire InputStream is expensive.
   *
   * @param temporaryDirectory to store temp data. This directory will be removed automatically once
   *                           the task finishes.
   * @param fetchBuffer        is used to fetch remote entity into local storage.
   *
   * @see FileUtils#copyLarge
   */
Fetches the input entity into the local storage.
This method might be preferred instead of  #open(), for example

-  InputFormat requires expensive random access on remote storage.
- Holding a connection until you consume the entire InputStream is expensive.

@param temporaryDirectory to store temp data. This directory will be removed automatically once
the task finishes.
@param fetchBuffer        is used to fetch remote entity into local storage.


Fetches the input entity into the local storage . This method might be preferred instead of #open () for example - InputFormat requires expensive random access on remote storage . - Holding a connection until you consume the entire InputStream is expensive . @param temporaryDirectory to store temp data . This directory will be removed automatically once the task finishes . @param fetchBuffer is used to fetch remote entity into local storage .
default CleanableFile fetch(File temporaryDirectory, byte[] fetchBuffer) throws IOException
  {
    final File tempFile = File.createTempFile("druid-input-entity", ".tmp", temporaryDirectory);
    LOG.debug("Fetching entity into file[%s]", tempFile.getAbsolutePath());
    final URI uri = getUri();
    final String message;
    if (uri == null) {
      message = StringUtils.format("Failed to fetch entity into local file [%s]", tempFile.getAbsolutePath());
    } else {
      message = StringUtils.format("Failed to fetch entity [%s] into local file [%s]", uri, tempFile.getAbsolutePath());
    }
    FileUtils.copyLarge(
        this::open,
        tempFile,
        fetchBuffer,
        getRetryCondition(),
        DEFAULT_MAX_NUM_FETCH_TRIES,
        message
    );
    return new CleanableFile()
    {
      @Override
      public File file()
      {
        return tempFile;
      }
      @Override
      public void close()
      {
        if (!tempFile.delete()) {
          LOG.warn("Failed to remove file[%s]", tempFile.getAbsolutePath());
        }
      }
    };
  }
default CleanableFile fetch ( File temporaryDirectory , byte [ ] fetchBuffer ) throws IOException { final File tempFile = File . createTempFile ( "druid-input-entity" , ".tmp" , temporaryDirectory ) ; LOG . debug ( "Fetching entity into file[%s]" , tempFile . getAbsolutePath ( ) ) ; final URI uri = getUri ( ) ; final String message ; if ( uri == null ) { message = StringUtils . format ( "Failed to fetch entity into local file [%s]" , tempFile . getAbsolutePath ( ) ) ; } else { message = StringUtils . format ( "Failed to fetch entity [%s] into local file [%s]" , uri , tempFile . getAbsolutePath ( ) ) ; } FileUtils . copyLarge ( this :: open , tempFile , fetchBuffer , getRetryCondition ( ) , DEFAULT_MAX_NUM_FETCH_TRIES , message ) ; return new CleanableFile ( ) { @ Override public File file ( ) { return tempFile ; } @ Override public void close ( ) { if ( ! tempFile . delete ( ) ) { LOG . warn ( "Failed to remove file[%s]" , tempFile . getAbsolutePath ( ) ) ; } } } ; }
Fetches the input entity into the local storage.
Fetches the input entity into the local storage .