plugins/transforms/dropbox/src/main/java/org/apache/hop/pipeline/transforms/dropbox/output/DropboxOutput.java
private boolean chunkedUploadFile( DbxClientV2 dbxClient, File localFile, String dropboxPath ) {
        long size = localFile.length();

        // assert our file is at least the chunk upload size. We make this assumption in the code
        // below to simplify the logic.
        if ( size < data.CHUNKED_UPLOAD_CHUNK_SIZE ) {
            log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.FileTooSmall" ) );
            return false;
        }

        long uploaded = 0L;
        DbxException thrown = null;

        IOUtil.ProgressListener progressListener = new IOUtil.ProgressListener() {
            long uploadedBytes = 0;
            @Override
            public void onProgress( long l ) {
                printProgress( l + uploadedBytes, size );
                if ( l == data.CHUNKED_UPLOAD_CHUNK_SIZE ) {
                    uploadedBytes += data.CHUNKED_UPLOAD_CHUNK_SIZE;
                }
            }
        };

        // Chunked uploads have 3 phases, each of which can accept uploaded bytes:
        //
        //    (1)  Start: initiate the upload and get an upload session ID
        //    (2) Append: upload chunks of the file to append to our session
        //    (3) Finish: commit the upload and close the session
        //
        // We track how many bytes we uploaded to determine which phase we should be in.
        String sessionId = null;
        for ( int i = 0; i < data.CHUNKED_UPLOAD_MAX_ATTEMPTS; ++i ) {
            if ( i > 0 ) {
                log.logDetailed( String.format( "Retrying chunked upload (%d / %d attempts)\n", i + 1, data.CHUNKED_UPLOAD_MAX_ATTEMPTS ) );
            }

            try ( InputStream in = new FileInputStream( localFile ) ) {
                // if this is a retry, make sure seek to the correct offset
                in.skip( uploaded );

                // (1) Start
                if ( sessionId == null ) {
                    sessionId = dbxClient.files().uploadSessionStart()
                            .uploadAndFinish( in, data.CHUNKED_UPLOAD_CHUNK_SIZE, progressListener )
                            .getSessionId();
                    uploaded += data.CHUNKED_UPLOAD_CHUNK_SIZE;
                    printProgress( uploaded, size );
                }

                UploadSessionCursor cursor = new UploadSessionCursor( sessionId, uploaded );

                // (2) Append
                while ( ( size - uploaded ) > data.CHUNKED_UPLOAD_CHUNK_SIZE ) {
                    dbxClient.files().uploadSessionAppendV2( cursor )
                            .uploadAndFinish( in, data.CHUNKED_UPLOAD_CHUNK_SIZE, progressListener );
                    uploaded += data.CHUNKED_UPLOAD_CHUNK_SIZE;
                    printProgress( uploaded, size );
                    cursor = new UploadSessionCursor( sessionId, uploaded );
                }

                // (3) Finish
                long remaining = size - uploaded;
                CommitInfo commitInfo = CommitInfo.newBuilder( dropboxPath )
                        .withMode( WriteMode.ADD )
                        .withClientModified( new Date( localFile.lastModified() ) )
                        .build();
                FileMetadata metadata = dbxClient.files().uploadSessionFinish( cursor, commitInfo )
                        .uploadAndFinish( in, remaining, progressListener );

                log.logBasic( metadata.toStringMultiline() );
                return true;
            } catch ( RetryException ex ) {
                thrown = ex;
                // RetryExceptions are never automatically retried by the client for uploads. Must
                // catch this exception even if DbxRequestConfig.getMaxRetries() > 0.
                sleepQuietly( ex.getBackoffMillis() );
                continue;
            } catch ( NetworkIOException ex ) {
                thrown = ex;
                // network issue with Dropbox (maybe a timeout?) try again
                continue;
            } catch ( UploadSessionLookupErrorException ex ) {
                if ( ex.errorValue.isIncorrectOffset() ) {
                    thrown = ex;
                    // server offset into the stream doesn't match our offset (uploaded). Seek to
                    // the expected offset according to the server and try again.
                    uploaded = ex.errorValue
                            .getIncorrectOffsetValue()
                            .getCorrectOffset();
                    continue;
                } else {
                    // Some other error occurred, give up.
                    log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.UploadError", ex.getMessage() ) );
                    return false;
                }
            } catch ( UploadSessionFinishErrorException ex ) {
                if ( ex.errorValue.isLookupFailed() && ex.errorValue.getLookupFailedValue().isIncorrectOffset() ) {
                    thrown = ex;
                    // server offset into the stream doesn't match our offset (uploaded). Seek to
                    // the expected offset according to the server and try again.
                    uploaded = ex.errorValue
                            .getLookupFailedValue()
                            .getIncorrectOffsetValue()
                            .getCorrectOffset();
                    continue;
                } else {
                    // some other error occurred, give up.
                    log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.UploadError", ex.getMessage() ) );
                    return false;
                }
            } catch ( DbxException ex ) {
                log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.UploadError", ex.getMessage() ) );
                return false;
            } catch ( IOException ex ) {
                log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.ErrorReadingFile", localFile.getName(), ex.getMessage() ) );
                return false;
            }
        }

        // if we made it here, then we must have run out of attempts
        log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.TooManyAttempts", thrown.getMessage() ) );
        return false;
    }
/**
     * Uploads a file in chunks using multiple requests. This approach is preferred for larger files
     * since it allows for more efficient processing of the file contents on the server side and
     * also allows partial uploads to be retried (e.g. network connection problem will not cause you
     * to re-upload all the bytes).
     *
     * @param dbxClient Dropbox user authenticated client
     * @param localFile local file to upload
     * @param dropboxPath Where to upload the file to within Dropbox
     */
Uploads a file in chunks using multiple requests. This approach is preferred for larger files
since it allows for more efficient processing of the file contents on the server side and
also allows partial uploads to be retried .

@param dbxClient Dropbox user authenticated client
@param localFile local file to upload
@param dropboxPath Where to upload the file to within Dropbox
Uploads a file in chunks using multiple requests . This approach is preferred for larger files since it allows for more efficient processing of the file contents on the server side and also allows partial uploads to be retried . @param dbxClient Dropbox user authenticated client @param localFile local file to upload @param dropboxPath Where to upload the file to within Dropbox
private boolean chunkedUploadFile( DbxClientV2 dbxClient, File localFile, String dropboxPath ) {
        long size = localFile.length();
        if ( size < data.CHUNKED_UPLOAD_CHUNK_SIZE ) {
            log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.FileTooSmall" ) );
            return false;
        }
        long uploaded = 0L;
        DbxException thrown = null;
        IOUtil.ProgressListener progressListener = new IOUtil.ProgressListener() {
            long uploadedBytes = 0;
            @Override
            public void onProgress( long l ) {
                printProgress( l + uploadedBytes, size );
                if ( l == data.CHUNKED_UPLOAD_CHUNK_SIZE ) {
                    uploadedBytes += data.CHUNKED_UPLOAD_CHUNK_SIZE;
                }
            }
        };
            (1)  Start: initiate the upload and get an upload session ID
            (2) Append: upload chunks of the file to append to our session
            (3) Finish: commit the upload and close the session
         We track how many bytes we uploaded to determine which phase we should be in.
        String sessionId = null;
        for ( int i = 0; i < data.CHUNKED_UPLOAD_MAX_ATTEMPTS; ++i ) {
            if ( i > 0 ) {
                log.logDetailed( String.format( "Retrying chunked upload (%d / %d attempts)\n", i + 1, data.CHUNKED_UPLOAD_MAX_ATTEMPTS ) );
            }
            try ( InputStream in = new FileInputStream( localFile ) ) {
                 if this is a retry, make sure seek to the correct offset
                in.skip( uploaded );
                 (1) Start
                if ( sessionId == null ) {
                    sessionId = dbxClient.files().uploadSessionStart()
                            .uploadAndFinish( in, data.CHUNKED_UPLOAD_CHUNK_SIZE, progressListener )
                            .getSessionId();
                    uploaded += data.CHUNKED_UPLOAD_CHUNK_SIZE;
                    printProgress( uploaded, size );
                }
                UploadSessionCursor cursor = new UploadSessionCursor( sessionId, uploaded );
                 (2) Append
                while ( ( size - uploaded ) > data.CHUNKED_UPLOAD_CHUNK_SIZE ) {
                    dbxClient.files().uploadSessionAppendV2( cursor )
                            .uploadAndFinish( in, data.CHUNKED_UPLOAD_CHUNK_SIZE, progressListener );
                    uploaded += data.CHUNKED_UPLOAD_CHUNK_SIZE;
                    printProgress( uploaded, size );
                    cursor = new UploadSessionCursor( sessionId, uploaded );
                }
                 (3) Finish
                long remaining = size - uploaded;
                CommitInfo commitInfo = CommitInfo.newBuilder( dropboxPath )
                        .withMode( WriteMode.ADD )
                        .withClientModified( new Date( localFile.lastModified() ) )
                        .build();
                FileMetadata metadata = dbxClient.files().uploadSessionFinish( cursor, commitInfo )
                        .uploadAndFinish( in, remaining, progressListener );
                log.logBasic( metadata.toStringMultiline() );
                return true;
            } catch ( RetryException ex ) {
                thrown = ex;
                 RetryExceptions are never automatically retried by the client for uploads. Must
                 catch this exception even if DbxRequestConfig.getMaxRetries() > 0.
                sleepQuietly( ex.getBackoffMillis() );
                continue;
            } catch ( NetworkIOException ex ) {
                thrown = ex;
                 network issue with Dropbox (maybe a timeout?) try again
                continue;
            } catch ( UploadSessionLookupErrorException ex ) {
                if ( ex.errorValue.isIncorrectOffset() ) {
                    thrown = ex;
                     server offset into the stream doesn't match our offset (uploaded). Seek to
                     the expected offset according to the server and try again.
                    uploaded = ex.errorValue
                            .getIncorrectOffsetValue()
                            .getCorrectOffset();
                    continue;
                } else {
                     Some other error occurred, give up.
                    log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.UploadError", ex.getMessage() ) );
                    return false;
                }
            } catch ( UploadSessionFinishErrorException ex ) {
                if ( ex.errorValue.isLookupFailed() && ex.errorValue.getLookupFailedValue().isIncorrectOffset() ) {
                    thrown = ex;
                     server offset into the stream doesn't match our offset (uploaded). Seek to
                     the expected offset according to the server and try again.
                    uploaded = ex.errorValue
                            .getLookupFailedValue()
                            .getIncorrectOffsetValue()
                            .getCorrectOffset();
                    continue;
                } else {
                     some other error occurred, give up.
                    log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.UploadError", ex.getMessage() ) );
                    return false;
                }
            } catch ( DbxException ex ) {
                log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.UploadError", ex.getMessage() ) );
                return false;
            } catch ( IOException ex ) {
                log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.ErrorReadingFile", localFile.getName(), ex.getMessage() ) );
                return false;
            }
        }
         if we made it here, then we must have run out of attempts
        log.logError( BaseMessages.getString( PKG, "DropboxOutput.Log.TooManyAttempts", thrown.getMessage() ) );
        return false;
    }
private boolean chunkedUploadFile ( DbxClientV2 dbxClient , File localFile , String dropboxPath ) { long size = localFile . length ( ) ; if ( size < data . CHUNKED_UPLOAD_CHUNK_SIZE ) { log . logError ( BaseMessages . getString ( PKG , "DropboxOutput.Log.FileTooSmall" ) ) ; return false ; } long uploaded = 0L ; DbxException thrown = null ; IOUtil . ProgressListener progressListener = new IOUtil . ProgressListener ( ) { long uploadedBytes = 0 ; @ Override public void onProgress ( long l ) { printProgress ( l + uploadedBytes , size ) ; if ( l == data . CHUNKED_UPLOAD_CHUNK_SIZE ) { uploadedBytes += data . CHUNKED_UPLOAD_CHUNK_SIZE ; } } } ; String sessionId = null ; for ( int i = 0 ; i < data . CHUNKED_UPLOAD_MAX_ATTEMPTS ; ++ i ) { if ( i > 0 ) { log . logDetailed ( String . format ( "Retrying chunked upload (%d / %d attempts)\n" , i + 1 , data . CHUNKED_UPLOAD_MAX_ATTEMPTS ) ) ; } try ( InputStream in = new FileInputStream ( localFile ) ) { in . skip ( uploaded ) ; if ( sessionId == null ) { sessionId = dbxClient . files ( ) . uploadSessionStart ( ) . uploadAndFinish ( in , data . CHUNKED_UPLOAD_CHUNK_SIZE , progressListener ) . getSessionId ( ) ; uploaded += data . CHUNKED_UPLOAD_CHUNK_SIZE ; printProgress ( uploaded , size ) ; } UploadSessionCursor cursor = new UploadSessionCursor ( sessionId , uploaded ) ; while ( ( size - uploaded ) > data . CHUNKED_UPLOAD_CHUNK_SIZE ) { dbxClient . files ( ) . uploadSessionAppendV2 ( cursor ) . uploadAndFinish ( in , data . CHUNKED_UPLOAD_CHUNK_SIZE , progressListener ) ; uploaded += data . CHUNKED_UPLOAD_CHUNK_SIZE ; printProgress ( uploaded , size ) ; cursor = new UploadSessionCursor ( sessionId , uploaded ) ; } long remaining = size - uploaded ; CommitInfo commitInfo = CommitInfo . newBuilder ( dropboxPath ) . withMode ( WriteMode . ADD ) . withClientModified ( new Date ( localFile . lastModified ( ) ) ) . build ( ) ; FileMetadata metadata = dbxClient . files ( ) . uploadSessionFinish ( cursor , commitInfo ) . uploadAndFinish ( in , remaining , progressListener ) ; log . logBasic ( metadata . toStringMultiline ( ) ) ; return true ; } catch ( RetryException ex ) { thrown = ex ; sleepQuietly ( ex . getBackoffMillis ( ) ) ; continue ; } catch ( NetworkIOException ex ) { thrown = ex ; continue ; } catch ( UploadSessionLookupErrorException ex ) { if ( ex . errorValue . isIncorrectOffset ( ) ) { thrown = ex ; uploaded = ex . errorValue . getIncorrectOffsetValue ( ) . getCorrectOffset ( ) ; continue ; } else { log . logError ( BaseMessages . getString ( PKG , "DropboxOutput.Log.UploadError" , ex . getMessage ( ) ) ) ; return false ; } } catch ( UploadSessionFinishErrorException ex ) { if ( ex . errorValue . isLookupFailed ( ) && ex . errorValue . getLookupFailedValue ( ) . isIncorrectOffset ( ) ) { thrown = ex ; uploaded = ex . errorValue . getLookupFailedValue ( ) . getIncorrectOffsetValue ( ) . getCorrectOffset ( ) ; continue ; } else { log . logError ( BaseMessages . getString ( PKG , "DropboxOutput.Log.UploadError" , ex . getMessage ( ) ) ) ; return false ; } } catch ( DbxException ex ) { log . logError ( BaseMessages . getString ( PKG , "DropboxOutput.Log.UploadError" , ex . getMessage ( ) ) ) ; return false ; } catch ( IOException ex ) { log . logError ( BaseMessages . getString ( PKG , "DropboxOutput.Log.ErrorReadingFile" , localFile . getName ( ) , ex . getMessage ( ) ) ) ; return false ; } } log . logError ( BaseMessages . getString ( PKG , "DropboxOutput.Log.TooManyAttempts" , thrown . getMessage ( ) ) ) ; return false ; }
Uploads a file in chunks using multiple requests.
Uploads a file in chunks using multiple requests .
// assert our file is at least the chunk upload size. We make this assumption in the code // below to simplify the logic. // Chunked uploads have 3 phases, each of which can accept uploaded bytes: // //    (1)  Start: initiate the upload and get an upload session ID //    (2) Append: upload chunks of the file to append to our session //    (3) Finish: commit the upload and close the session // // We track how many bytes we uploaded to determine which phase we should be in. // if this is a retry, make sure seek to the correct offset // (1) Start // (2) Append // (3) Finish // RetryExceptions are never automatically retried by the client for uploads. Must // catch this exception even if DbxRequestConfig.getMaxRetries() > 0. // network issue with Dropbox (maybe a timeout?) try again // server offset into the stream doesn't match our offset (uploaded). Seek to // the expected offset according to the server and try again. // Some other error occurred, give up. // server offset into the stream doesn't match our offset (uploaded). Seek to // the expected offset according to the server and try again. // some other error occurred, give up. // if we made it here, then we must have run out of attempts