java/openmldb-import/src/main/java/com/_4paradigm/openmldb/importer/IndexRegionBuilder.java
public Tablet.Segment buildPartialSegment(int segId, int sizeLimit) {
            Preconditions.checkArgument(sizeLimit >= 0, "sizeLimit should >=0, 0 will return a null segment");
            if (keyList == null) {
                // After the first buildPartialSegment() called, adding more key entries is useless.
                keyList = new ArrayList<>(keyEntries.keySet());
            }
            Tablet.Segment.Builder builder = Tablet.Segment.newBuilder();
            builder.setId(segId);
            int estimatedSize = BulkLoadRequestSize.segmentReservedSize;
            int i = keyIdxCursor, j = timeEntriesIdxCursor, k = timeEntryIdxCursor;
            logger.debug("start with {}-{}-{}", i, j, k);

            int entryCount = 0;
            for (; i < keyList.size(); i++) {
                String key = keyList.get(i);
                Tablet.Segment.KeyEntries.Builder keyEntriesBuilder = builder.addKeyEntriesBuilder()
                        .setKey(copyFromUtf8(key));
                // tolerant KeyEntry(repeated)
                estimatedSize += BulkLoadRequestSize.repeatedTolerance + key.length();

                List<Map<Long, Integer>> timeEntriesList = keyEntries.get(key);
                // TODO(hw): timeEntriesList.size() == tsCnt
                for (; j < timeEntriesList.size(); j++) {
                    Map<Long, Integer> timeEntries = timeEntriesList.get(j);
                    List<Long> timeList = new ArrayList<>(timeEntries.keySet());
                    if (timeList.isEmpty()) {
                        // will set KeyEntryId, so empty key entry could be skipped
                        continue;
                    }

                    Tablet.Segment.KeyEntries.KeyEntry.Builder keyEntryBuilder = keyEntriesBuilder.addKeyEntryBuilder().setKeyEntryId(j);
                    estimatedSize += BulkLoadRequestSize.repeatedTolerance + BulkLoadRequestSize.commonReservedSize;

                    for (; k < timeList.size(); k++) {
                        keyEntryBuilder.addTimeEntryBuilder().setTime(timeList.get(k)).
                                setBlockId(timeEntries.get(timeList.get(k)));
                        estimatedSize += BulkLoadRequestSize.estimateTimeEntrySize;
                        if (logger.isDebugEnabled()) {
                            logger.debug("add one time entry({}-{}-{}), real size {}, cur estimated size {}, limit {}"
                                    , i, j, k, builder.build().getSerializedSize(), estimatedSize, sizeLimit);
                        }

                        if (estimatedSize > sizeLimit) {
                            // revert this one
                            Preconditions.checkState(keyEntryBuilder.getTimeEntryCount() > 0);
                            logger.debug("revert this time entry");
                            keyEntryBuilder.removeTimeEntry(keyEntryBuilder.getTimeEntryCount() - 1);
                            if (keyEntryBuilder.getTimeEntryCount() == 0) {
                                logger.debug("revert the whole empty key entry");
                                keyEntriesBuilder.removeKeyEntry(keyEntriesBuilder.getKeyEntryCount() - 1);
                                if (keyEntriesBuilder.getKeyEntryCount() == 0) {
                                    logger.debug("revert the whole empty key entries");
                                    builder.removeKeyEntries(builder.getKeyEntriesCount() - 1);
                                }
                            }
                            Tablet.Segment req = builder.build();
                            int realSize = req.getSerializedSize();
                            logger.debug("segment req real size {}, entry count {}", realSize, entryCount);
                            Preconditions.checkState(realSize <= sizeLimit,
                                    "real size %d must <= size limit %d", realSize, sizeLimit);
                            // next partial building starts from k
                            // If limit is too small, cursors may not incr.
                            updateIdxCursors(i, j, k);
                            return entryCount == 0 ? null : req;
                        }
                        entryCount++;
                    }
                    k = 0;
                }
                j = 0;
            }
            Preconditions.checkState(i == keyList.size());

            updateIdxCursors(i, j, k);
            return entryCount == 0 ? null : builder.build();
        }
// won't build message which size == 0, only null or at least one index entry
won't build message which size == 0, only null or at least one index entry
won ' t build message which size == 0 only null or at least one index entry
public Tablet.Segment buildPartialSegment(int segId, int sizeLimit) {
            Preconditions.checkArgument(sizeLimit >= 0, "sizeLimit should >=0, 0 will return a null segment");
            if (keyList == null) {
                keyList = new ArrayList<>(keyEntries.keySet());
            }
            Tablet.Segment.Builder builder = Tablet.Segment.newBuilder();
            builder.setId(segId);
            int estimatedSize = BulkLoadRequestSize.segmentReservedSize;
            int i = keyIdxCursor, j = timeEntriesIdxCursor, k = timeEntryIdxCursor;
            logger.debug("start with {}-{}-{}", i, j, k);
            int entryCount = 0;
            for (; i < keyList.size(); i++) {
                String key = keyList.get(i);
                Tablet.Segment.KeyEntries.Builder keyEntriesBuilder = builder.addKeyEntriesBuilder()
                        .setKey(copyFromUtf8(key));
                estimatedSize += BulkLoadRequestSize.repeatedTolerance + key.length();
                List<Map<Long, Integer>> timeEntriesList = keyEntries.get(key);
                for (; j < timeEntriesList.size(); j++) {
                    Map<Long, Integer> timeEntries = timeEntriesList.get(j);
                    List<Long> timeList = new ArrayList<>(timeEntries.keySet());
                    if (timeList.isEmpty()) {
                        continue;
                    }
                    Tablet.Segment.KeyEntries.KeyEntry.Builder keyEntryBuilder = keyEntriesBuilder.addKeyEntryBuilder().setKeyEntryId(j);
                    estimatedSize += BulkLoadRequestSize.repeatedTolerance + BulkLoadRequestSize.commonReservedSize;
                    for (; k < timeList.size(); k++) {
                        keyEntryBuilder.addTimeEntryBuilder().setTime(timeList.get(k)).
                                setBlockId(timeEntries.get(timeList.get(k)));
                        estimatedSize += BulkLoadRequestSize.estimateTimeEntrySize;
                        if (logger.isDebugEnabled()) {
                            logger.debug("add one time entry({}-{}-{}), real size {}, cur estimated size {}, limit {}"
                                    , i, j, k, builder.build().getSerializedSize(), estimatedSize, sizeLimit);
                        }
                        if (estimatedSize > sizeLimit) {
                            Preconditions.checkState(keyEntryBuilder.getTimeEntryCount() > 0);
                            logger.debug("revert this time entry");
                            keyEntryBuilder.removeTimeEntry(keyEntryBuilder.getTimeEntryCount() - 1);
                            if (keyEntryBuilder.getTimeEntryCount() == 0) {
                                logger.debug("revert the whole empty key entry");
                                keyEntriesBuilder.removeKeyEntry(keyEntriesBuilder.getKeyEntryCount() - 1);
                                if (keyEntriesBuilder.getKeyEntryCount() == 0) {
                                    logger.debug("revert the whole empty key entries");
                                    builder.removeKeyEntries(builder.getKeyEntriesCount() - 1);
                                }
                            }
                            Tablet.Segment req = builder.build();
                            int realSize = req.getSerializedSize();
                            logger.debug("segment req real size {}, entry count {}", realSize, entryCount);
                            Preconditions.checkState(realSize <= sizeLimit,
                                    "real size %d must <= size limit %d", realSize, sizeLimit);
                            updateIdxCursors(i, j, k);
                            return entryCount == 0 ? null : req;
                        }
                        entryCount++;
                    }
                    k = 0;
                }
                j = 0;
            }
            Preconditions.checkState(i == keyList.size());
            updateIdxCursors(i, j, k);
            return entryCount == 0 ? null : builder.build();
        }
public Tablet . Segment buildPartialSegment ( int segId , int sizeLimit ) { Preconditions . checkArgument ( sizeLimit >= 0 , "sizeLimit should >=0, 0 will return a null segment" ) ; if ( keyList == null ) { keyList = new ArrayList < > ( keyEntries . keySet ( ) ) ; } Tablet . Segment . Builder builder = Tablet . Segment . newBuilder ( ) ; builder . setId ( segId ) ; int estimatedSize = BulkLoadRequestSize . segmentReservedSize ; int i = keyIdxCursor , j = timeEntriesIdxCursor , k = timeEntryIdxCursor ; logger . debug ( "start with {}-{}-{}" , i , j , k ) ; int entryCount = 0 ; for ( ; i < keyList . size ( ) ; i ++ ) { String key = keyList . get ( i ) ; Tablet . Segment . KeyEntries . Builder keyEntriesBuilder = builder . addKeyEntriesBuilder ( ) . setKey ( copyFromUtf8 ( key ) ) ; estimatedSize += BulkLoadRequestSize . repeatedTolerance + key . length ( ) ; List < Map < Long , Integer > > timeEntriesList = keyEntries . get ( key ) ; for ( ; j < timeEntriesList . size ( ) ; j ++ ) { Map < Long , Integer > timeEntries = timeEntriesList . get ( j ) ; List < Long > timeList = new ArrayList < > ( timeEntries . keySet ( ) ) ; if ( timeList . isEmpty ( ) ) { continue ; } Tablet . Segment . KeyEntries . KeyEntry . Builder keyEntryBuilder = keyEntriesBuilder . addKeyEntryBuilder ( ) . setKeyEntryId ( j ) ; estimatedSize += BulkLoadRequestSize . repeatedTolerance + BulkLoadRequestSize . commonReservedSize ; for ( ; k < timeList . size ( ) ; k ++ ) { keyEntryBuilder . addTimeEntryBuilder ( ) . setTime ( timeList . get ( k ) ) . setBlockId ( timeEntries . get ( timeList . get ( k ) ) ) ; estimatedSize += BulkLoadRequestSize . estimateTimeEntrySize ; if ( logger . isDebugEnabled ( ) ) { logger . debug ( "add one time entry({}-{}-{}), real size {}, cur estimated size {}, limit {}" , i , j , k , builder . build ( ) . getSerializedSize ( ) , estimatedSize , sizeLimit ) ; } if ( estimatedSize > sizeLimit ) { Preconditions . checkState ( keyEntryBuilder . getTimeEntryCount ( ) > 0 ) ; logger . debug ( "revert this time entry" ) ; keyEntryBuilder . removeTimeEntry ( keyEntryBuilder . getTimeEntryCount ( ) - 1 ) ; if ( keyEntryBuilder . getTimeEntryCount ( ) == 0 ) { logger . debug ( "revert the whole empty key entry" ) ; keyEntriesBuilder . removeKeyEntry ( keyEntriesBuilder . getKeyEntryCount ( ) - 1 ) ; if ( keyEntriesBuilder . getKeyEntryCount ( ) == 0 ) { logger . debug ( "revert the whole empty key entries" ) ; builder . removeKeyEntries ( builder . getKeyEntriesCount ( ) - 1 ) ; } } Tablet . Segment req = builder . build ( ) ; int realSize = req . getSerializedSize ( ) ; logger . debug ( "segment req real size {}, entry count {}" , realSize , entryCount ) ; Preconditions . checkState ( realSize <= sizeLimit , "real size %d must <= size limit %d" , realSize , sizeLimit ) ; updateIdxCursors ( i , j , k ) ; return entryCount == 0 ? null : req ; } entryCount ++ ; } k = 0 ; } j = 0 ; } Preconditions . checkState ( i == keyList . size ( ) ) ; updateIdxCursors ( i , j , k ) ; return entryCount == 0 ? null : builder . build ( ) ; }
won't build message which size == 0, only null or at least one index entry
won ' t build message which size == 0 only null or at least one index entry
// After the first buildPartialSegment() called, adding more key entries is useless. // tolerant KeyEntry(repeated) // TODO(hw): timeEntriesList.size() == tsCnt // will set KeyEntryId, so empty key entry could be skipped // revert this one // next partial building starts from k // If limit is too small, cursors may not incr.